#include "../AiDemos.hpp" #include #include "../../fw/Router.hpp" #include "../../fw/Templates.hpp" #include "llm/api_client.h" #include "llm/config.h" #include "security/crypto/Base64.h" #include "vendors/audio/assemblyai/client.h" #include "vendors/audio/assemblyai/transcripts.h" #include "vendors/audio/cartesia/client.h" #include "vendors/audio/deepgram/client.h" #include "vendors/audio/elevenlabs/client.h" #include "vendors/audio/hume/client.h" #include "vendors/audio/audio_service.h" #include "vendors/audio/cartesia/tts_socket.h" #include "vendors/audio/deepgram/streams.h" #include "vendors/audio/elevenlabs/tts_stream.h" #include "vendors/audio/hume/sockets.h" #include "vendors/audio/providers.h" #include "vendors/audio/speech.h" #include "vendors/audio/assemblyai/stream.h" #include "vendors/audio/elevenlabs/scribe_stream.h" #include "vendors/audio/speechmatics/batch.h" #include "vendors/audio/speechmatics/realtime.h" namespace molo::example::app { // audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics and Hume // through src/vendors/audio. GET routes need no credential. Every route that // calls a provider is a POST: 503 not_configured without the provider's key, // 422 on bad input. Anything that generates audio, writes, or deletes is a // dry run unless the body says "confirm":true. namespace { namespace au = audio; using Json = clients::body::Json; constexpr std::size_t kMaxBody = 4u * 1024u * 1024u; constexpr std::size_t kMaxAudioEcho = 8u * 1024u * 1024u; // audio returned to the browser as base64 const char kPage[] = R"AUDIOP({% extends "base.html" %} {% block content %}

Audio

Speech synthesis and recognition across ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics and Hume. Anything that generates audio, writes or deletes is a dry run until the body says "confirm":true.

Checking provider keys...


{% endblock %} )AUDIOP"; fw::Response json_out(const Json& j, int status = 200) { fw::Response res = fw::JsonResponse(j.dump(), status); res.headers.set("Cache-Control", "no-store"); return res; } [[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } template auto guarded(F&& f) -> decltype(f()) { try { return f(); } catch (const llm::NotConfiguredError& e) { throw fw::HttpError(503, "not_configured", e.message()); } catch (const llm::ApiError& e) { if (e.http_status() == 400 && e.type() == "INVALID_ARGUMENT") bad(e.message()); throw fw::HttpError(502, "audio_error", "HTTP " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); } catch (const clients::body::JsonError&) { bad("body is not valid JSON"); } } class Body { public: explicit Body(const fw::Request& req) { if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); if (req.body.empty()) return; try { doc_ = Json::parse(req.body); } catch (const clients::body::JsonError&) { bad("body is not valid JSON"); } if (!doc_.is_object()) bad("body must be a JSON object"); } std::string text(const char* key, bool required = false) const { const Json* v = doc_.find(key); if (!v || v->is_null()) { if (required) bad(std::string(key) + " is required"); return ""; } if (!v->is_string()) bad(std::string(key) + " must be a string"); if (required && v->as_string().empty()) bad(std::string(key) + " is required"); return v->as_string(); } double real(const char* key, double fallback = 0) const { const Json* v = doc_.find(key); if (!v || v->is_null()) return fallback; if (!v->is_number()) bad(std::string(key) + " must be a number"); return v->as_number(); } long number(const char* key, long fallback = 0) const { return static_cast(real(key, static_cast(fallback))); } bool flag(const char* key) const { const Json* v = doc_.find(key); if (!v || v->is_null()) return false; if (!v->is_bool()) bad(std::string(key) + " must be true or false"); return v->as_bool(); } Json object(const char* key) const { const Json* v = doc_.find(key); if (!v || v->is_null()) return Json(); if (!v->is_object()) bad(std::string(key) + " must be an object"); return *v; } std::vector strings(const char* key) const { std::vector out; const Json* v = doc_.find(key); if (!v || v->is_null()) return out; if (!v->is_array()) bad(std::string(key) + " must be an array"); for (const Json& s : v->as_array()) { if (!s.is_string()) bad(std::string(key) + " must hold strings"); out.push_back(s.as_string()); } return out; } std::string bytes(const char* key) const { std::string b64 = text(key), raw; if (b64.empty()) return raw; if (!security::crypto::Base64::decode(b64, raw)) bad(std::string(key) + " is not valid base64"); return raw; } bool confirm() const { return flag("confirm"); } const Json& doc() const { return doc_; } private: Json doc_ = Json::object(); }; std::string b64(const std::string& raw) { return security::crypto::Base64::encode(raw); } const au::ProviderInfo& provider_of(const Body& b) { std::string name = b.text("provider", true); const au::ProviderInfo* p = au::Providers::find(name); if (!p) bad("provider must be one of elevenlabs, assemblyai, deepgram, cartesia, speechmatics, hume"); return *p; } Json dry_run(const std::string& provider, const std::string& what, const std::string& request) { Json o = Json::object(); o.set("dry_run", Json(true)); o.set("provider", Json(provider)); o.set("what", Json(what)); o.set("request", Json(request)); o.set("hint", Json("send \"confirm\":true to run it")); return o; } Json audio_json(const std::string& audio, const std::string& content_type) { Json o = Json::object(); o.set("content_type", Json(content_type)); o.set("bytes", Json(static_cast(audio.size()))); if (audio.size() <= kMaxAudioEcho) o.set("audio_base64", Json(b64(audio))); else o.set("audio_omitted", Json("over the 8 MiB echo cap")); return o; } // The service for a provider's table, built from the environment. std::unique_ptr service_for(const std::string& name) { if (name == "elevenlabs") return std::make_unique(au::elevenlabs::calls(), au::elevenlabs::ElevenEndpoints::from_env()); if (name == "assemblyai") return std::make_unique(au::assemblyai::calls(), au::assemblyai::AssemblyEndpoints::from_env()); if (name == "deepgram") return std::make_unique(au::deepgram::calls(), au::deepgram::DeepgramEndpoints::from_env()); if (name == "cartesia") return std::make_unique(au::cartesia::calls(), au::cartesia::CartesiaEndpoints::from_env()); if (name == "speechmatics") return std::make_unique(au::speechmatics::calls(), au::speechmatics::SpeechmaticsEndpoints::from_env()); return std::make_unique(au::hume::calls(), au::hume::HumeEndpoints::from_env()); } template auto with_segments(const std::vector& s, F&& f) -> decltype(f(std::initializer_list{})) { switch (s.size()) { case 0: return f({}); case 1: return f({s[0]}); case 2: return f({s[0], s[1]}); case 3: return f({s[0], s[1], s[2]}); case 4: return f({s[0], s[1], s[2], s[3]}); default: bad("at most 4 path segments"); } } class AudioRoutes { public: Json tts(const Body& b) const { const au::ProviderInfo& p = provider_of(b); if (!p.synthesizer) bad(std::string(p.title) + " has no speech synthesis product"); au::SpeechRequest r; r.text = b.text("text", true); r.voice = b.text("voice"); r.model = b.text("model"); r.language = b.text("language"); if (!b.text("format").empty()) r.format = b.text("format"); r.sample_rate = static_cast(b.number("sample_rate")); r.speed = b.real("speed"); return guarded([&]() -> Json { au::validate(r); auto synth = std::make_shared(au::SpeechRegistry::synthesizer(p.name)); if (!b.confirm()) { std::size_t chunks = au::split_text(r.text, std::min(synth->max_chars(), std::size_t{5000})).size(); Json o = dry_run(p.name, "synthesize", std::to_string(r.text.size()) + " characters, " + r.format); o.set("chunks", Json(static_cast(chunks))); o.set("max_chars_per_call", Json(static_cast(synth->max_chars()))); return o; } au::SpeechResult out = synth->synthesize(r); Json o = audio_json(out.audio, out.content_type); o.set("provider", Json(out.provider)); o.set("format", Json(out.format)); return o; }); } Json stt(const Body& b) const { const au::ProviderInfo& p = provider_of(b); if (!p.recognizer) bad(std::string(p.title) + " has no speech recognition product"); au::TranscribeRequest r; r.url = b.text("url"); r.audio = b.bytes("data_base64"); if (!b.text("filename").empty()) r.filename = b.text("filename"); if (!b.text("content_type").empty()) r.content_type = b.text("content_type"); r.language = b.text("language"); r.model = b.text("model"); r.diarize = b.flag("diarize"); r.keywords = b.strings("keywords"); if (b.number("max_wait_seconds") > 0) r.max_wait_seconds = static_cast(b.number("max_wait_seconds")); return guarded([&]() -> Json { au::validate(r); if (!b.confirm()) { return dry_run(p.name, "transcribe", (r.url.empty() ? std::to_string(r.audio.size()) + " bytes of audio" : r.url) + (r.diarize ? ", diarized" : "")); } au::Transcript t = au::SpeechRegistry::recognizer(p.name)->transcribe(r); Json o = Json::object(); o.set("provider", Json(t.provider)); o.set("text", Json(t.text)); o.set("language", Json(t.language)); o.set("duration", Json(t.duration)); Json words = Json::array(); for (const au::TranscriptWord& w : t.words) { if (words.as_array().size() >= 2000) break; Json j = Json::object(); j.set("text", Json(w.text)); j.set("start", Json(w.start)); j.set("end", Json(w.end)); if (!w.speaker.empty()) j.set("speaker", Json(w.speaker)); words.push(j); } o.set("words", words); return o; }); } Json voices(const Body& b) const { const au::ProviderInfo& p = provider_of(b); if (!p.synthesizer) bad(std::string(p.title) + " has no voices"); return guarded([&]() -> Json { return au::SpeechRegistry::synthesizer(p.name)->voices(); }); } // Any row of a provider's call table by id. Reads run; everything else is a dry run unless confirmed. Json call(const Body& b) const { const au::ProviderInfo& p = provider_of(b); std::string id = b.text("id", true); std::vector segments = b.strings("segments"); Json query = b.object("query"); Json body = b.object("body"); return guarded([&]() -> Json { auto svc = service_for(p.name); const au::CallSpec& spec = svc->table().get(id); std::string kind = spec.body; au::Query q = svc->query_from(id, query); std::string method = spec.method; std::string preview = with_segments(segments, [&](std::initializer_list s) { return svc->preview(id, s, kind == "json" && body.is_object() ? &body : nullptr, q); }); bool write = method != "GET"; if (write && !b.confirm()) return dry_run(p.name, id, preview); Json out = Json::object(); out.set("provider", Json(p.name)); out.set("id", Json(id)); if (kind == "bytes") { std::string data = b.bytes("data_base64"); std::string ct = b.text("content_type"); llm::RawResponse raw = with_segments(segments, [&](std::initializer_list s) { return svc->call_bytes(id, s, data, ct, q); }); out.set("response", Json(raw.body.substr(0, 65536))); return out; } if (kind == "form") { clients::body::MultipartBody form; Json texts = b.object("form_text"); if (texts.is_object()) for (const auto& [k, v] : texts.as_object()) { if (!v.is_string()) bad("form_text values must be strings"); form.add_text(k, v.as_string()); } std::string data = b.bytes("data_base64"); if (!data.empty()) { std::string field = b.text("file_field").empty() ? "file" : b.text("file_field"); std::string ct = b.text("content_type").empty() ? "application/octet-stream" : b.text("content_type"); form.add_file(field, b.text("filename").empty() ? "upload" : b.text("filename"), ct, data); } llm::RawResponse raw = with_segments(segments, [&](std::initializer_list s) { return svc->call_form_raw(id, s, form, q); }); out.set("response", Json(raw.body.substr(0, 65536))); return out; } if (au::is_destructive(spec)) { out.set("response", with_segments(segments, [&](std::initializer_list s) { return svc->call_destructive(id, s, kind == "json" ? &body : nullptr, q, true); })); return out; } std::string resp = spec.response; if (resp == "bytes" || resp == "text") { llm::RawResponse raw = with_segments(segments, [&](std::initializer_list s) { return svc->call_raw(id, s, kind == "json" ? &body : nullptr, q); }); Json a = resp == "text" ? Json::object() : audio_json(raw.body, raw.content_type); if (resp == "text") a.set("text", Json(raw.body.substr(0, 65536))); out.set("response", a); return out; } out.set("response", with_segments(segments, [&](std::initializer_list s) { return svc->call(id, s, kind == "json" ? &body : nullptr, q); })); return out; }); } // One shot streaming synthesis over the provider's WebSocket. Json session(const Body& b) const { const au::ProviderInfo& p = provider_of(b); std::string text = b.text("text", true); std::string voice = b.text("voice"); std::string name = p.name; if (name != "elevenlabs" && name != "deepgram" && name != "cartesia" && name != "hume") bad("streaming synthesis is available for elevenlabs, deepgram, cartesia and hume"); if (name != "hume" && name != "deepgram" && voice.empty()) bad("voice is required"); return guarded([&]() -> Json { std::string key = llm::Config::get(p.key_env); if (!b.confirm()) return dry_run(name, "streaming synthesis", std::to_string(text.size()) + " characters over WebSocket"); if (key.empty()) throw llm::NotConfiguredError(name, p.key_env); Json o = Json::object(); o.set("provider", Json(name)); if (name == "elevenlabs") { au::elevenlabs::TtsStreamOptions opt; opt.voice_id = voice; opt.model_id = b.text("model"); opt.output_format = b.text("output_format"); au::elevenlabs::TtsInputStream s(key, opt); s.connect(); au::elevenlabs::TtsStreamResult r = s.speak(text, 20000); s.close(); if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error); Json a = audio_json(r.audio, "audio/mpeg"); a.set("chunks", Json(static_cast(r.chunks))); a.set("final", Json(r.is_final)); return a; } if (name == "deepgram") { au::deepgram::StreamParams params; params.add("model", voice.empty() ? "aura-2-thalia-en" : voice).add("encoding", "linear16").add("sample_rate", "24000"); au::deepgram::SpeakStream s(key, params); s.connect(); s.speak(text); s.flush(); au::deepgram::SpeakStreamResult r = s.collect(20000); s.end(); s.close(); if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error); Json a = audio_json(r.audio, "audio/pcm"); a.set("chunks", Json(static_cast(r.chunks))); a.set("flushed", Json(r.flushed)); return a; } if (name == "cartesia") { au::cartesia::TtsSocket s(key); s.connect(); au::cartesia::TtsSocketRequest r; r.context_id = "demo-1"; r.transcript = text; r.voice_id = voice; r.continue_ = 0; s.generate(r); au::cartesia::TtsSocketResult res = s.collect("demo-1", 20000); s.close(); if (!res.error.empty()) throw llm::ApiError(name, 502, "stream_error", res.error); Json a = audio_json(res.audio, "audio/pcm"); a.set("chunks", Json(static_cast(res.chunks))); a.set("done", Json(res.done)); return a; } au::hume::HumeStreamOptions opt; opt.format_type = "mp3"; opt.no_binary = 1; au::hume::HumeTtsStream s(key, opt); s.connect(); s.send_text(text, voice); s.finish(); au::hume::HumeStreamResult r = s.collect(20000); s.close(); if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error); Json a = audio_json(r.audio, "audio/mpeg"); a.set("chunks", Json(static_cast(r.chunks))); a.set("last_chunk", Json(r.last_chunk)); return a; }); } // Local: the URL a WebSocket channel would open, with the credential left out. Json ws_preview(const Body& b) const { const au::ProviderInfo& p = provider_of(b); std::string channel = b.text("channel", true); Json params = b.object("params"); auto str = [&](const char* k) { const Json* v = params.is_object() ? params.find(k) : nullptr; return v && v->is_string() ? v->as_string() : std::string(); }; return guarded([&]() -> Json { std::string url; std::string key = "preview-only"; if (std::string(p.name) == "elevenlabs" && channel == "tts-stream-input") { au::elevenlabs::TtsStreamOptions o; o.voice_id = str("voice_id"); o.model_id = str("model_id"); o.output_format = str("output_format"); url = au::elevenlabs::TtsInputStream(key, o).url(); } else if (std::string(p.name) == "elevenlabs" && channel == "stt-realtime") { au::elevenlabs::ScribeStreamOptions o; o.model_id = str("model_id"); o.audio_format = str("audio_format"); o.commit_strategy = str("commit_strategy"); url = au::elevenlabs::ScribeStream(key, o).url(); } else if (std::string(p.name) == "deepgram" && (channel == "listen-v1" || channel == "listen-v2")) { au::deepgram::StreamParams sp; if (params.is_object()) for (const auto& [k, v] : params.as_object()) if (v.is_string()) sp.add(k, v.as_string()); url = au::deepgram::ListenStream(key, sp, channel == "listen-v2").url(); } else if (std::string(p.name) == "deepgram" && (channel == "speak-v1" || channel == "speak-v2")) { au::deepgram::StreamParams sp; if (params.is_object()) for (const auto& [k, v] : params.as_object()) if (v.is_string()) sp.add(k, v.as_string()); url = au::deepgram::SpeakStream(key, sp, channel == "speak-v2").url(); } else if (std::string(p.name) == "assemblyai" && channel == "streaming") { au::assemblyai::StreamParams sp; if (params.is_object()) for (const auto& [k, v] : params.as_object()) if (v.is_string()) sp.add(k, v.as_string()); url = au::assemblyai::TranscriptStream(key, sp).url(); } else if (std::string(p.name) == "cartesia" && channel == "tts-websocket") { url = au::cartesia::TtsSocket(key).url(); } else if (std::string(p.name) == "speechmatics" && channel == "realtime") { au::speechmatics::RealtimeConfig c; if (!str("language").empty()) c.language = str("language"); url = au::speechmatics::RealtimeSession(key, c).url(); } else if (std::string(p.name) == "hume" && channel == "tts-stream-input") { url = au::hume::HumeTtsStream(key, au::hume::HumeStreamOptions()).url(); } else if (std::string(p.name) == "hume" && channel == "evi-chat") { au::hume::EviOptions o; o.config_id = str("config_id"); url = au::hume::EviChat(key, o).url(); } else { bad(std::string("no such channel for ") + p.name + ": " + channel); } Json o = Json::object(); o.set("provider", Json(p.name)); o.set("channel", Json(channel)); o.set("url", Json(url)); o.set("credential", Json("sent as a header, never in the URL")); return o; }); } // Local: sentence aware splitting with the package's caps. Json split(const Body& b) const { std::string text = b.text("text", true); long max_chars = b.number("max_chars", 500); if (max_chars < 1 || max_chars > 100000) bad("max_chars must be 1..100000"); return guarded([&]() -> Json { Json arr = Json::array(); for (const std::string& c : au::split_text(text, static_cast(max_chars))) arr.push(Json(c)); Json o = Json::object(); o.set("chunks", arr); o.set("count", Json(static_cast(arr.as_array().size()))); return o; }); } }; } // namespace void register_ai_audio_demo(fw::Router& router, Services& services) { fw::Environment* env = services.templates.get(); services.templates->add_template("ai_audio.html", kPage); services.add_feature("ai-audio", "/ai/audio"); auto routes = std::make_shared(); router.get("/ai/audio", "ai-audio-page", [env](fw::Request req) { (void)req; fw::TemplateCtx ctx; ctx["title"] = std::string("Audio"); return fw::Response(200, env->render("ai_audio.html", ctx), "text/html; charset=utf-8"); }); router.get("/ai/audio/status", "ai-audio-status", [](fw::Request req) { (void)req; Json providers = Json::array(); for (const au::ProviderInfo& p : au::Providers::all()) { Json j = Json::object(); j.set("name", Json(p.name)); j.set("title", Json(p.title)); j.set("key_env", Json(p.key_env)); j.set("key_configured", Json(!llm::Config::get(p.key_env).empty())); if (std::string(p.extra_env).size()) { j.set("extra_env", Json(p.extra_env)); j.set("extra_configured", Json(!llm::Config::get(p.extra_env).empty())); } j.set("key_page", Json(p.key_page)); j.set("synthesizer", Json(p.synthesizer)); j.set("recognizer", Json(p.recognizer)); j.set("websockets", Json(p.websockets)); j.set("calls", Json(static_cast(p.table().size()))); providers.push(j); } Json absent = Json::array(); for (const au::Providers::Absent& a : au::Providers::absent()) { Json j = Json::object(); j.set("name", Json(a.name)); j.set("reason", Json(a.reason)); absent.push(j); } Json o = Json::object(); o.set("providers", providers); o.set("absent", absent); o.set("calls_total", Json(static_cast(au::Providers::call_count()))); o.set("max_body_bytes", Json(static_cast(kMaxBody))); return json_out(o); }); router.get("/ai/audio/calls", "ai-audio-calls", [](fw::Request req) { Json arr = Json::array(); std::string want; if (auto it = req.query.find("provider"); it != req.query.end()) want = it->second; for (const au::ProviderInfo& p : au::Providers::all()) { if (!want.empty() && want != p.name) continue; for (std::size_t i = 0; i < p.table().size(); ++i) { const au::CallSpec& s = p.table().at(i); Json row = Json::object(); row.set("provider", Json(p.name)); row.set("id", Json(s.id)); row.set("http", Json(s.method)); row.set("path", Json(s.path)); row.set("host", Json(s.host)); row.set("body", Json(s.body)); row.set("response", Json(s.response)); arr.push(row); if (arr.as_array().size() >= 2000) break; } } Json o = Json::object(); o.set("calls", arr); o.set("truncated", Json(want.empty() && au::Providers::call_count() > 2000)); return json_out(o); }); using Handler = Json (AudioRoutes::*)(const Body&) const; struct Route { const char* path; const char* name; Handler handler; }; const Route table[] = { {"/ai/audio/tts", "ai-audio-tts", &AudioRoutes::tts}, {"/ai/audio/stt", "ai-audio-stt", &AudioRoutes::stt}, {"/ai/audio/voices", "ai-audio-voices", &AudioRoutes::voices}, {"/ai/audio/call", "ai-audio-call", &AudioRoutes::call}, {"/ai/audio/session", "ai-audio-session", &AudioRoutes::session}, {"/ai/audio/ws/preview", "ai-audio-ws-preview", &AudioRoutes::ws_preview}, {"/ai/audio/split", "ai-audio-split", &AudioRoutes::split}, }; for (const Route& r : table) { Handler h = r.handler; router.post(r.path, r.name, [routes, h](fw::Request req) { Body body(req); return json_out(((*routes).*h)(body)); }); } } } // namespace molo::example::app