From 6d80dd022398a53f6308aa36b5323dfbac3d3b70 Mon Sep 17 00:00:00 2001 From: retoor Date: Sun, 25 Oct 2026 15:36:57 +0100 Subject: [PATCH] feat(clients): audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics, Hume Table driven REST (509 rows verified against vendor specs by tools/check_spec.py), WebSockets over the src/google/live client, neutral SpeechSynthesizer / SpeechRecognizer seam, /ai/audio demo with dry runs, 110 package tests and 5 route tests. PlayHT has no public API; LeMUR is gone from AssemblyAI's spec. Co-Authored-By: Claude Sonnet 5.5 --- example/app/ai/AudioDemo.cpp | 623 ++++++++++++++- example/assets/ai_audio.js | 82 ++ src/vendors/audio/SPEC.md | 273 +++++++ src/vendors/audio/assemblyai/calls.cpp | 33 + src/vendors/audio/assemblyai/calls.h | 30 + src/vendors/audio/assemblyai/client.cpp | 51 ++ src/vendors/audio/assemblyai/client.h | 38 + src/vendors/audio/assemblyai/gateway.cpp | 63 ++ src/vendors/audio/assemblyai/gateway.h | 44 ++ src/vendors/audio/assemblyai/speech.cpp | 53 ++ src/vendors/audio/assemblyai/speech.h | 31 + src/vendors/audio/assemblyai/stream.cpp | 131 ++++ src/vendors/audio/assemblyai/stream.h | 81 ++ src/vendors/audio/assemblyai/transcripts.cpp | 183 +++++ src/vendors/audio/assemblyai/transcripts.h | 71 ++ src/vendors/audio/audio_service.cpp | 211 +++++ src/vendors/audio/audio_service.h | 86 +++ src/vendors/audio/call_spec.cpp | 132 ++++ src/vendors/audio/call_spec.h | 65 ++ src/vendors/audio/caps.h | 44 ++ src/vendors/audio/cartesia/calls.cpp | 72 ++ src/vendors/audio/cartesia/calls.h | 39 + src/vendors/audio/cartesia/client.cpp | 35 + src/vendors/audio/cartesia/client.h | 31 + src/vendors/audio/cartesia/speech.cpp | 79 ++ src/vendors/audio/cartesia/speech.h | 43 ++ src/vendors/audio/cartesia/stt.cpp | 177 +++++ src/vendors/audio/cartesia/stt.h | 93 +++ src/vendors/audio/cartesia/tts.cpp | 117 +++ src/vendors/audio/cartesia/tts.h | 68 ++ src/vendors/audio/cartesia/tts_socket.cpp | 127 +++ src/vendors/audio/cartesia/tts_socket.h | 71 ++ src/vendors/audio/cartesia/voices.cpp | 90 +++ src/vendors/audio/cartesia/voices.h | 35 + src/vendors/audio/deepgram/calls.cpp | 70 ++ src/vendors/audio/deepgram/calls.h | 38 + src/vendors/audio/deepgram/client.cpp | 44 ++ src/vendors/audio/deepgram/client.h | 38 + src/vendors/audio/deepgram/listen.cpp | 87 +++ src/vendors/audio/deepgram/listen.h | 39 + src/vendors/audio/deepgram/manage.cpp | 100 +++ src/vendors/audio/deepgram/manage.h | 40 + src/vendors/audio/deepgram/read.cpp | 53 ++ src/vendors/audio/deepgram/read.h | 30 + src/vendors/audio/deepgram/service.cpp | 14 + src/vendors/audio/deepgram/service.h | 29 + src/vendors/audio/deepgram/speak.cpp | 59 ++ src/vendors/audio/deepgram/speak.h | 40 + src/vendors/audio/deepgram/speech.cpp | 103 +++ src/vendors/audio/deepgram/speech.h | 43 ++ src/vendors/audio/deepgram/streams.cpp | 325 ++++++++ src/vendors/audio/deepgram/streams.h | 153 ++++ src/vendors/audio/elevenlabs/account.cpp | 41 + src/vendors/audio/elevenlabs/account.h | 30 + src/vendors/audio/elevenlabs/agents.cpp | 113 +++ src/vendors/audio/elevenlabs/agents.h | 39 + src/vendors/audio/elevenlabs/calls.cpp | 344 +++++++++ src/vendors/audio/elevenlabs/calls.h | 72 ++ src/vendors/audio/elevenlabs/client.cpp | 45 ++ src/vendors/audio/elevenlabs/client.h | 35 + .../audio/elevenlabs/convai_session.cpp | 171 ++++ src/vendors/audio/elevenlabs/convai_session.h | 71 ++ src/vendors/audio/elevenlabs/creative.cpp | 98 +++ src/vendors/audio/elevenlabs/creative.h | 48 ++ src/vendors/audio/elevenlabs/dubbing.cpp | 58 ++ src/vendors/audio/elevenlabs/dubbing.h | 37 + .../audio/elevenlabs/scribe_stream.cpp | 128 +++ src/vendors/audio/elevenlabs/scribe_stream.h | 64 ++ src/vendors/audio/elevenlabs/service.cpp | 19 + src/vendors/audio/elevenlabs/service.h | 36 + src/vendors/audio/elevenlabs/speech.cpp | 97 +++ src/vendors/audio/elevenlabs/speech.h | 47 ++ src/vendors/audio/elevenlabs/stt.cpp | 66 ++ src/vendors/audio/elevenlabs/stt.h | 43 ++ src/vendors/audio/elevenlabs/tts.cpp | 110 +++ src/vendors/audio/elevenlabs/tts.h | 64 ++ src/vendors/audio/elevenlabs/tts_stream.cpp | 149 ++++ src/vendors/audio/elevenlabs/tts_stream.h | 77 ++ src/vendors/audio/elevenlabs/voices.cpp | 145 ++++ src/vendors/audio/elevenlabs/voices.h | 65 ++ src/vendors/audio/hosts.cpp | 153 ++++ src/vendors/audio/hosts.h | 91 +++ src/vendors/audio/hume/calls.cpp | 68 ++ src/vendors/audio/hume/calls.h | 45 ++ src/vendors/audio/hume/client.cpp | 35 + src/vendors/audio/hume/client.h | 33 + src/vendors/audio/hume/evi.cpp | 110 +++ src/vendors/audio/hume/evi.h | 44 ++ src/vendors/audio/hume/sockets.cpp | 252 ++++++ src/vendors/audio/hume/sockets.h | 111 +++ src/vendors/audio/hume/speech.cpp | 55 ++ src/vendors/audio/hume/speech.h | 31 + src/vendors/audio/hume/tts.cpp | 134 ++++ src/vendors/audio/hume/tts.h | 76 ++ src/vendors/audio/providers.cpp | 50 ++ src/vendors/audio/providers.h | 41 + src/vendors/audio/query.cpp | 75 ++ src/vendors/audio/query.h | 33 + src/vendors/audio/registry.cpp | 45 ++ src/vendors/audio/socket.cpp | 180 +++++ src/vendors/audio/socket.h | 112 +++ src/vendors/audio/speech.cpp | 162 ++++ src/vendors/audio/speech.h | 152 ++++ src/vendors/audio/speechmatics/batch.cpp | 160 ++++ src/vendors/audio/speechmatics/batch.h | 65 ++ src/vendors/audio/speechmatics/calls.cpp | 36 + src/vendors/audio/speechmatics/calls.h | 29 + src/vendors/audio/speechmatics/client.cpp | 54 ++ src/vendors/audio/speechmatics/client.h | 40 + src/vendors/audio/speechmatics/manage.cpp | 38 + src/vendors/audio/speechmatics/manage.h | 25 + src/vendors/audio/speechmatics/realtime.cpp | 138 ++++ src/vendors/audio/speechmatics/realtime.h | 70 ++ src/vendors/audio/speechmatics/speech.cpp | 94 +++ src/vendors/audio/speechmatics/speech.h | 47 ++ src/vendors/audio/speechmatics/tts.cpp | 38 + src/vendors/audio/speechmatics/tts.h | 30 + src/vendors/audio/tools/check_spec.py | 731 ++++++++++++++++++ tests/example/ai/audio/confirm_test.cpp | 34 + tests/example/ai/audio/local_test.cpp | 34 + tests/example/ai/audio/page_test.cpp | 21 + tests/example/ai/audio/routes_test.cpp | 66 ++ tests/example/ai/audio/status_test.cpp | 35 + tests/src/vendors/audio/rig.h | 100 +++ tests/src/vendors/audio/test_assemblyai.cpp | 268 +++++++ tests/src/vendors/audio/test_cartesia.cpp | 342 ++++++++ tests/src/vendors/audio/test_core.cpp | 628 +++++++++++++++ tests/src/vendors/audio/test_deepgram.cpp | 395 ++++++++++ tests/src/vendors/audio/test_elevenlabs.cpp | 707 +++++++++++++++++ tests/src/vendors/audio/test_hume.cpp | 317 ++++++++ tests/src/vendors/audio/test_live.cpp | 139 ++++ tests/src/vendors/audio/test_speechmatics.cpp | 275 +++++++ 132 files changed, 14060 insertions(+), 3 deletions(-) create mode 100644 example/assets/ai_audio.js create mode 100644 src/vendors/audio/SPEC.md create mode 100644 src/vendors/audio/assemblyai/calls.cpp create mode 100644 src/vendors/audio/assemblyai/calls.h create mode 100644 src/vendors/audio/assemblyai/client.cpp create mode 100644 src/vendors/audio/assemblyai/client.h create mode 100644 src/vendors/audio/assemblyai/gateway.cpp create mode 100644 src/vendors/audio/assemblyai/gateway.h create mode 100644 src/vendors/audio/assemblyai/speech.cpp create mode 100644 src/vendors/audio/assemblyai/speech.h create mode 100644 src/vendors/audio/assemblyai/stream.cpp create mode 100644 src/vendors/audio/assemblyai/stream.h create mode 100644 src/vendors/audio/assemblyai/transcripts.cpp create mode 100644 src/vendors/audio/assemblyai/transcripts.h create mode 100644 src/vendors/audio/audio_service.cpp create mode 100644 src/vendors/audio/audio_service.h create mode 100644 src/vendors/audio/call_spec.cpp create mode 100644 src/vendors/audio/call_spec.h create mode 100644 src/vendors/audio/caps.h create mode 100644 src/vendors/audio/cartesia/calls.cpp create mode 100644 src/vendors/audio/cartesia/calls.h create mode 100644 src/vendors/audio/cartesia/client.cpp create mode 100644 src/vendors/audio/cartesia/client.h create mode 100644 src/vendors/audio/cartesia/speech.cpp create mode 100644 src/vendors/audio/cartesia/speech.h create mode 100644 src/vendors/audio/cartesia/stt.cpp create mode 100644 src/vendors/audio/cartesia/stt.h create mode 100644 src/vendors/audio/cartesia/tts.cpp create mode 100644 src/vendors/audio/cartesia/tts.h create mode 100644 src/vendors/audio/cartesia/tts_socket.cpp create mode 100644 src/vendors/audio/cartesia/tts_socket.h create mode 100644 src/vendors/audio/cartesia/voices.cpp create mode 100644 src/vendors/audio/cartesia/voices.h create mode 100644 src/vendors/audio/deepgram/calls.cpp create mode 100644 src/vendors/audio/deepgram/calls.h create mode 100644 src/vendors/audio/deepgram/client.cpp create mode 100644 src/vendors/audio/deepgram/client.h create mode 100644 src/vendors/audio/deepgram/listen.cpp create mode 100644 src/vendors/audio/deepgram/listen.h create mode 100644 src/vendors/audio/deepgram/manage.cpp create mode 100644 src/vendors/audio/deepgram/manage.h create mode 100644 src/vendors/audio/deepgram/read.cpp create mode 100644 src/vendors/audio/deepgram/read.h create mode 100644 src/vendors/audio/deepgram/service.cpp create mode 100644 src/vendors/audio/deepgram/service.h create mode 100644 src/vendors/audio/deepgram/speak.cpp create mode 100644 src/vendors/audio/deepgram/speak.h create mode 100644 src/vendors/audio/deepgram/speech.cpp create mode 100644 src/vendors/audio/deepgram/speech.h create mode 100644 src/vendors/audio/deepgram/streams.cpp create mode 100644 src/vendors/audio/deepgram/streams.h create mode 100644 src/vendors/audio/elevenlabs/account.cpp create mode 100644 src/vendors/audio/elevenlabs/account.h create mode 100644 src/vendors/audio/elevenlabs/agents.cpp create mode 100644 src/vendors/audio/elevenlabs/agents.h create mode 100644 src/vendors/audio/elevenlabs/calls.cpp create mode 100644 src/vendors/audio/elevenlabs/calls.h create mode 100644 src/vendors/audio/elevenlabs/client.cpp create mode 100644 src/vendors/audio/elevenlabs/client.h create mode 100644 src/vendors/audio/elevenlabs/convai_session.cpp create mode 100644 src/vendors/audio/elevenlabs/convai_session.h create mode 100644 src/vendors/audio/elevenlabs/creative.cpp create mode 100644 src/vendors/audio/elevenlabs/creative.h create mode 100644 src/vendors/audio/elevenlabs/dubbing.cpp create mode 100644 src/vendors/audio/elevenlabs/dubbing.h create mode 100644 src/vendors/audio/elevenlabs/scribe_stream.cpp create mode 100644 src/vendors/audio/elevenlabs/scribe_stream.h create mode 100644 src/vendors/audio/elevenlabs/service.cpp create mode 100644 src/vendors/audio/elevenlabs/service.h create mode 100644 src/vendors/audio/elevenlabs/speech.cpp create mode 100644 src/vendors/audio/elevenlabs/speech.h create mode 100644 src/vendors/audio/elevenlabs/stt.cpp create mode 100644 src/vendors/audio/elevenlabs/stt.h create mode 100644 src/vendors/audio/elevenlabs/tts.cpp create mode 100644 src/vendors/audio/elevenlabs/tts.h create mode 100644 src/vendors/audio/elevenlabs/tts_stream.cpp create mode 100644 src/vendors/audio/elevenlabs/tts_stream.h create mode 100644 src/vendors/audio/elevenlabs/voices.cpp create mode 100644 src/vendors/audio/elevenlabs/voices.h create mode 100644 src/vendors/audio/hosts.cpp create mode 100644 src/vendors/audio/hosts.h create mode 100644 src/vendors/audio/hume/calls.cpp create mode 100644 src/vendors/audio/hume/calls.h create mode 100644 src/vendors/audio/hume/client.cpp create mode 100644 src/vendors/audio/hume/client.h create mode 100644 src/vendors/audio/hume/evi.cpp create mode 100644 src/vendors/audio/hume/evi.h create mode 100644 src/vendors/audio/hume/sockets.cpp create mode 100644 src/vendors/audio/hume/sockets.h create mode 100644 src/vendors/audio/hume/speech.cpp create mode 100644 src/vendors/audio/hume/speech.h create mode 100644 src/vendors/audio/hume/tts.cpp create mode 100644 src/vendors/audio/hume/tts.h create mode 100644 src/vendors/audio/providers.cpp create mode 100644 src/vendors/audio/providers.h create mode 100644 src/vendors/audio/query.cpp create mode 100644 src/vendors/audio/query.h create mode 100644 src/vendors/audio/registry.cpp create mode 100644 src/vendors/audio/socket.cpp create mode 100644 src/vendors/audio/socket.h create mode 100644 src/vendors/audio/speech.cpp create mode 100644 src/vendors/audio/speech.h create mode 100644 src/vendors/audio/speechmatics/batch.cpp create mode 100644 src/vendors/audio/speechmatics/batch.h create mode 100644 src/vendors/audio/speechmatics/calls.cpp create mode 100644 src/vendors/audio/speechmatics/calls.h create mode 100644 src/vendors/audio/speechmatics/client.cpp create mode 100644 src/vendors/audio/speechmatics/client.h create mode 100644 src/vendors/audio/speechmatics/manage.cpp create mode 100644 src/vendors/audio/speechmatics/manage.h create mode 100644 src/vendors/audio/speechmatics/realtime.cpp create mode 100644 src/vendors/audio/speechmatics/realtime.h create mode 100644 src/vendors/audio/speechmatics/speech.cpp create mode 100644 src/vendors/audio/speechmatics/speech.h create mode 100644 src/vendors/audio/speechmatics/tts.cpp create mode 100644 src/vendors/audio/speechmatics/tts.h create mode 100644 src/vendors/audio/tools/check_spec.py create mode 100644 tests/example/ai/audio/confirm_test.cpp create mode 100644 tests/example/ai/audio/local_test.cpp create mode 100644 tests/example/ai/audio/page_test.cpp create mode 100644 tests/example/ai/audio/routes_test.cpp create mode 100644 tests/example/ai/audio/status_test.cpp create mode 100644 tests/src/vendors/audio/rig.h create mode 100644 tests/src/vendors/audio/test_assemblyai.cpp create mode 100644 tests/src/vendors/audio/test_cartesia.cpp create mode 100644 tests/src/vendors/audio/test_core.cpp create mode 100644 tests/src/vendors/audio/test_deepgram.cpp create mode 100644 tests/src/vendors/audio/test_elevenlabs.cpp create mode 100644 tests/src/vendors/audio/test_hume.cpp create mode 100644 tests/src/vendors/audio/test_live.cpp create mode 100644 tests/src/vendors/audio/test_speechmatics.cpp diff --git a/example/app/ai/AudioDemo.cpp b/example/app/ai/AudioDemo.cpp index 2edea9c..af4ae69 100644 --- a/example/app/ai/AudioDemo.cpp +++ b/example/app/ai/AudioDemo.cpp @@ -1,13 +1,630 @@ #include "../AiDemos.hpp" +#include + #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "llm/api_client.h" +#include "llm/config.h" +#include "security/crypto/Base64.h" +#include "vendors/audio/assemblyai/client.h" +#include "vendors/audio/assemblyai/transcripts.h" +#include "vendors/audio/cartesia/client.h" +#include "vendors/audio/deepgram/client.h" +#include "vendors/audio/elevenlabs/client.h" +#include "vendors/audio/hume/client.h" +#include "vendors/audio/audio_service.h" +#include "vendors/audio/cartesia/tts_socket.h" +#include "vendors/audio/deepgram/streams.h" +#include "vendors/audio/elevenlabs/tts_stream.h" +#include "vendors/audio/hume/sockets.h" +#include "vendors/audio/providers.h" +#include "vendors/audio/speech.h" +#include "vendors/audio/assemblyai/stream.h" +#include "vendors/audio/elevenlabs/scribe_stream.h" +#include "vendors/audio/speechmatics/batch.h" +#include "vendors/audio/speechmatics/realtime.h" namespace molo::example::app { -// audio. Stub: the package agent fills this in. +// audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics and Hume +// through src/vendors/audio. GET routes need no credential. Every route that +// calls a provider is a POST: 503 not_configured without the provider's key, +// 422 on bad input. Anything that generates audio, writes, or deletes is a +// dry run unless the body says "confirm":true. + +namespace { + +namespace au = audio; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 4u * 1024u * 1024u; +constexpr std::size_t kMaxAudioEcho = 8u * 1024u * 1024u; // audio returned to the browser as base64 + +const char kPage[] = R"AUDIOP({% extends "base.html" %} +{% block content %} +

Audio

+

Speech synthesis and recognition across ElevenLabs, AssemblyAI, Deepgram, Cartesia, +Speechmatics and Hume. Anything that generates audio, writes or deletes is a dry run until the body says +"confirm":true.

+

Checking provider keys...

+
+ + + +
+ + +

+
+ + +{% endblock %} +)AUDIOP"; + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } + +template +auto guarded(F&& f) -> decltype(f()) { + try { + return f(); + } catch (const llm::NotConfiguredError& e) { + throw fw::HttpError(503, "not_configured", e.message()); + } catch (const llm::ApiError& e) { + if (e.http_status() == 400 && e.type() == "INVALID_ARGUMENT") bad(e.message()); + throw fw::HttpError(502, "audio_error", + "HTTP " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +class Body { + public: + explicit Body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return; + try { + doc_ = Json::parse(req.body); + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } + if (!doc_.is_object()) bad("body must be a JSON object"); + } + std::string text(const char* key, bool required = false) const { + const Json* v = doc_.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + if (required && v->as_string().empty()) bad(std::string(key) + " is required"); + return v->as_string(); + } + double real(const char* key, double fallback = 0) const { + const Json* v = doc_.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + return v->as_number(); + } + long number(const char* key, long fallback = 0) const { return static_cast(real(key, static_cast(fallback))); } + bool flag(const char* key) const { + const Json* v = doc_.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); + } + Json object(const char* key) const { + const Json* v = doc_.find(key); + if (!v || v->is_null()) return Json(); + if (!v->is_object()) bad(std::string(key) + " must be an object"); + return *v; + } + std::vector strings(const char* key) const { + std::vector out; + const Json* v = doc_.find(key); + if (!v || v->is_null()) return out; + if (!v->is_array()) bad(std::string(key) + " must be an array"); + for (const Json& s : v->as_array()) { + if (!s.is_string()) bad(std::string(key) + " must hold strings"); + out.push_back(s.as_string()); + } + return out; + } + std::string bytes(const char* key) const { + std::string b64 = text(key), raw; + if (b64.empty()) return raw; + if (!security::crypto::Base64::decode(b64, raw)) bad(std::string(key) + " is not valid base64"); + return raw; + } + bool confirm() const { return flag("confirm"); } + const Json& doc() const { return doc_; } + + private: + Json doc_ = Json::object(); +}; + +std::string b64(const std::string& raw) { return security::crypto::Base64::encode(raw); } + +const au::ProviderInfo& provider_of(const Body& b) { + std::string name = b.text("provider", true); + const au::ProviderInfo* p = au::Providers::find(name); + if (!p) bad("provider must be one of elevenlabs, assemblyai, deepgram, cartesia, speechmatics, hume"); + return *p; +} + +Json dry_run(const std::string& provider, const std::string& what, const std::string& request) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set("provider", Json(provider)); + o.set("what", Json(what)); + o.set("request", Json(request)); + o.set("hint", Json("send \"confirm\":true to run it")); + return o; +} + +Json audio_json(const std::string& audio, const std::string& content_type) { + Json o = Json::object(); + o.set("content_type", Json(content_type)); + o.set("bytes", Json(static_cast(audio.size()))); + if (audio.size() <= kMaxAudioEcho) o.set("audio_base64", Json(b64(audio))); + else o.set("audio_omitted", Json("over the 8 MiB echo cap")); + return o; +} + +// The service for a provider's table, built from the environment. +std::unique_ptr service_for(const std::string& name) { + if (name == "elevenlabs") return std::make_unique(au::elevenlabs::calls(), au::elevenlabs::ElevenEndpoints::from_env()); + if (name == "assemblyai") return std::make_unique(au::assemblyai::calls(), au::assemblyai::AssemblyEndpoints::from_env()); + if (name == "deepgram") return std::make_unique(au::deepgram::calls(), au::deepgram::DeepgramEndpoints::from_env()); + if (name == "cartesia") return std::make_unique(au::cartesia::calls(), au::cartesia::CartesiaEndpoints::from_env()); + if (name == "speechmatics") return std::make_unique(au::speechmatics::calls(), au::speechmatics::SpeechmaticsEndpoints::from_env()); + return std::make_unique(au::hume::calls(), au::hume::HumeEndpoints::from_env()); +} + +template +auto with_segments(const std::vector& s, F&& f) -> decltype(f(std::initializer_list{})) { + switch (s.size()) { + case 0: return f({}); + case 1: return f({s[0]}); + case 2: return f({s[0], s[1]}); + case 3: return f({s[0], s[1], s[2]}); + case 4: return f({s[0], s[1], s[2], s[3]}); + default: bad("at most 4 path segments"); + } +} + +class AudioRoutes { + public: + Json tts(const Body& b) const { + const au::ProviderInfo& p = provider_of(b); + if (!p.synthesizer) bad(std::string(p.title) + " has no speech synthesis product"); + au::SpeechRequest r; + r.text = b.text("text", true); + r.voice = b.text("voice"); + r.model = b.text("model"); + r.language = b.text("language"); + if (!b.text("format").empty()) r.format = b.text("format"); + r.sample_rate = static_cast(b.number("sample_rate")); + r.speed = b.real("speed"); + return guarded([&]() -> Json { + au::validate(r); + auto synth = std::make_shared(au::SpeechRegistry::synthesizer(p.name)); + if (!b.confirm()) { + std::size_t chunks = au::split_text(r.text, std::min(synth->max_chars(), std::size_t{5000})).size(); + Json o = dry_run(p.name, "synthesize", std::to_string(r.text.size()) + " characters, " + r.format); + o.set("chunks", Json(static_cast(chunks))); + o.set("max_chars_per_call", Json(static_cast(synth->max_chars()))); + return o; + } + au::SpeechResult out = synth->synthesize(r); + Json o = audio_json(out.audio, out.content_type); + o.set("provider", Json(out.provider)); + o.set("format", Json(out.format)); + return o; + }); + } + + Json stt(const Body& b) const { + const au::ProviderInfo& p = provider_of(b); + if (!p.recognizer) bad(std::string(p.title) + " has no speech recognition product"); + au::TranscribeRequest r; + r.url = b.text("url"); + r.audio = b.bytes("data_base64"); + if (!b.text("filename").empty()) r.filename = b.text("filename"); + if (!b.text("content_type").empty()) r.content_type = b.text("content_type"); + r.language = b.text("language"); + r.model = b.text("model"); + r.diarize = b.flag("diarize"); + r.keywords = b.strings("keywords"); + if (b.number("max_wait_seconds") > 0) r.max_wait_seconds = static_cast(b.number("max_wait_seconds")); + return guarded([&]() -> Json { + au::validate(r); + if (!b.confirm()) { + return dry_run(p.name, "transcribe", + (r.url.empty() ? std::to_string(r.audio.size()) + " bytes of audio" : r.url) + (r.diarize ? ", diarized" : "")); + } + au::Transcript t = au::SpeechRegistry::recognizer(p.name)->transcribe(r); + Json o = Json::object(); + o.set("provider", Json(t.provider)); + o.set("text", Json(t.text)); + o.set("language", Json(t.language)); + o.set("duration", Json(t.duration)); + Json words = Json::array(); + for (const au::TranscriptWord& w : t.words) { + if (words.as_array().size() >= 2000) break; + Json j = Json::object(); + j.set("text", Json(w.text)); + j.set("start", Json(w.start)); + j.set("end", Json(w.end)); + if (!w.speaker.empty()) j.set("speaker", Json(w.speaker)); + words.push(j); + } + o.set("words", words); + return o; + }); + } + + Json voices(const Body& b) const { + const au::ProviderInfo& p = provider_of(b); + if (!p.synthesizer) bad(std::string(p.title) + " has no voices"); + return guarded([&]() -> Json { return au::SpeechRegistry::synthesizer(p.name)->voices(); }); + } + + // Any row of a provider's call table by id. Reads run; everything else is a dry run unless confirmed. + Json call(const Body& b) const { + const au::ProviderInfo& p = provider_of(b); + std::string id = b.text("id", true); + std::vector segments = b.strings("segments"); + Json query = b.object("query"); + Json body = b.object("body"); + return guarded([&]() -> Json { + auto svc = service_for(p.name); + const au::CallSpec& spec = svc->table().get(id); + std::string kind = spec.body; + au::Query q = svc->query_from(id, query); + std::string method = spec.method; + std::string preview = with_segments(segments, [&](std::initializer_list s) { + return svc->preview(id, s, kind == "json" && body.is_object() ? &body : nullptr, q); + }); + bool write = method != "GET"; + if (write && !b.confirm()) return dry_run(p.name, id, preview); + Json out = Json::object(); + out.set("provider", Json(p.name)); + out.set("id", Json(id)); + if (kind == "bytes") { + std::string data = b.bytes("data_base64"); + std::string ct = b.text("content_type"); + llm::RawResponse raw = with_segments(segments, [&](std::initializer_list s) { + return svc->call_bytes(id, s, data, ct, q); + }); + out.set("response", Json(raw.body.substr(0, 65536))); + return out; + } + if (kind == "form") { + clients::body::MultipartBody form; + Json texts = b.object("form_text"); + if (texts.is_object()) + for (const auto& [k, v] : texts.as_object()) { + if (!v.is_string()) bad("form_text values must be strings"); + form.add_text(k, v.as_string()); + } + std::string data = b.bytes("data_base64"); + if (!data.empty()) { + std::string field = b.text("file_field").empty() ? "file" : b.text("file_field"); + std::string ct = b.text("content_type").empty() ? "application/octet-stream" : b.text("content_type"); + form.add_file(field, b.text("filename").empty() ? "upload" : b.text("filename"), ct, data); + } + llm::RawResponse raw = with_segments(segments, [&](std::initializer_list s) { + return svc->call_form_raw(id, s, form, q); + }); + out.set("response", Json(raw.body.substr(0, 65536))); + return out; + } + if (au::is_destructive(spec)) { + out.set("response", with_segments(segments, [&](std::initializer_list s) { + return svc->call_destructive(id, s, kind == "json" ? &body : nullptr, q, true); + })); + return out; + } + std::string resp = spec.response; + if (resp == "bytes" || resp == "text") { + llm::RawResponse raw = with_segments(segments, [&](std::initializer_list s) { + return svc->call_raw(id, s, kind == "json" ? &body : nullptr, q); + }); + Json a = resp == "text" ? Json::object() : audio_json(raw.body, raw.content_type); + if (resp == "text") a.set("text", Json(raw.body.substr(0, 65536))); + out.set("response", a); + return out; + } + out.set("response", with_segments(segments, [&](std::initializer_list s) { + return svc->call(id, s, kind == "json" ? &body : nullptr, q); + })); + return out; + }); + } + + // One shot streaming synthesis over the provider's WebSocket. + Json session(const Body& b) const { + const au::ProviderInfo& p = provider_of(b); + std::string text = b.text("text", true); + std::string voice = b.text("voice"); + std::string name = p.name; + if (name != "elevenlabs" && name != "deepgram" && name != "cartesia" && name != "hume") + bad("streaming synthesis is available for elevenlabs, deepgram, cartesia and hume"); + if (name != "hume" && name != "deepgram" && voice.empty()) bad("voice is required"); + return guarded([&]() -> Json { + std::string key = llm::Config::get(p.key_env); + if (!b.confirm()) return dry_run(name, "streaming synthesis", std::to_string(text.size()) + " characters over WebSocket"); + if (key.empty()) throw llm::NotConfiguredError(name, p.key_env); + Json o = Json::object(); + o.set("provider", Json(name)); + if (name == "elevenlabs") { + au::elevenlabs::TtsStreamOptions opt; + opt.voice_id = voice; + opt.model_id = b.text("model"); + opt.output_format = b.text("output_format"); + au::elevenlabs::TtsInputStream s(key, opt); + s.connect(); + au::elevenlabs::TtsStreamResult r = s.speak(text, 20000); + s.close(); + if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error); + Json a = audio_json(r.audio, "audio/mpeg"); + a.set("chunks", Json(static_cast(r.chunks))); + a.set("final", Json(r.is_final)); + return a; + } + if (name == "deepgram") { + au::deepgram::StreamParams params; + params.add("model", voice.empty() ? "aura-2-thalia-en" : voice).add("encoding", "linear16").add("sample_rate", "24000"); + au::deepgram::SpeakStream s(key, params); + s.connect(); + s.speak(text); + s.flush(); + au::deepgram::SpeakStreamResult r = s.collect(20000); + s.end(); + s.close(); + if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error); + Json a = audio_json(r.audio, "audio/pcm"); + a.set("chunks", Json(static_cast(r.chunks))); + a.set("flushed", Json(r.flushed)); + return a; + } + if (name == "cartesia") { + au::cartesia::TtsSocket s(key); + s.connect(); + au::cartesia::TtsSocketRequest r; + r.context_id = "demo-1"; + r.transcript = text; + r.voice_id = voice; + r.continue_ = 0; + s.generate(r); + au::cartesia::TtsSocketResult res = s.collect("demo-1", 20000); + s.close(); + if (!res.error.empty()) throw llm::ApiError(name, 502, "stream_error", res.error); + Json a = audio_json(res.audio, "audio/pcm"); + a.set("chunks", Json(static_cast(res.chunks))); + a.set("done", Json(res.done)); + return a; + } + au::hume::HumeStreamOptions opt; + opt.format_type = "mp3"; + opt.no_binary = 1; + au::hume::HumeTtsStream s(key, opt); + s.connect(); + s.send_text(text, voice); + s.finish(); + au::hume::HumeStreamResult r = s.collect(20000); + s.close(); + if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error); + Json a = audio_json(r.audio, "audio/mpeg"); + a.set("chunks", Json(static_cast(r.chunks))); + a.set("last_chunk", Json(r.last_chunk)); + return a; + }); + } + + // Local: the URL a WebSocket channel would open, with the credential left out. + Json ws_preview(const Body& b) const { + const au::ProviderInfo& p = provider_of(b); + std::string channel = b.text("channel", true); + Json params = b.object("params"); + auto str = [&](const char* k) { + const Json* v = params.is_object() ? params.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); + }; + return guarded([&]() -> Json { + std::string url; + std::string key = "preview-only"; + if (std::string(p.name) == "elevenlabs" && channel == "tts-stream-input") { + au::elevenlabs::TtsStreamOptions o; + o.voice_id = str("voice_id"); + o.model_id = str("model_id"); + o.output_format = str("output_format"); + url = au::elevenlabs::TtsInputStream(key, o).url(); + } else if (std::string(p.name) == "elevenlabs" && channel == "stt-realtime") { + au::elevenlabs::ScribeStreamOptions o; + o.model_id = str("model_id"); + o.audio_format = str("audio_format"); + o.commit_strategy = str("commit_strategy"); + url = au::elevenlabs::ScribeStream(key, o).url(); + } else if (std::string(p.name) == "deepgram" && (channel == "listen-v1" || channel == "listen-v2")) { + au::deepgram::StreamParams sp; + if (params.is_object()) + for (const auto& [k, v] : params.as_object()) + if (v.is_string()) sp.add(k, v.as_string()); + url = au::deepgram::ListenStream(key, sp, channel == "listen-v2").url(); + } else if (std::string(p.name) == "deepgram" && (channel == "speak-v1" || channel == "speak-v2")) { + au::deepgram::StreamParams sp; + if (params.is_object()) + for (const auto& [k, v] : params.as_object()) + if (v.is_string()) sp.add(k, v.as_string()); + url = au::deepgram::SpeakStream(key, sp, channel == "speak-v2").url(); + } else if (std::string(p.name) == "assemblyai" && channel == "streaming") { + au::assemblyai::StreamParams sp; + if (params.is_object()) + for (const auto& [k, v] : params.as_object()) + if (v.is_string()) sp.add(k, v.as_string()); + url = au::assemblyai::TranscriptStream(key, sp).url(); + } else if (std::string(p.name) == "cartesia" && channel == "tts-websocket") { + url = au::cartesia::TtsSocket(key).url(); + } else if (std::string(p.name) == "speechmatics" && channel == "realtime") { + au::speechmatics::RealtimeConfig c; + if (!str("language").empty()) c.language = str("language"); + url = au::speechmatics::RealtimeSession(key, c).url(); + } else if (std::string(p.name) == "hume" && channel == "tts-stream-input") { + url = au::hume::HumeTtsStream(key, au::hume::HumeStreamOptions()).url(); + } else if (std::string(p.name) == "hume" && channel == "evi-chat") { + au::hume::EviOptions o; + o.config_id = str("config_id"); + url = au::hume::EviChat(key, o).url(); + } else { + bad(std::string("no such channel for ") + p.name + ": " + channel); + } + Json o = Json::object(); + o.set("provider", Json(p.name)); + o.set("channel", Json(channel)); + o.set("url", Json(url)); + o.set("credential", Json("sent as a header, never in the URL")); + return o; + }); + } + + // Local: sentence aware splitting with the package's caps. + Json split(const Body& b) const { + std::string text = b.text("text", true); + long max_chars = b.number("max_chars", 500); + if (max_chars < 1 || max_chars > 100000) bad("max_chars must be 1..100000"); + return guarded([&]() -> Json { + Json arr = Json::array(); + for (const std::string& c : au::split_text(text, static_cast(max_chars))) arr.push(Json(c)); + Json o = Json::object(); + o.set("chunks", arr); + o.set("count", Json(static_cast(arr.as_array().size()))); + return o; + }); + } +}; + +} // namespace + void register_ai_audio_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + fw::Environment* env = services.templates.get(); + services.templates->add_template("ai_audio.html", kPage); + services.add_feature("ai-audio", "/ai/audio"); + auto routes = std::make_shared(); + + router.get("/ai/audio", "ai-audio-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Audio"); + return fw::Response(200, env->render("ai_audio.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/ai/audio/status", "ai-audio-status", [](fw::Request req) { + (void)req; + Json providers = Json::array(); + for (const au::ProviderInfo& p : au::Providers::all()) { + Json j = Json::object(); + j.set("name", Json(p.name)); + j.set("title", Json(p.title)); + j.set("key_env", Json(p.key_env)); + j.set("key_configured", Json(!llm::Config::get(p.key_env).empty())); + if (std::string(p.extra_env).size()) { + j.set("extra_env", Json(p.extra_env)); + j.set("extra_configured", Json(!llm::Config::get(p.extra_env).empty())); + } + j.set("key_page", Json(p.key_page)); + j.set("synthesizer", Json(p.synthesizer)); + j.set("recognizer", Json(p.recognizer)); + j.set("websockets", Json(p.websockets)); + j.set("calls", Json(static_cast(p.table().size()))); + providers.push(j); + } + Json absent = Json::array(); + for (const au::Providers::Absent& a : au::Providers::absent()) { + Json j = Json::object(); + j.set("name", Json(a.name)); + j.set("reason", Json(a.reason)); + absent.push(j); + } + Json o = Json::object(); + o.set("providers", providers); + o.set("absent", absent); + o.set("calls_total", Json(static_cast(au::Providers::call_count()))); + o.set("max_body_bytes", Json(static_cast(kMaxBody))); + return json_out(o); + }); + + router.get("/ai/audio/calls", "ai-audio-calls", [](fw::Request req) { + Json arr = Json::array(); + std::string want; + if (auto it = req.query.find("provider"); it != req.query.end()) want = it->second; + for (const au::ProviderInfo& p : au::Providers::all()) { + if (!want.empty() && want != p.name) continue; + for (std::size_t i = 0; i < p.table().size(); ++i) { + const au::CallSpec& s = p.table().at(i); + Json row = Json::object(); + row.set("provider", Json(p.name)); + row.set("id", Json(s.id)); + row.set("http", Json(s.method)); + row.set("path", Json(s.path)); + row.set("host", Json(s.host)); + row.set("body", Json(s.body)); + row.set("response", Json(s.response)); + arr.push(row); + if (arr.as_array().size() >= 2000) break; + } + } + Json o = Json::object(); + o.set("calls", arr); + o.set("truncated", Json(want.empty() && au::Providers::call_count() > 2000)); + return json_out(o); + }); + + using Handler = Json (AudioRoutes::*)(const Body&) const; + struct Route { const char* path; const char* name; Handler handler; }; + const Route table[] = { + {"/ai/audio/tts", "ai-audio-tts", &AudioRoutes::tts}, + {"/ai/audio/stt", "ai-audio-stt", &AudioRoutes::stt}, + {"/ai/audio/voices", "ai-audio-voices", &AudioRoutes::voices}, + {"/ai/audio/call", "ai-audio-call", &AudioRoutes::call}, + {"/ai/audio/session", "ai-audio-session", &AudioRoutes::session}, + {"/ai/audio/ws/preview", "ai-audio-ws-preview", &AudioRoutes::ws_preview}, + {"/ai/audio/split", "ai-audio-split", &AudioRoutes::split}, + }; + for (const Route& r : table) { + Handler h = r.handler; + router.post(r.path, r.name, [routes, h](fw::Request req) { + Body body(req); + return json_out(((*routes).*h)(body)); + }); + } } } // namespace molo::example::app diff --git a/example/assets/ai_audio.js b/example/assets/ai_audio.js new file mode 100644 index 0000000..00365d5 --- /dev/null +++ b/example/assets/ai_audio.js @@ -0,0 +1,82 @@ +// Audio page: thin client over /ai/audio/* (all limits are enforced server side). +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("au-out"), err = $("au-error"), player = $("au-audio"); + + var templates = { + "tts": { text: "Hello from the audio demo.", voice: "", format: "mp3", confirm: false }, + "stt": { url: "https://example.com/audio.mp3", language: "en", diarize: false, confirm: false }, + "voices": {}, + "call": { id: "get_models", segments: [], query: {}, body: {}, confirm: false }, + "session": { text: "Streaming hello.", voice: "", confirm: false }, + "ws/preview": { channel: "tts-stream-input", params: { voice_id: "VOICE_ID" } }, + "split": { text: "One sentence. Another sentence. A third one.", max_chars: 20 } + }; + var providerDefaults = { + "elevenlabs": { call: { id: "get_models" }, ws: "tts-stream-input" }, + "deepgram": { call: { id: "models_list_3" }, ws: "listen-v1" }, + "cartesia": { call: { id: "get_status" }, ws: "tts-websocket" }, + "speechmatics": { call: { id: "jobs_list", query: { limit: 5 } }, ws: "realtime" }, + "hume": { call: { id: "voices_list", query: { provider: "HUME_AI", page_size: 5 } }, ws: "tts-stream-input" }, + "assemblyai": { call: { id: "listTranscripts", query: { limit: 5 } }, ws: "streaming" } + }; + + function show(j, ok) { + if (ok) { + err.textContent = ""; + if (j && j.audio_base64) { + player.hidden = false; + player.src = "data:" + (j.content_type || "audio/mpeg") + ";base64," + j.audio_base64; + var c = {}; + for (var k in j) { if (k !== "audio_base64") { c[k] = j[k]; } } + j = c; + } else { player.hidden = true; } + out.textContent = JSON.stringify(j, null, 2); + } else { + err.textContent = j.message || "Request failed"; + out.textContent = ""; + player.hidden = true; + } + } + + function fill() { + var route = $("au-route").value, prov = $("au-provider").value; + var t = JSON.parse(JSON.stringify(templates[route])); + var d = providerDefaults[prov] || {}; + if (route === "call" && d.call) { for (var k in d.call) { t[k] = d.call[k]; } } + if (route === "ws/preview" && d.ws) { t.channel = d.ws; } + if (route !== "split") { t.provider = prov; } + $("au-body").value = JSON.stringify(t, null, 1); + } + + function send() { + var body; + try { body = JSON.parse($("au-body").value || "{}"); } + catch (e) { err.textContent = "Body is not valid JSON"; return; } + err.textContent = ""; + out.textContent = "..."; + fetch("/ai/audio/" + $("au-route").value, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + show(j, res.ok); + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + + fetch("/ai/audio/status").then(function (r) { return r.json(); }).then(function (s) { + var missing = [], found = []; + s.providers.forEach(function (p) { (p.key_configured ? found : missing).push(p.key_env); }); + $("au-config").textContent = (found.length ? "Keys found: " + found.join(", ") + ". " : "") + + (missing.length ? "No key for: " + missing.join(", ") + " (those routes answer 503)." : ""); + }); + $("au-route").onchange = fill; + $("au-provider").onchange = fill; + $("au-send").onclick = send; + fill(); +})(); diff --git a/src/vendors/audio/SPEC.md b/src/vendors/audio/SPEC.md new file mode 100644 index 0000000..75fdb7a --- /dev/null +++ b/src/vendors/audio/SPEC.md @@ -0,0 +1,273 @@ +# Audio vendors - Specification + +Owned directory: `src/vendors/audio/`. Namespace `molo::audio` (one sub +namespace per provider: `elevenlabs`, `assemblyai`, `deepgram`, `cartesia`, +`speechmatics`, `hume`). Plan: `plans/18-breadth.md` section 2. Method: plans +16 and 17. Depends on `src/llm` (`ApiClient`, `Service`, `ApiError`, +`SseDecoder`, `Config`) and `src/google/live` (RFC 6455 `WsClient`). It does +not edit either. + +Status 2026-10-25: implemented; local suite green; no vendor key was available, +so no success response from any vendor has been seen (see Verified live and +Unverified). + +## 1. Scope + +| Provider | REST rows | WebSockets | Neutral seam | +|---|---|---|---| +| ElevenLabs | 325 | TTS stream-input, realtime STT (Scribe), conversational AI agent | synthesizer, recognizer | +| AssemblyAI | 14 (api 10, streaming token 1, LLM Gateway 3) | streaming v3 | recognizer | +| Deepgram | 51 | listen v1, listen v2 (Flux), speak v1, speak v2, voice agent | synthesizer, recognizer | +| Cartesia | 53 | TTS (contexts), STT turns (auto finalize), STT manual finalize | synthesizer, recognizer | +| Speechmatics | 17 (batch, TTS preview, temporary keys, management) | realtime v2 | synthesizer, recognizer | +| Hume | 49 | TTS stream-input, EVI chat | synthesizer | + +ElevenLabs rows cover text to speech (buffered, streamed, timestamps), speech +to text, voices (list, get, settings, shared library, voice design, PVC and +cloning reads and writes), dubbing, sound effects, music, conversational AI +agents (agents, conversations, knowledge base, tools, phone numbers, tests, +MCP servers, batch calling, ...), history, models, user, subscription, +pronunciation dictionaries, voice changer, audio isolation, forced alignment, +text to dialogue. Excluded on purpose (explicit list in `tools/check_spec.py` +`EL_EXCLUDE`, 96 operations): workspace administration, Studio projects, +Audio Native, productions, flows, speech engine, assets, service accounts. + +PlayHT: no public API exists to wrap. `api.play.ht` and `api.play.ai` do not +resolve (checked 2026-10-25). Nothing is invented; `Providers::absent()` says so. + +AssemblyAI LeMUR: no longer in the OpenAPI document or the docs index, and +`POST /lemur/v3/generate/task` answers 404 unauthenticated. The LLM Gateway +(chat completions, speech understanding, models) is wrapped instead. +AssemblyAI's Sync and Dictation APIs have docs pages but no machine readable +specification: not wrapped (Gaps). + +Hume has no speech to text product. EVI is a voice agent; Expression +Measurement analyses prosody (REST batch runs wrapped; its streaming +WebSocket is not). + +## 2. Module map (one concern per file) + +Shared layer (`src/vendors/audio/`): + +| File | Concern | +|---|---| +| `caps.h` | static ceilings | +| `call_spec.h/.cpp` | `CallSpec` row, `CallTable`, path rendering with id validation, destructive heuristic | +| `query.h/.cpp` | percent-encoded query builder with a size cap | +| `hosts.h/.cpp` | `Hosts` (named hosts), `ByteStreamer` (incremental bodies, JSON error sniff), `HostFactory`, allowlisted base URL overrides | +| `audio_service.h/.cpp` | `AudioService`: table driven `call`, `call_raw`, `call_bytes`, `call_form`, `call_stream`, `call_destructive`, `preview`; every guard before the network | +| `socket.h/.cpp` | `AudioSocket` over `WsClient`, `Vocabulary` (client / server message names) | +| `speech.h/.cpp` | the neutral seam: `SpeechSynthesizer`, `SpeechRecognizer`, `validate`, `split_text`, `ChunkedSynthesizer`, `PollPolicy` / `poll` | +| `providers.h/.cpp` | one descriptor per vendor (env var, key page, products, table) | +| `registry.cpp` | `SpeechRegistry`: name to seam object | + +Per provider (`/`): `calls.cpp` (the table between `CALLS-BEGIN` / +`CALLS-END`), `calls.h` (ids), `client.h/.cpp` (endpoints, auth, hosts), +service classes (ElevenLabs: `tts`, `stt`, `voices`, `creative`, `dubbing`, +`agents`, `account`, plus `tts_stream`, `scribe_stream`, `convai_session`; +AssemblyAI: `transcripts`, `gateway`, `stream`; Deepgram: `listen`, `speak`, +`read`, `manage`, `streams`; Cartesia: `tts`, `voices`, `stt`, `tts_socket`; +Speechmatics: `batch`, `tts`, `manage`, `realtime`; Hume: `tts`, `evi`, +`sockets`), and `speech.h/.cpp` (the seam implementations). + +Demo: `example/app/ai/AudioDemo.cpp`, asset `example/assets/ai_audio.js`. +Tests: `tests/src/vendors/audio/`, `tests/example/ai/audio/`. + +## 3. The neutral seam + +`SpeechSynthesizer::synthesize(SpeechRequest)` returns audio bytes; +`SpeechRecognizer::transcribe(TranscribeRequest)` returns a `Transcript` (text, +language, duration, words with times and speakers, the provider's own document). +Formats are `mp3`, `wav`, `pcm` and map to each provider's own names (ElevenLabs +`mp3_44100_128`, Deepgram `encoding=linear16&container=wav`, Cartesia +`{container, encoding, sample_rate}`, ...). `ChunkedSynthesizer` splits text over +a provider's per call limit at sentence boundaries and joins mp3 / pcm audio +(wav is refused past one chunk). Asynchronous recognizers (AssemblyAI, +Speechmatics) upload, submit and poll with `poll()`: the clock is the sum of +the sleeps, the interval has a floor of 500 ms, the wait a ceiling of 1800 s. +The seam lives in this package; `src/llm` is untouched. + +## 4. Static caps (see `caps.h` and each `*Caps` struct) + +| Knob | Value | +|---|---| +| Synthesis input | 100000 chars (ElevenLabs 40000, Deepgram 2000, Cartesia 20000, Hume 5000 per utterance) | +| Splitter chunks | 64 per call, 5000 chars default | +| JSON body | 8 MiB; demo request body 4 MiB | +| Audio in / out kept | 128 MiB / 64 MiB; streamed audio 64 MiB | +| Poll | interval 500..15000 ms, wait <= 1800 s | +| Page walks | 20 pages, 1000 items | +| WebSocket | session 1800 s, wait per read <= 120 s, 4096 events per collect, frame <= 4 MiB | +| Keyword / keyterm lists | 100 | + +## 5. Contracts + +- Key present or `NotConfiguredError` (HTTP 503) before any traffic; local + validation comes first, so bad input is a 400 (`INVALID_ARGUMENT`), not a 503. +- The key travels in a header only: `xi-api-key` (ElevenLabs), `Authorization: + ` (AssemblyAI), `Authorization: Token ` (Deepgram), `Authorization: + Bearer ` + `Cartesia-Version: 2026-08-14` (Cartesia), `Authorization: + Bearer ` (Speechmatics), `X-Hume-Api-Key` (Hume). Never in a URL, an + error or `url()` of a socket. Two documented exceptions are handled: an + AssemblyAI temporary token and an ElevenLabs signed agent URL carry a + credential in the URL by the vendor's design; both classes hide it from `url()`. +- Every REST call is a table row. Path segments pass `Service::valid_id` (an + `{email}` segment is validated and percent-encoded). Body fields and query + parameters are checked against the row; required fields must be present. +- `call` refuses destructive rows (DELETE, or ids with cancel / disable / + remove / revoke / purge / leave); `call_destructive` needs `confirm=true`. + Typed writes (create agent, clone voice, start dubbing, create API key, ...) + take `confirm` too. +- Base URL overrides (`ELEVENLABS_BASE_URL`, `DEEPGRAM_BASE_URL`, + `ASSEMBLYAI_BASE_URL`, `SPEECHMATICS_BATCH_URL`, `SPEECHMATICS_RT_URL`) are + accepted only when they equal a documented regional host. +- Credentials: `ELEVENLABS_API_KEY`, `ASSEMBLYAI_API_KEY`, `DEEPGRAM_API_KEY`, + `CARTESIA_API_KEY`, `SPEECHMATICS_API_KEY`, `HUME_API_KEY`, and + `SPEECHMATICS_MANAGEMENT_TOKEN` for the Speechmatics management host. + +## 6. Required credentials and consoles + +| Provider | Env var | Page | +|---|---|---| +| ElevenLabs | `ELEVENLABS_API_KEY` | https://elevenlabs.io/app/settings/api-keys (unverified) | +| AssemblyAI | `ASSEMBLYAI_API_KEY` | https://www.assemblyai.com/dashboard/api-keys (unverified) | +| Deepgram | `DEEPGRAM_API_KEY` | https://console.deepgram.com (unverified) | +| Cartesia | `CARTESIA_API_KEY` | https://play.cartesia.ai/keys (unverified) | +| Speechmatics | `SPEECHMATICS_API_KEY` | https://portal.speechmatics.com/settings/api-keys/ (docs) | +| Speechmatics | `SPEECHMATICS_MANAGEMENT_TOKEN` | docs.speechmatics.com/administration/management-tokens | +| Hume | `HUME_API_KEY` | https://app.hume.ai/keys (unverified) | + +## Sources + +| Provider | Machine readable sources used | Second source | +|---|---|---| +| ElevenLabs | `https://api.elevenlabs.io/openapi.json` (421 operations); AsyncAPI 2.6 `https://elevenlabs.io/docs/asyncapi/{elevenagents,api-reference,api-reference-2}.json` | `llms.txt` page index | +| AssemblyAI | `https://www.assemblyai.com/docs/openapi.yaml` (v1.3.4), `docs/api-reference/specs/{streaming-token,llm-gateway,streaming}.yaml` (AsyncAPI 3) | `assemblyai-node-sdk` generated types, `llms.txt` | +| Deepgram | `deepgram-api-specs` `openapi.yml` and `asyncapi.yml` (Fern generated) | `llms.txt` | +| Cartesia | `cartesia-python` (Stainless, generated from Cartesia's OpenAPI): `api.md`, `.stats.yml` (53 endpoints), `*_params.py`, event types; `docs.cartesia.ai/api-reference/tts/websocket.md` (AsyncAPI page) | docs host answers bots with a challenge page, so no direct OpenAPI fetch | +| Speechmatics | `speechmatics-python-sdk` (`batch`, `rt`, `tts`), `docs.speechmatics.com/api-ref/*.md` and `get-started/authentication.md` | no OpenAPI file published outside the rendered docs | +| Hume | `dev.hume.ai/openapi/{text-to-speech-tts,speech-to-speech-evi,expression-measurement-api}.json`; `hume-python-sdk` (`tts/raw_client.py` for the synthesis calls the OpenAPI omits, event types for the sockets) | `llms.txt` | + +## Verified against spec + +`python3 -I src/vendors/audio/tools/check_spec.py` (stdlib only, read-only +network), run 2026-10-25, exit 0: + +``` +elevenlabs ok enum values checked 28, rows checked 325, spec operations covered 325, spec operations excluded 96, ws names checked 95 +assemblyai ok rows checked 14, spec operations covered 14, ws names checked 42 +deepgram ok rows checked 52, spec operations covered 50, ws names checked 133 +cartesia ok rows checked 53, sdk endpoints (stats) 53, ws names checked 40 +speechmatics ok enum values checked 6, rows checked 17, ws names checked 29 +hume ok rows checked 49, spec operations covered 43, ws names checked 37 +``` + +It fails on: a row absent from its source; a body field, required field, or +query parameter not in the source; an operationId mismatch; a spec operation +in a covered area that is neither in the table nor excluded; a WebSocket query +parameter or message name in the C++ that the AsyncAPI document or SDK types do +not contain; the ElevenLabs `output_format` enum differing from the code; and +AssemblyAI's spec mentioning LeMUR again. Mutation check: renaming a Deepgram +query parameter in `calls.cpp` produced two failures (then reverted). +YAML documents are scanned by indentation (no YAML parser in the stdlib). + +## Verified live + +Unauthenticated probes, 2026-10-25 (curl, and `MOLO_LIVE_NET=1 make ai-test` +runs `audio_live.unauthenticated_probes` against the same hosts): + +| Probe | Result | +|---|---| +| `GET api.elevenlabs.io/v1/user`, `/v1/history`, `/v1/convai/agents`, `POST /v1/text-to-speech/{id}` | 401 `authentication_error`, body `{"detail":{"type","code","message","status","request_id"}}` | +| `GET api.elevenlabs.io/v1/voices` | 200 without a key (public premade voices) | +| `GET api.elevenlabs.io/v1/models` | 404 `workspace_not_found` ("Workspace 1anonymous1") | +| `GET/POST api.assemblyai.com/v2/transcript` | 401 `{"error": "Authentication error, API token missing/invalid"}` | +| `GET streaming.assemblyai.com/v3/token` | 422: header `authorization` and query `expires_in_seconds` required | +| `GET llm-gateway.assemblyai.com/v1/models` | 200 without a key | +| `POST api.assemblyai.com/lemur/v3/generate/task` | 404 `Not found` | +| `GET api.deepgram.com/v1/projects`, `POST /v1/listen`, `POST /v1/speak` | 401 `INVALID_AUTH` / `UNAUTHORIZED` | +| `GET api.deepgram.com/v1/models`, `agent.deepgram.com/v1/agent/settings/think/models` | 200 without a key | +| `GET api.cartesia.ai/` | 200 `{"ok":true,"version":"2026-08-14"}` | +| `GET api.cartesia.ai/voices`, `/agents` | 401 `Unauthorized: You must be logged in...` | +| `POST api.cartesia.ai/tts/bytes` without `Cartesia-Version` | 400 `Cartesia-Version header is required in YYYY-MM-DD form, latest version is 2026-08-14` | +| `GET asr.api.speechmatics.com/v2/jobs`, `eu1.asr...`, `POST preview.tts.speechmatics.com/generate/sarah` | 401 (nginx page) | +| `GET mp.speechmatics.com/v1/projects` | 401 `{"code":401,"error":"Unauthorized"}` | +| `GET api.hume.ai/v0/tts/voices`, `/v0/evi/chats`, `/v0/evi/configs`, `POST /v0/tts` | 401 `Credentials are required to access this resource.` | +| `GET api.cloud.hume.ai/v1/expression/run` | 401 `{"message":"Unauthorized"}` | +| `GET api.hume.ai/v0/batch/jobs` | 401 (the older batch path still exists; not wrapped, not in the current spec) | +| `api.play.ht`, `api.play.ai` | do not resolve | + +Opt-in authenticated read-only tests (`MOLO_LIVE_NET=1` plus the key; one +smallest ElevenLabs generation only with `MOLO_LIVE_PAID=1`) print a loud +`SKIP` without the key. + +Local suite: `make ai-test` runs 110 audio tests (`audio_core`, `audio_elevenlabs`, +`audio_deepgram`, `audio_assemblyai`, `audio_cartesia`, `audio_speechmatics`, +`audio_hume`, `audio_live`), all WebSockets against a test-owned loopback +server; `make -f example/example.mk example-test` runs 5 audio route tests +(98 checks). + +## Unverified + +Everything below was inferred, not read from a published specification or seen +in a response: + +- Any success response body of any vendor. Folding code (`fold`, `collect`) is + checked against shapes copied from the OpenAPI examples or SDK types, labelled + as fixtures in the tests. +- Authentication on the REST hosts with a real key (header names come from + specs and SDKs; the 401s above prove the paths, not the credential style). +- ElevenLabs `speed` range of voice settings (the OpenAPI states none), the TTS + length ceiling of 40000 characters (largest model limit in the docs). +- Deepgram TTS REST 2000 character limit (TTS guide), `POST /v1/listen` with a + raw audio body and `Content-Type` (the Fern OpenAPI lists only the JSON `url` + form; row source `guide:`), the `Token` scheme for the Agent host. +- AssemblyAI `speakers_expected` range 1..100, LLM Gateway `role` set and + `temperature` range 0..2, and whether `speech_models` is now required on + `POST /v2/transcript`. +- Cartesia: `/tts/websocket` auth by header (the docs page lists only the + `cartesia_version` parameter; the SDK sends the headers), infill and voice + changer multipart part names (`output_format[...]`, `voice[id]`, taken from + the SDK aliases). +- Speechmatics: management API base `https://mp.speechmatics.com/v1` (the + temporary key page shows it; the 401 above shows the host answers), the + management paths use hyphens (`/api-keys`) while the temporary key endpoint + uses an underscore (`/api_keys`), both as documented; job status `expired`; + TTS voices `sarah theo megan jack` and the two formats (SDK enum; no list + endpoint exists). +- Hume: the Expression Measurement host `api.cloud.hume.ai` comes from its + OpenAPI `servers`; `format.type` values mp3 / wav / pcm and + `include_timestamp_types` values from the SDK; WebSocket `Sec-WebSocket` + upgrade acceptance with the header key (the SDK sends the same header). +- ElevenLabs agent WebSocket: a private agent needs a signed URL; the client + messages used (`conversation_initiation_client_data`, `user_audio_chunk`, + `user_message`, `pong`, `client_tool_result`) are in the AsyncAPI document, + their runtime ordering is not exercised against the real service. + +## Gaps + +- `ApiClient::stream` is POST-only SSE with no status or headers, so incremental + audio uses this package's own `ByteStreamer`. It cannot see the HTTP status of a + stream; it treats a small JSON object with `detail` / `error` / `message` at + the start of the body as the vendor's error. Owner `src/llm`: expose status and + headers on streams and a raw-bytes stream. +- `ApiClient::post_multipart` is POST-only; PUT / PATCH multipart goes through + `request_raw` with the encoded body. Owner `src/llm`. +- The shared HTTP client opens a circuit breaker after ten consecutive failures + per client; tests split long call sequences across rigs because of it. +- Not wrapped: AssemblyAI Sync and Dictation APIs (no specification), AssemblyAI + legacy streaming v2, Hume Expression Measurement streaming socket, Deepgram + `/v1/agent` REST configuration beyond the rows in the OpenAPI document (all + are in the table), ElevenLabs multi-context and dialogue WebSockets and the + Realtime Translation socket (documented in the AsyncAPI, no typed client), + Speechmatics Agent STT socket, Speechmatics Flow. +- ElevenLabs excluded areas (section 1) can be added by moving a prefix from + `EL_EXCLUDE` to `EL_INCLUDE` and regenerating rows. +- No retries and no rate-limit handling beyond what the shared client does; no + usage / cost accounting. +- The Deepgram `Token` prefix is applied at `HostFactory::from_env`; tests + inject the final header value. A temporary JWT (`Bearer`) is not supported. + +## Required APIs + +Not a Google package: no Google services are used. diff --git a/src/vendors/audio/assemblyai/calls.cpp b/src/vendors/audio/assemblyai/calls.cpp new file mode 100644 index 0000000..193a35b --- /dev/null +++ b/src/vendors/audio/assemblyai/calls.cpp @@ -0,0 +1,33 @@ +#include "vendors/audio/assemblyai/calls.h" + +namespace molo::audio::assemblyai { + +// One row per REST call. tools/check_spec.py parses the block between the +// markers: {id, method, path, host, body, fields, required, query, response, source}. +// CALLS-BEGIN +static const CallSpec kRows[] = { + {"listTranscripts", "GET", "/v2/transcript", "api", "-", "", "", "limit,status,created_on,before_id,after_id,throttled_only", "json", "openapi:listTranscripts"}, + {"createTranscript", "POST", "/v2/transcript", "api", "json", "audio_url,audio_end_at,audio_start_from,auto_chapters,auto_highlights,content_safety,content_safety_confidence,custom_spelling,disfluencies,domain,entity_detection,filter_profanity,format_text,iab_categories,keyterms_prompt,language_code,language_codes,language_confidence_threshold,language_detection,language_detection_options,multichannel,prompt,punctuate,redact_pii,redact_pii_audio,redact_pii_audio_options,redact_pii_audio_quality,redact_pii_policies,redact_pii_sub,redact_pii_return_unredacted,redact_static_entities,sentiment_analysis,speaker_labels,speaker_options,speakers_expected,speech_models,speech_threshold,speech_understanding,summarization,summary_model,summary_type,remove_audio_tags,temperature,webhook_auth_header_name,webhook_auth_header_value,webhook_url,custom_topics,speech_model,topics", "audio_url", "", "json", "openapi:createTranscript"}, + {"getTranscript", "GET", "/v2/transcript/{transcript_id}", "api", "-", "", "", "", "json", "openapi:getTranscript"}, + {"deleteTranscript", "DELETE", "/v2/transcript/{transcript_id}", "api", "-", "", "", "", "json", "openapi:deleteTranscript"}, + {"getTranscriptParagraphs", "GET", "/v2/transcript/{transcript_id}/paragraphs", "api", "-", "", "", "", "json", "openapi:getTranscriptParagraphs"}, + {"getRedactedAudio", "GET", "/v2/transcript/{transcript_id}/redacted-audio", "api", "-", "", "", "", "json", "openapi:getRedactedAudio"}, + {"getTranscriptSentences", "GET", "/v2/transcript/{transcript_id}/sentences", "api", "-", "", "", "", "json", "openapi:getTranscriptSentences"}, + {"wordSearch", "GET", "/v2/transcript/{transcript_id}/word-search", "api", "-", "", "", "words", "json", "openapi:wordSearch"}, + {"getSubtitles", "GET", "/v2/transcript/{transcript_id}/{subtitle_format}", "api", "-", "", "", "chars_per_caption", "text", "openapi:getSubtitles"}, + {"uploadFile", "POST", "/v2/upload", "api", "bytes", "", "", "", "json", "openapi:uploadFile"}, + {"generateStreamingToken", "GET", "/v3/token", "streaming", "-", "", "", "expires_in_seconds,max_session_duration_seconds", "json", "openapi-streaming-token:generateStreamingToken"}, + {"createChatCompletion", "POST", "/chat/completions", "gateway", "json", "messages,prompt,transcript_id,model,model_region,max_tokens,temperature,stream,tools,tool_choice,response_format,fallbacks,fallback_config,post_processing_steps", "model", "", "json", "openapi-llm-gateway:createChatCompletion"}, + {"listAvailableModels", "GET", "/models", "gateway", "-", "", "", "", "json", "openapi-llm-gateway:listAvailableModels"}, + {"createSpeechUnderstanding", "POST", "/understanding", "gateway", "json", "transcript_id,speech_understanding", "transcript_id,speech_understanding", "", "json", "openapi-llm-gateway:createSpeechUnderstanding"}, +}; +// CALLS-END + +static_assert(sizeof(kRows) / sizeof(kRows[0]) == 14, "assemblyai call table size"); + +const CallTable& calls() { + static const CallTable table("assemblyai", kRows, sizeof(kRows) / sizeof(kRows[0])); + return table; +} + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/calls.h b/src/vendors/audio/assemblyai/calls.h new file mode 100644 index 0000000..e8bd147 --- /dev/null +++ b/src/vendors/audio/assemblyai/calls.h @@ -0,0 +1,30 @@ +#pragma once + +// The AssemblyAI call table: the pre-recorded API (https://www.assemblyai.com/docs/openapi.yaml), +// the streaming token endpoint (docs/api-reference/specs/streaming-token.yaml) and the +// LLM Gateway (docs/api-reference/specs/llm-gateway.yaml). Hosts: api, streaming, gateway. + +#include "vendors/audio/call_spec.h" + +namespace molo::audio::assemblyai { + +const CallTable& calls(); + +struct AssemblyCalls { + static constexpr const char* kUpload = "uploadFile"; + static constexpr const char* kSubmit = "createTranscript"; + static constexpr const char* kList = "listTranscripts"; + static constexpr const char* kGet = "getTranscript"; + static constexpr const char* kDelete = "deleteTranscript"; + static constexpr const char* kSentences = "getTranscriptSentences"; + static constexpr const char* kParagraphs = "getTranscriptParagraphs"; + static constexpr const char* kWordSearch = "wordSearch"; + static constexpr const char* kSubtitles = "getSubtitles"; + static constexpr const char* kRedactedAudio = "getRedactedAudio"; + static constexpr const char* kStreamingToken = "generateStreamingToken"; + static constexpr const char* kChat = "createChatCompletion"; + static constexpr const char* kGatewayModels = "listAvailableModels"; + static constexpr const char* kUnderstanding = "createSpeechUnderstanding"; +}; + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/client.cpp b/src/vendors/audio/assemblyai/client.cpp new file mode 100644 index 0000000..25cb5de --- /dev/null +++ b/src/vendors/audio/assemblyai/client.cpp @@ -0,0 +1,51 @@ +#include "vendors/audio/assemblyai/client.h" + +#include "llm/config.h" + +namespace molo::audio::assemblyai { + +std::string AssemblyEndpoints::base_url() { + return allowlisted_base(kBaseEnv, kDefaultBase, {kDefaultBase, kEuBase}); +} + +bool AssemblyEndpoints::eu() { return base_url() == kEuBase; } + +std::string AssemblyEndpoints::streaming_host_url() { + return eu() ? "https://streaming.eu.assemblyai.com" : "https://streaming.assemblyai.com"; +} + +llm::ApiClientConfig AssemblyEndpoints::config(const std::string& base) { + llm::ApiClientConfig c; + c.vendor = kVendor; + c.base_url = base; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "Authorization"; + return c; +} + +std::shared_ptr AssemblyEndpoints::from_env() { + auto hosts = std::make_shared(); + hosts->add("api", HostFactory::from_env(config(base_url()))); + hosts->add("streaming", HostFactory::from_env(config(streaming_host_url()))); + hosts->add("gateway", HostFactory::from_env(config(kGatewayBase))); + return hosts; +} + +std::shared_ptr AssemblyEndpoints::with(AudioHost api, AudioHost streaming, AudioHost gateway) { + auto hosts = std::make_shared(); + hosts->add("api", std::move(api)); + hosts->add("streaming", std::move(streaming)); + hosts->add("gateway", std::move(gateway)); + return hosts; +} + +std::string AssemblyEndpoints::key() { return llm::Config::get(kKeyEnv); } + +AudioSocket::Headers AssemblyEndpoints::ws_headers(const std::string& key) { + return {{"Authorization", key}}; +} + +std::string AssemblyEndpoints::ws_url() { return to_ws_url(streaming_host_url()) + "/v3/ws"; } + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/client.h b/src/vendors/audio/assemblyai/client.h new file mode 100644 index 0000000..e6fc5d2 --- /dev/null +++ b/src/vendors/audio/assemblyai/client.h @@ -0,0 +1,38 @@ +#pragma once + +// AssemblyAI endpoints and credentials. The key is the whole Authorization +// header value (no Bearer prefix) on every host. +// api https://api.assemblyai.com (EU: https://api.eu.assemblyai.com) +// streaming https://streaming.assemblyai.com token endpoint; WebSocket wss://streaming.assemblyai.com/v3/ws +// gateway https://llm-gateway.assemblyai.com/v1 +// ASSEMBLYAI_BASE_URL selects the EU API host; the streaming host follows it. + +#include +#include + +#include "vendors/audio/hosts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::assemblyai { + +struct AssemblyEndpoints { + static constexpr const char* kVendor = "assemblyai"; + static constexpr const char* kKeyEnv = "ASSEMBLYAI_API_KEY"; + static constexpr const char* kBaseEnv = "ASSEMBLYAI_BASE_URL"; + static constexpr const char* kDefaultBase = "https://api.assemblyai.com"; + static constexpr const char* kEuBase = "https://api.eu.assemblyai.com"; + static constexpr const char* kGatewayBase = "https://llm-gateway.assemblyai.com/v1"; + + static std::string base_url(); + static bool eu(); + static std::string streaming_host_url(); // https://streaming[.eu].assemblyai.com + static llm::ApiClientConfig config(const std::string& base); + static std::shared_ptr from_env(); + // Injected hosts (tests): "api", "streaming", "gateway". + static std::shared_ptr with(AudioHost api, AudioHost streaming, AudioHost gateway); + static std::string key(); + static AudioSocket::Headers ws_headers(const std::string& key); + static std::string ws_url(); // wss://streaming[.eu].assemblyai.com/v3/ws +}; + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/gateway.cpp b/src/vendors/audio/assemblyai/gateway.cpp new file mode 100644 index 0000000..60a1f76 --- /dev/null +++ b/src/vendors/audio/assemblyai/gateway.cpp @@ -0,0 +1,63 @@ +#include "vendors/audio/assemblyai/gateway.h" + +#include "llm/caps.h" +#include "vendors/audio/caps.h" + +namespace molo::audio::assemblyai { + +using C = AssemblyCalls; + +Json Gateway::build(const GatewayChat& c) const { + require_text("model", c.model, 128); + bool has_messages = !c.messages.empty(); + bool has_prompt = !c.prompt.empty(); + if (has_messages == has_prompt) bad("give exactly one of messages and prompt"); + if (c.messages.size() > llm::Caps::kMaxMessages) bad("too many messages"); + Json o = Json::object(); + o.set("model", Json(c.model)); + if (has_messages) { + Json arr = Json::array(); + for (const auto& m : c.messages) { + if (m.role != "system" && m.role != "user" && m.role != "assistant" && m.role != "tool") + bad("message role must be system, user, assistant or tool"); + require_text("message content", m.content, AudioCaps::kMaxTextChars, false); + Json j = Json::object(); + j.set("role", Json(m.role)); + j.set("content", Json(m.content)); + arr.push(j); + } + o.set("messages", arr); + } else { + require_text("prompt", c.prompt, AudioCaps::kMaxTextChars); + o.set("prompt", Json(c.prompt)); + } + if (!c.transcript_id.empty()) { + check_id("transcript", c.transcript_id); + o.set("transcript_id", Json(c.transcript_id)); + } + if (c.max_tokens < 0 || c.max_tokens > 1000000) bad("max_tokens is out of range"); + if (c.max_tokens > 0) o.set("max_tokens", Json(c.max_tokens)); + if (c.temperature >= 0) { + require_range("temperature", c.temperature, 0, 2); + o.set("temperature", Json(c.temperature)); + } + return o; +} + +Json Gateway::chat(const GatewayChat& c) { + Json body = build(c); + return call(C::kChat, {}, &body); +} + +Json Gateway::models() { return call(C::kGatewayModels); } + +Json Gateway::understanding(const std::string& transcript_id, const Json& su) { + check_id("transcript", transcript_id); + if (!su.is_object()) bad("speech_understanding must be an object"); + Json body = Json::object(); + body.set("transcript_id", Json(transcript_id)); + body.set("speech_understanding", su); + return call(C::kUnderstanding, {}, &body); +} + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/gateway.h b/src/vendors/audio/assemblyai/gateway.h new file mode 100644 index 0000000..7a24456 --- /dev/null +++ b/src/vendors/audio/assemblyai/gateway.h @@ -0,0 +1,44 @@ +#pragma once + +// AssemblyAI LLM Gateway (https://llm-gateway.assemblyai.com/v1): chat +// completions over text or a transcript id, speech understanding, model list. +// LeMUR, the earlier transcript LLM API, no longer appears in the published +// OpenAPI document or the docs index and answers 404 (see SPEC); the gateway +// is its replacement and the only LLM surface wrapped here. + +#include +#include + +#include "vendors/audio/assemblyai/calls.h" +#include "vendors/audio/assemblyai/client.h" +#include "vendors/audio/audio_service.h" + +namespace molo::audio::assemblyai { + +struct GatewayMessage { + std::string role; // system | user | assistant | tool + std::string content; +}; + +struct GatewayChat { + std::string model; // required, for example a model id from models() + std::vector messages; // or prompt + transcript_id + std::string prompt; + std::string transcript_id; + int max_tokens = 0; // 0 unset + double temperature = -1; // 0..2, -1 unset +}; + +class Gateway : public AudioService { + public: + explicit Gateway(std::shared_ptr hosts) : AudioService(calls(), std::move(hosts)) {} + Gateway() : Gateway(AssemblyEndpoints::from_env()) {} + + Json build(const GatewayChat& chat) const; + Json chat(const GatewayChat& chat); + Json models(); + // Applies speech understanding (translation, speaker identification, custom formatting) to a finished transcript. + Json understanding(const std::string& transcript_id, const Json& speech_understanding); +}; + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/speech.cpp b/src/vendors/audio/assemblyai/speech.cpp new file mode 100644 index 0000000..4b3a8fa --- /dev/null +++ b/src/vendors/audio/assemblyai/speech.cpp @@ -0,0 +1,53 @@ +#include "vendors/audio/assemblyai/speech.h" + +namespace molo::audio::assemblyai { + +TranscriptRequest AssemblyRecognizer::map(const TranscribeRequest& r, const std::string& audio_url) const { + TranscriptRequest t; + t.audio_url = audio_url; + if (!r.model.empty()) t.speech_models.push_back(r.model); + if (r.language.empty()) t.language_detection = 1; + else t.language_code = r.language; + t.punctuate = 1; + t.format_text = 1; + if (r.diarize) t.speaker_labels = 1; + t.keyterms_prompt = r.keywords; + return t; +} + +Transcript AssemblyRecognizer::fold(const Json& doc) { + Transcript t; + t.provider = "assemblyai"; + t.raw = doc; + if (!doc.is_object()) return t; + t.text = llm::Service::string_at(doc, "text"); + t.language = llm::Service::string_at(doc, "language_code"); + if (const Json* d = doc.find("audio_duration"); d && d->is_number()) t.duration = d->as_number(); + if (const Json* words = doc.find("words"); words && words->is_array()) { + for (const Json& w : words->as_array()) { + if (!w.is_object()) continue; + TranscriptWord word; + word.text = llm::Service::string_at(w, "text"); + // AssemblyAI times are milliseconds. + if (const Json* x = w.find("start"); x && x->is_number()) word.start = x->as_number() / 1000.0; + if (const Json* x = w.find("end"); x && x->is_number()) word.end = x->as_number() / 1000.0; + if (const Json* x = w.find("confidence"); x && x->is_number()) word.confidence = x->as_number(); + word.speaker = llm::Service::string_at(w, "speaker"); + t.words.push_back(word); + } + } + return t; +} + +Transcript AssemblyRecognizer::transcribe(const TranscribeRequest& r) { + validate(r); + std::string url = r.url; + if (url.empty()) url = transcripts_.upload(r.audio); + Json submitted = transcripts_.submit(map(r, url)); + std::string id = llm::Service::string_at(submitted, "id"); + if (id.empty()) throw llm::ApiError("assemblyai", 502, "BAD_RESPONSE", "submit returned no id"); + PollPolicy policy = PollPolicy::clamped(3000, r.max_wait_seconds); + return fold(transcripts_.wait(id, policy, sleep_ms_)); +} + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/speech.h b/src/vendors/audio/assemblyai/speech.h new file mode 100644 index 0000000..99d447e --- /dev/null +++ b/src/vendors/audio/assemblyai/speech.h @@ -0,0 +1,31 @@ +#pragma once + +// AssemblyAI behind the neutral recognizer seam. AssemblyAI has no speech +// synthesis product, so there is no synthesizer. + +#include +#include + +#include "vendors/audio/assemblyai/transcripts.h" +#include "vendors/audio/speech.h" + +namespace molo::audio::assemblyai { + +class AssemblyRecognizer : public SpeechRecognizer { + public: + explicit AssemblyRecognizer(std::shared_ptr hosts = AssemblyEndpoints::from_env(), + std::function sleep_ms = nullptr) + : transcripts_(std::move(hosts)), sleep_ms_(std::move(sleep_ms)) {} + std::string name() const override { return "assemblyai"; } + bool configured() const override { return transcripts_.configured(); } + // Uploads local audio, submits, polls to completion within max_wait_seconds. + Transcript transcribe(const TranscribeRequest& request) override; + TranscriptRequest map(const TranscribeRequest& request, const std::string& audio_url) const; + static Transcript fold(const Json& doc); + + private: + Transcripts transcripts_; + std::function sleep_ms_; +}; + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/stream.cpp b/src/vendors/audio/assemblyai/stream.cpp new file mode 100644 index 0000000..f749125 --- /dev/null +++ b/src/vendors/audio/assemblyai/stream.cpp @@ -0,0 +1,131 @@ +#include "vendors/audio/assemblyai/stream.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::assemblyai { + +namespace { + +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("assemblyai", why); } + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} + +Json typed(const char* type) { + Json m = Json::object(); + m.set("type", Json(type)); + return m; +} + +} // namespace + +const std::vector& TranscriptStream::query_params() { + // streaming.yaml, channel parameters. + static const std::vector v = { + "token", "speech_model", "language_codes", "language_detection", "domain", "mode", + "session_heartbeat", "encoding", "sample_rate", "min_turn_silence", "max_turn_silence", + "end_of_turn_confidence_threshold", "vad_threshold", "interruption_delay", "include_partial_turns", + "format_turns", "prompt", "keyterms_prompt", "agent_context", "previous_context_n_turns", + "speaker_labels", "max_speakers", "speaker_labels_revision_interval_ms", "voice_focus", + "voice_focus_threshold", "redact_pii", "redact_pii_policies", "redact_pii_sub", "filter_profanity", + "llm_gateway", "inactivity_timeout"}; + return v; +} + +Vocabulary TranscriptStream::vocabulary() { + return Vocabulary("type", {"UpdateConfiguration", "ForceEndpoint", "Terminate", "KeepAlive"}, + {"Begin", "SpeechStarted", "Turn", "SpeakerRevision", "Termination", "Heartbeat", + "LLMGatewayResponse"}); +} + +TranscriptStream::TranscriptStream(std::string api_key, StreamParams params, std::string url, SocketOptions socket) { + Query q; + for (const auto& [k, v] : params.pairs) { + bool ok = false; + for (const auto& a : query_params()) ok = ok || a == k; + if (!ok || k == "token") invalid("unknown stream parameter '" + k + "'"); + if (v.size() > 4096) invalid("stream parameter value is too long"); + q.add(k, v); + } + AudioSocket::Headers headers; + std::string base = url.empty() ? AssemblyEndpoints::ws_url() : url; + public_url_ = base + q.str(); + if (!params.token.empty()) { + q.add("token", params.token); // temporary token travels in the URL by the provider's design + } else { + headers = AssemblyEndpoints::ws_headers(api_key); + } + socket_ = std::make_unique("assemblyai", base + q.str(), headers, vocabulary(), std::move(socket)); +} + +void TranscriptStream::update_configuration(const Json& fields) { + if (!fields.is_object() || fields.as_object().empty()) invalid("fields must be a non-empty object"); + Json m = fields; + m.set("type", Json("UpdateConfiguration")); + socket_->send_json(m); +} +void TranscriptStream::force_endpoint() { socket_->send_json(typed("ForceEndpoint")); } +void TranscriptStream::keep_alive() { socket_->send_json(typed("KeepAlive")); } +void TranscriptStream::terminate() { socket_->send_json(typed("Terminate")); } + +std::optional TranscriptStream::as_turn(const SocketEvent& ev) { + if (ev.kind != SocketEvent::Kind::kText || !ev.json_ok || ev.type != "Turn") return std::nullopt; + StreamTurn t; + t.raw = ev.json; + t.transcript = str_at(ev.json, "transcript"); + if (const Json* x = ev.json.find("end_of_turn"); x && x->is_bool()) t.end_of_turn = x->as_bool(); + if (const Json* x = ev.json.find("turn_is_formatted"); x && x->is_bool()) t.formatted = x->as_bool(); + if (const Json* x = ev.json.find("turn_order"); x && x->is_number()) t.turn_order = x->as_int(); + if (const Json* x = ev.json.find("end_of_turn_confidence"); x && x->is_number()) t.end_of_turn_confidence = x->as_number(); + t.speaker_label = str_at(ev.json, "speaker_label"); + return t; +} + +StreamResult TranscriptStream::collect(int max_wait_ms) { + StreamResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + ++out.events; + if (ev->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + if (!ev->json_ok) continue; + if (ev->type == "Begin") { + out.session_id = str_at(ev->json, "id"); + } else if (ev->type == "SpeechStarted") { + ++out.speech_started; + } else if (ev->type == "Turn") { + auto t = as_turn(*ev); + if (t && t->end_of_turn) { + // With format_turns the server repeats the final turn once formatted: keep the formatted one. + if (!out.final_turns.empty() && t->formatted) { + std::string& last = out.final_turns.back(); + std::size_t at = out.transcript.rfind(last); + last = t->transcript; + if (at != std::string::npos) out.transcript = out.transcript.substr(0, at) + last; + } else if (!t->transcript.empty()) { + out.final_turns.push_back(t->transcript); + out.transcript += (out.transcript.empty() ? "" : " ") + t->transcript; + } + out.partial.clear(); + } else if (t) { + out.partial = t->transcript; + } + } else if (ev->type == "Termination") { + out.terminated = true; + if (const Json* d = ev->json.find("audio_duration_seconds"); d && d->is_number()) out.audio_duration_seconds = d->as_number(); + return out; + } else if (ev->json.find("error")) { + out.error = str_at(ev->json, "error"); + if (out.error.empty()) out.error = "stream error"; + return out; + } + } + return out; +} + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/stream.h b/src/vendors/audio/assemblyai/stream.h new file mode 100644 index 0000000..13152d4 --- /dev/null +++ b/src/vendors/audio/assemblyai/stream.h @@ -0,0 +1,81 @@ +#pragma once + +// AssemblyAI streaming speech to text (v3): wss://streaming.assemblyai.com/v3/ws. +// Messages follow https://www.assemblyai.com/docs/api-reference/specs/streaming.yaml: +// client sends binary audio plus UpdateConfiguration / ForceEndpoint / Terminate / +// KeepAlive, server sends Begin / SpeechStarted / Turn / SpeakerRevision / +// Termination / Heartbeat / LLMGatewayResponse. + +#include +#include +#include +#include + +#include "vendors/audio/assemblyai/client.h" +#include "vendors/audio/audio_service.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::assemblyai { + +struct StreamParams { + // name -> value; names are checked against the channel's query parameters. + std::vector> pairs; + std::string token; // temporary token instead of the Authorization header (browser style) + StreamParams& add(const std::string& name, const std::string& value) { + pairs.emplace_back(name, value); + return *this; + } +}; + +struct StreamTurn { + std::string transcript; + bool end_of_turn = false; + bool formatted = false; + long turn_order = -1; + double end_of_turn_confidence = 0; + std::string speaker_label; + Json raw; +}; + +struct StreamResult { + std::string session_id; + std::vector final_turns; // end_of_turn transcripts (formatted ones when format_turns is on) + std::string transcript; // joined by a space + std::string partial; + double audio_duration_seconds = -1; // from Termination + std::size_t speech_started = 0; + std::string error; + bool terminated = false; + bool closed = false; + std::size_t events = 0; +}; + +class TranscriptStream { + public: + TranscriptStream(std::string api_key, StreamParams params, std::string url = "", SocketOptions socket = {}); + + // The URL without a temporary token (safe to log). + std::string url() const { return public_url_; } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void send_audio(const std::string& audio) { socket_->send_binary(audio); } + void update_configuration(const Json& fields); // {"type":"UpdateConfiguration",...} + void force_endpoint(); // {"type":"ForceEndpoint"} + void keep_alive(); // {"type":"KeepAlive"} + void terminate(); // {"type":"Terminate"} + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + static std::optional as_turn(const SocketEvent& ev); + // Reads until Termination, error, close, or silence of `max_wait_ms`. + StreamResult collect(int max_wait_ms); + + static const std::vector& query_params(); + static Vocabulary vocabulary(); + + private: + std::string public_url_; + std::unique_ptr socket_; +}; + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/transcripts.cpp b/src/vendors/audio/assemblyai/transcripts.cpp new file mode 100644 index 0000000..a78f58a --- /dev/null +++ b/src/vendors/audio/assemblyai/transcripts.cpp @@ -0,0 +1,183 @@ +#include "vendors/audio/assemblyai/transcripts.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::assemblyai { + +using C = AssemblyCalls; + +namespace { +void tri(Json& o, const char* name, int v) { + if (v >= 0) o.set(name, Json(v == 1)); +} +} // namespace + +std::string Transcripts::upload(const std::string& audio) { + if (audio.size() > AssemblyCaps::kMaxUploadBytes) bad("audio is over the byte cap"); + llm::RawResponse raw = call_bytes(C::kUpload, {}, audio, "application/octet-stream"); + Json doc; + try { + doc = Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError("assemblyai", 502, "BAD_RESPONSE", e.what()); + } + const Json* url = doc.is_object() ? doc.find("upload_url") : nullptr; + if (!url || !url->is_string()) throw llm::ApiError("assemblyai", 502, "BAD_RESPONSE", "upload_url missing"); + return url->as_string(); +} + +Json Transcripts::build(const TranscriptRequest& r) const { + if (r.audio_url.rfind("https://", 0) != 0 && r.audio_url.rfind("http://", 0) != 0) + bad("audio_url must be http or https (use upload() for local audio)"); + require_text("audio_url", r.audio_url, AudioCaps::kMaxFieldChars); + Json o = Json::object(); + o.set("audio_url", Json(r.audio_url)); + if (!r.speech_models.empty()) { + if (r.speech_models.size() > 8) bad("at most 8 speech_models"); + Json arr = Json::array(); + for (const auto& m : r.speech_models) { + require_text("speech model", m, 64); + arr.push(Json(m)); + } + o.set("speech_models", arr); + } + if (!r.language_code.empty()) { + require_text("language_code", r.language_code, 16); + o.set("language_code", Json(r.language_code)); + } + tri(o, "language_detection", r.language_detection); + tri(o, "punctuate", r.punctuate); + tri(o, "format_text", r.format_text); + tri(o, "disfluencies", r.disfluencies); + tri(o, "multichannel", r.multichannel); + tri(o, "speaker_labels", r.speaker_labels); + if (r.speakers_expected < 0 || r.speakers_expected > 100) bad("speakers_expected must be 0 or 1..100"); + if (r.speakers_expected > 0) o.set("speakers_expected", Json(r.speakers_expected)); + if (!r.keyterms_prompt.empty()) { + if (r.keyterms_prompt.size() > AssemblyCaps::kMaxKeyterms) bad("too many keyterms_prompt entries"); + Json arr = Json::array(); + for (const auto& k : r.keyterms_prompt) { + require_text("keyterm", k, 100); + arr.push(Json(k)); + } + o.set("keyterms_prompt", arr); + } + if (!r.prompt.empty()) { + require_text("prompt", r.prompt, 10000); + o.set("prompt", Json(r.prompt)); + } + tri(o, "auto_chapters", r.auto_chapters); + tri(o, "sentiment_analysis", r.sentiment_analysis); + tri(o, "entity_detection", r.entity_detection); + tri(o, "summarization", r.summarization); + if (!r.webhook_url.empty()) { + if (r.webhook_url.rfind("https://", 0) != 0) bad("webhook_url must be https"); + require_text("webhook_url", r.webhook_url, AudioCaps::kMaxFieldChars); + o.set("webhook_url", Json(r.webhook_url)); + } + if (r.extra.is_object()) + for (const auto& [k, v] : r.extra.as_object()) { + if (o.find(k)) bad("extra repeats the field '" + k + "'"); + o.set(k, v); + } + check_body(table_.get(C::kSubmit), &o); + return o; +} + +Json Transcripts::submit(const TranscriptRequest& r) { + Json body = build(r); + return call(C::kSubmit, {}, &body); +} + +Json Transcripts::get(const std::string& id) { + check_id("transcript", id); + return call(C::kGet, {id}); +} + +Json Transcripts::list(const Json& q) { + if (q.is_object()) { + const Json* l = q.find("limit"); + if (l && l->is_number() && (l->as_int() < 1 || l->as_int() > AssemblyCaps::kMaxListLimit)) + bad("limit must be 1..200"); + } + return call(C::kList, {}, nullptr, query_from(C::kList, q)); +} + +Json Transcripts::remove(const std::string& id, bool confirm) { + check_id("transcript", id); + return call_destructive(C::kDelete, {id}, nullptr, Query(), confirm); +} + +Json Transcripts::sentences(const std::string& id) { + check_id("transcript", id); + return call(C::kSentences, {id}); +} + +Json Transcripts::paragraphs(const std::string& id) { + check_id("transcript", id); + return call(C::kParagraphs, {id}); +} + +Json Transcripts::word_search(const std::string& id, const std::vector& words) { + check_id("transcript", id); + if (words.empty() || words.size() > AssemblyCaps::kMaxSearchWords) bad("words must hold 1..50 entries"); + std::string joined; + for (const auto& w : words) { + require_text("word", w, 100); + if (w.find(',') != std::string::npos) bad("words must not contain commas"); + joined += (joined.empty() ? "" : ",") + w; + } + Query q; + q.add("words", joined); + return call(C::kWordSearch, {id}, nullptr, q); +} + +std::string Transcripts::subtitles(const std::string& id, const std::string& format, int chars_per_caption) { + check_id("transcript", id); + if (format != "srt" && format != "vtt") bad("format must be srt or vtt"); + Query q; + if (chars_per_caption != 0) { + if (chars_per_caption < 1 || chars_per_caption > 1000) bad("chars_per_caption must be 0 or 1..1000"); + q.add_int("chars_per_caption", chars_per_caption); + } + return call_raw(C::kSubtitles, {id, format}, nullptr, q).body; +} + +Json Transcripts::redacted_audio(const std::string& id) { + check_id("transcript", id); + return call(C::kRedactedAudio, {id}); +} + +Json Transcripts::wait(const std::string& id, const PollPolicy& policy, const std::function& sleep_ms) { + check_id("transcript", id); + Json last; + std::string status; + PollOutcome out = poll( + policy, + [&](int) { + last = get(id); + status = Service::string_at(last, "status"); + return status == "completed" || status == "error"; + }, + sleep_ms); + if (!out.done) + throw llm::ApiError("assemblyai", 408, "poll_timeout", + "transcript " + id + " is still " + (status.empty() ? "pending" : status) + " after " + + std::to_string(out.waited_ms / 1000) + " s"); + if (status == "error") + throw llm::ApiError("assemblyai", 422, "transcript_error", Service::string_at(last, "error", "transcription failed")); + return last; +} + +Json Transcripts::streaming_token(int expires, int max_session) { + if (expires < 1 || expires > 600) bad("expires_in_seconds must be 1..600"); + Query q; + q.add_int("expires_in_seconds", expires); + if (max_session != 0) { + if (max_session < 60 || max_session > 10800) bad("max_session_duration_seconds must be 60..10800"); + q.add_int("max_session_duration_seconds", max_session); + } + return call(C::kStreamingToken, {}, nullptr, q); +} + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/assemblyai/transcripts.h b/src/vendors/audio/assemblyai/transcripts.h new file mode 100644 index 0000000..326188a --- /dev/null +++ b/src/vendors/audio/assemblyai/transcripts.h @@ -0,0 +1,71 @@ +#pragma once + +// Pre-recorded transcription: upload, submit, poll, read, list, delete, and +// the derived reads (sentences, paragraphs, word search, subtitles, redacted +// audio). + +#include +#include +#include +#include + +#include "vendors/audio/assemblyai/calls.h" +#include "vendors/audio/assemblyai/client.h" +#include "vendors/audio/audio_service.h" +#include "vendors/audio/speech.h" + +namespace molo::audio::assemblyai { + +struct AssemblyCaps { + static constexpr std::size_t kMaxUploadBytes = 128u * 1024u * 1024u; // our ceiling (provider allows 5 GB) + static constexpr std::size_t kMaxWordBoost = 1000; + static constexpr std::size_t kMaxKeyterms = 1000; // keyterms_prompt limit in the docs is 1000 for pre-recorded + static constexpr int kMaxListLimit = 200; + static constexpr std::size_t kMaxSearchWords = 50; +}; + +struct TranscriptRequest { + std::string audio_url; // required; upload() returns one + std::vector speech_models; // e.g. universal-3-pro, universal-2 (priority order) + std::string language_code; // "" with language_detection + int language_detection = -1; + int punctuate = -1, format_text = -1, disfluencies = -1, multichannel = -1; + int speaker_labels = -1; + int speakers_expected = 0; // 0 unset, else 1..100 + std::vector keyterms_prompt; // <= kMaxKeyterms + std::string prompt; + int auto_chapters = -1, sentiment_analysis = -1, entity_detection = -1, summarization = -1; + std::string webhook_url; + Json extra; // any other createTranscript field: object, validated against the table +}; + +class Transcripts : public AudioService { + public: + explicit Transcripts(std::shared_ptr hosts) + : AudioService(calls(), std::move(hosts)) {} + Transcripts() : Transcripts(AssemblyEndpoints::from_env()) {} + + // Uploads bytes; returns the upload_url to use as audio_url. + std::string upload(const std::string& audio); + Json build(const TranscriptRequest& request) const; + Json submit(const TranscriptRequest& request); + Json get(const std::string& transcript_id); + Json list(const Json& query_object = Json()); + Json remove(const std::string& transcript_id, bool confirm); + Json sentences(const std::string& transcript_id); + Json paragraphs(const std::string& transcript_id); + Json word_search(const std::string& transcript_id, const std::vector& words); + std::string subtitles(const std::string& transcript_id, const std::string& format, int chars_per_caption = 0); + Json redacted_audio(const std::string& transcript_id); + + // Polls get() until status is completed or error, within the policy. + // `sleep_ms` null = real sleep. Returns the final document; throws + // llm::ApiError (408, "poll_timeout") when the cap is reached first and the + // transcript is still queued or processing, and (422) when status is error. + Json wait(const std::string& transcript_id, const PollPolicy& policy, + const std::function& sleep_ms = nullptr); + // Streaming session token (temporary, for browsers). + Json streaming_token(int expires_in_seconds, int max_session_duration_seconds = 0); +}; + +} // namespace molo::audio::assemblyai diff --git a/src/vendors/audio/audio_service.cpp b/src/vendors/audio/audio_service.cpp new file mode 100644 index 0000000..d077669 --- /dev/null +++ b/src/vendors/audio/audio_service.cpp @@ -0,0 +1,211 @@ +#include "vendors/audio/audio_service.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio { + +AudioService::AudioService(const CallTable& table, std::shared_ptr hosts) + : llm::Service(hosts->first().api), + table_(table), + hosts_(std::move(hosts)), + vendor_(table.provider()) {} + +void AudioService::bad(const std::string& why) const { throw llm::ApiError::invalid(vendor_, why); } + +const AudioHost& AudioService::host_of(const CallSpec& spec) const { + return hosts_->get(spec.host, vendor_); +} + +bool AudioService::configured_for(const std::string& id) const { + return host_of(table_.get(id)).configured(); +} + +void AudioService::require_text(const std::string& what, const std::string& s, std::size_t max_chars, + bool required) const { + if (required && s.empty()) bad(what + " is required"); + if (s.size() > max_chars) bad(what + " is over " + std::to_string(max_chars) + " characters"); + for (unsigned char c : s) + if (c == 0) bad(what + " has a NUL byte"); +} + +void AudioService::require_range(const std::string& what, double v, double lo, double hi) const { + if (!(v >= lo && v <= hi)) bad(what + " must be in " + std::to_string(lo) + ".." + std::to_string(hi)); +} + +void AudioService::require_confirm(const std::string& what, bool confirm) const { + if (!confirm) bad(what + " is destructive or costs money: pass confirm=true"); +} + +void AudioService::check_body(const CallSpec& spec, const Json* body) const { + std::string kind = spec.body; + if (kind != "json") { + if (body) bad(std::string(spec.id) + " takes no JSON body"); + return; + } + auto required = csv_list(spec.required); + if (!body) { + if (!required.empty()) bad(std::string(spec.id) + " needs a JSON body"); + return; + } + if (!body->is_object()) bad("body must be a JSON object"); + if (body->dump().size() > AudioCaps::kMaxJsonBodyBytes) bad("body is over the byte cap"); + std::string fields = spec.fields; + if (fields != "*") { + for (const auto& [k, v] : body->as_object()) { + (void)v; + if (!csv_has(spec.fields, k)) bad("unknown body field '" + k + "' for " + spec.id); + } + } + for (const auto& r : required) { + const Json* v = body->find(r); + if (!v || v->is_null()) bad("missing required field '" + r + "' for " + spec.id); + } +} + +void AudioService::check_form(const CallSpec& spec, const clients::body::MultipartBody& form) const { + if (std::string(spec.body) != "form") bad(std::string(spec.id) + " is not a multipart call"); + if (form.encoded_size() > AudioCaps::kMaxAudioInBytes) bad("upload is over the byte cap"); + std::string fields = spec.fields; + for (const auto& part : form.parts()) { + if (fields != "*" && !csv_has(spec.fields, part.name)) + bad("unknown form field '" + part.name + "' for " + spec.id); + } + for (const auto& r : csv_list(spec.required)) { + bool found = false; + for (const auto& part : form.parts()) found = found || part.name == r; + if (!found) bad("missing required form field '" + r + "' for " + spec.id); + } +} + +void AudioService::check_query(const CallSpec& spec, const Query& query) const { + for (const auto& [k, v] : query.pairs()) { + (void)v; + if (!csv_has(spec.query, k)) bad("unknown query parameter '" + k + "' for " + spec.id); + } +} + +Query AudioService::query_from(const std::string& id, const Json& object) const { + const CallSpec& spec = table_.get(id); + Query q; + if (object.is_null()) return q; + if (!object.is_object()) bad("query must be a JSON object"); + for (const auto& [k, v] : object.as_object()) { + if (!csv_has(spec.query, k)) bad("unknown query parameter '" + k + "' for " + id); + if (!q.add_json(k, v)) bad("query parameter '" + k + "' must be a scalar or a list of scalars"); + } + return q; +} + +std::string AudioService::path_of(const CallSpec& spec, std::initializer_list segments, + const Query& query) const { + check_query(spec, query); + std::string path = render_path(vendor_, spec, segments); + try { + return path + query.str(); + } catch (const std::length_error&) { + bad("query string is over the cap"); + } +} + +std::string AudioService::preview(const std::string& id, std::initializer_list segments, + const Json* body, const Query& query) const { + const CallSpec& spec = table_.get(id); + check_body(spec, body); + return std::string(spec.method) + " " + path_of(spec, segments, query); +} + +Json AudioService::call(const std::string& id, std::initializer_list segments, + const Json* body, const Query& query) { + const CallSpec& spec = table_.get(id); + if (is_destructive(spec)) bad(std::string(spec.id) + " is destructive: use call_destructive"); + std::string kind = spec.response; + if (kind == "bytes" || kind == "text") bad(std::string(spec.id) + " returns " + kind + ": use call_raw"); + if (std::string(spec.body) == "form" || std::string(spec.body) == "bytes") + bad(std::string(spec.id) + " needs a " + spec.body + " body: use call_form / call_bytes"); + check_body(spec, body); + std::string path = path_of(spec, segments, query); + return host_of(spec).api->request(method_of(spec), path, body); +} + +llm::RawResponse AudioService::call_raw(const std::string& id, + std::initializer_list segments, + const Json* body, const Query& query) { + const CallSpec& spec = table_.get(id); + if (is_destructive(spec)) bad(std::string(spec.id) + " is destructive: use call_destructive"); + if (std::string(spec.body) == "form" || std::string(spec.body) == "bytes") + bad(std::string(spec.id) + " needs a " + spec.body + " body"); + check_body(spec, body); + std::string path = path_of(spec, segments, query); + std::string payload = body ? body->dump() : std::string(); + return host_of(spec).api->request_raw(method_of(spec), path, body ? &payload : nullptr, + "application/json"); +} + +llm::RawResponse AudioService::call_bytes(const std::string& id, + std::initializer_list segments, + const std::string& data, const std::string& content_type, + const Query& query) { + const CallSpec& spec = table_.get(id); + if (std::string(spec.body) != "bytes") bad(std::string(spec.id) + " does not take raw bytes"); + if (data.empty()) bad("audio data is required"); + if (data.size() > AudioCaps::kMaxAudioInBytes) bad("audio is over the byte cap"); + if (content_type.empty() || content_type.size() > 128) bad("content type is required"); + std::string path = path_of(spec, segments, query); + return host_of(spec).api->request_raw(method_of(spec), path, &data, content_type); +} + +Json AudioService::call_form(const std::string& id, std::initializer_list segments, + const clients::body::MultipartBody& form, const Query& query) { + const CallSpec& spec = table_.get(id); + if (is_destructive(spec)) bad(std::string(spec.id) + " is destructive: use call_destructive"); + check_form(spec, form); + std::string path = path_of(spec, segments, query); + if (method_of(spec) == clients::Method::kPost) return host_of(spec).api->post_multipart(path, form); + // PUT / PATCH multipart: the shared client only has POST, so send the encoded body. + std::string payload = form.encode(); + llm::RawResponse raw = host_of(spec).api->request_raw(method_of(spec), path, &payload, form.content_type()); + if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object(); + try { + return Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError(vendor_, 502, "BAD_RESPONSE", e.what()); + } +} + +llm::RawResponse AudioService::call_form_raw(const std::string& id, + std::initializer_list segments, + const clients::body::MultipartBody& form, + const Query& query) { + const CallSpec& spec = table_.get(id); + if (is_destructive(spec)) bad(std::string(spec.id) + " is destructive: use call_destructive"); + check_form(spec, form); + std::string path = path_of(spec, segments, query); + std::string payload = form.encode(); + return host_of(spec).api->request_raw(method_of(spec), path, &payload, form.content_type()); +} + +ByteStreamer::Result AudioService::call_stream(const std::string& id, + std::initializer_list segments, + const Json* body, const Query& query, + const ChunkSink& sink) { + const CallSpec& spec = table_.get(id); + if (std::string(spec.response) != "bytes") bad(std::string(spec.id) + " does not stream bytes"); + if (is_destructive(spec)) bad(std::string(spec.id) + " is destructive: use call_destructive"); + check_body(spec, body); + std::string path = path_of(spec, segments, query); + const AudioHost& host = host_of(spec); + return host.streamer->stream(method_of(spec), path, body, sink); +} + +Json AudioService::call_destructive(const std::string& id, + std::initializer_list segments, const Json* body, + const Query& query, bool confirm) { + const CallSpec& spec = table_.get(id); + if (!is_destructive(spec)) bad(std::string(spec.id) + " is not a destructive call"); + check_body(spec, body); + std::string path = path_of(spec, segments, query); + require_confirm(spec.id, confirm); + return host_of(spec).api->request(method_of(spec), path, body); +} + +} // namespace molo::audio diff --git a/src/vendors/audio/audio_service.h b/src/vendors/audio/audio_service.h new file mode 100644 index 0000000..11851aa --- /dev/null +++ b/src/vendors/audio/audio_service.h @@ -0,0 +1,86 @@ +#pragma once + +// Base of every provider service in this package. Table driven: a call is +// named by its id in the provider's CallTable, never by a literal path. The +// base validates path segments, body fields, required fields, query +// parameters and the destructive guard before anything reaches the network. + +#include +#include +#include + +#include "clients/body/form.h" +#include "llm/api_client.h" +#include "vendors/audio/call_spec.h" +#include "vendors/audio/hosts.h" +#include "vendors/audio/query.h" + +namespace molo::audio { + +class AudioService : public llm::Service { + public: + AudioService(const CallTable& table, std::shared_ptr hosts); + + const CallTable& table() const { return table_; } + const std::string& vendor() const { return vendor_; } + const Hosts& hosts() const { return *hosts_; } + // True when the host the call lives on has a key. + bool configured_for(const std::string& id) const; + + // JSON in, JSON out. Rows whose response is bytes or text, and destructive + // rows, are refused here: use call_raw / call_destructive. + Json call(const std::string& id, std::initializer_list segments = {}, + const Json* body = nullptr, const Query& query = Query()); + // Any row. Returns the buffered response (audio, text, files). + llm::RawResponse call_raw(const std::string& id, std::initializer_list segments = {}, + const Json* body = nullptr, const Query& query = Query()); + // Raw octets as request body (AssemblyAI upload, Deepgram listen). + llm::RawResponse call_bytes(const std::string& id, std::initializer_list segments, + const std::string& data, const std::string& content_type, + const Query& query = Query()); + // multipart/form-data request; response parsed as JSON. + Json call_form(const std::string& id, std::initializer_list segments, + const clients::body::MultipartBody& form, const Query& query = Query()); + // multipart/form-data request; response returned buffered (voice changer, isolation). + llm::RawResponse call_form_raw(const std::string& id, std::initializer_list segments, + const clients::body::MultipartBody& form, + const Query& query = Query()); + // Incremental response body (streamed audio). `sink` returns false to stop. + ByteStreamer::Result call_stream(const std::string& id, + std::initializer_list segments, const Json* body, + const Query& query, const ChunkSink& sink); + // Destructive rows (DELETE, cancel, disable, remove, revoke): explicit, no default. + Json call_destructive(const std::string& id, std::initializer_list segments, + const Json* body, const Query& query, bool confirm); + + // Everything the call would send, without sending it: the final path with + // query string. Throws llm::ApiError::invalid exactly like the real call. + std::string preview(const std::string& id, std::initializer_list segments, + const Json* body = nullptr, const Query& query = Query()) const; + + // Checks (throw llm::ApiError::invalid). + void check_body(const CallSpec& spec, const Json* body) const; + void check_form(const CallSpec& spec, const clients::body::MultipartBody& form) const; + void check_query(const CallSpec& spec, const Query& query) const; + void require_text(const std::string& what, const std::string& s, std::size_t max_chars, + bool required = true) const; + void require_range(const std::string& what, double v, double lo, double hi) const; + void require_confirm(const std::string& what, bool confirm) const; + [[noreturn]] void bad(const std::string& why) const; + + // Converts a JSON object of scalars / arrays to a Query restricted to the + // call's declared query parameters (unknown names are refused). + Query query_from(const std::string& id, const Json& object) const; + + protected: + const AudioHost& host_of(const CallSpec& spec) const; + std::string path_of(const CallSpec& spec, std::initializer_list segments, + const Query& query) const; + void guard_method_kind(const CallSpec& spec, const char* want_body) const; + + const CallTable& table_; + std::shared_ptr hosts_; + std::string vendor_; +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/call_spec.cpp b/src/vendors/audio/call_spec.cpp new file mode 100644 index 0000000..6157b42 --- /dev/null +++ b/src/vendors/audio/call_spec.cpp @@ -0,0 +1,132 @@ +#include "vendors/audio/call_spec.h" + +#include + +#include "llm/api_client.h" +#include "llm/error.h" + +namespace molo::audio { + +const CallSpec* CallTable::find(const std::string& id) const { + for (std::size_t i = 0; i < count_; ++i) + if (id == rows_[i].id) return &rows_[i]; + return nullptr; +} + +const CallSpec& CallTable::get(const std::string& id) const { + const CallSpec* s = find(id); + if (!s) throw llm::ApiError::invalid(provider_, "unknown call id: " + id); + return *s; +} + +std::vector csv_list(const char* csv) { + std::vector out; + std::string cur; + for (const char* p = csv; p && *p; ++p) { + if (*p == ',') { + if (!cur.empty()) out.push_back(cur); + cur.clear(); + } else { + cur.push_back(*p); + } + } + if (!cur.empty()) out.push_back(cur); + return out; +} + +bool csv_has(const char* csv, const std::string& name) { + std::string cur; + for (const char* p = csv; ; ++p) { + if (*p == ',' || *p == '\0') { + if (cur == name) return true; + cur.clear(); + if (*p == '\0') return false; + } else { + cur.push_back(*p); + } + } +} + +clients::Method method_of(const CallSpec& spec) { + std::string m = spec.method; + if (m == "GET") return clients::Method::kGet; + if (m == "POST") return clients::Method::kPost; + if (m == "PUT") return clients::Method::kPut; + if (m == "PATCH") return clients::Method::kPatch; + return clients::Method::kDelete; +} + +std::size_t placeholder_count(const CallSpec& spec) { + std::size_t n = 0; + for (const char* p = spec.path; *p; ++p) n += *p == '{'; + return n; +} + +bool is_destructive(const CallSpec& spec) { + if (std::string(spec.method) == "DELETE") return true; + std::string id = spec.id; + for (char& c : id) c = static_cast(std::tolower(static_cast(c))); + for (const char* word : {"delete", "cancel", "disable", "remove", "revoke", "purge", "leave"}) + if (id.find(word) != std::string::npos) return true; + return false; +} + +namespace { + +std::string percent(const std::string& s) { + static const char* hex = "0123456789ABCDEF"; + std::string out; + for (unsigned char c : s) { + if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { + out.push_back(static_cast(c)); + } else { + out.push_back('%'); + out.push_back(hex[c >> 4]); + out.push_back(hex[c & 15]); + } + } + return out; +} + +bool plausible_email(const std::string& s) { + if (s.size() < 3 || s.size() > 254) return false; + std::size_t at = s.find('@'); + if (at == std::string::npos || at == 0 || at + 1 >= s.size() || s.find('@', at + 1) != std::string::npos) + return false; + for (unsigned char c : s) + if (c <= 32 || c == '/' || c == '\\' || c == '?' || c == '#') return false; + return true; +} + +} // namespace + +std::string render_path(const std::string& vendor, const CallSpec& spec, + std::initializer_list segments) { + std::string path = spec.path; + std::string out; + auto it = segments.begin(); + for (std::size_t i = 0; i < path.size(); ++i) { + if (path[i] != '{') { + out.push_back(path[i]); + continue; + } + std::size_t close = path.find('}', i); + if (close == std::string::npos || it == segments.end()) + throw llm::ApiError::invalid(vendor, "wrong number of path segments for " + std::string(spec.id)); + std::string name = path.substr(i + 1, close - i - 1); + if (name.find("email") != std::string::npos) { + if (!plausible_email(*it)) throw llm::ApiError::invalid(vendor, "bad email path segment"); + out += percent(*it); + } else { + if (!llm::Service::valid_id(*it)) throw llm::ApiError::invalid(vendor, "bad " + name + " path segment"); + out += *it; + } + ++it; + i = close; + } + if (it != segments.end()) + throw llm::ApiError::invalid(vendor, "wrong number of path segments for " + std::string(spec.id)); + return out; +} + +} // namespace molo::audio diff --git a/src/vendors/audio/call_spec.h b/src/vendors/audio/call_spec.h new file mode 100644 index 0000000..75e7ef4 --- /dev/null +++ b/src/vendors/audio/call_spec.h @@ -0,0 +1,65 @@ +#pragma once + +// The call table type shared by every provider. A provider owns exactly one +// table (a static array of CallSpec rows in its calls.cpp). Every REST call +// the package can make is one row: HTTP method, path template, the host key, +// request body kind and field list, query parameter list, response kind and +// the spec operation it was taken from. tools/check_spec.py verifies the rows +// against the vendor's published specification. + +#include +#include +#include +#include + +#include "clients/request.h" + +namespace molo::audio { + +struct CallSpec { + const char* id; // operationId (or a stable name where the spec has none) + const char* method; // GET POST PUT PATCH DELETE + const char* path; // path template exactly as the spec prints it: /v1/x/{id} + const char* host; // host key: "api", "gateway", "streaming", ... + const char* body; // "-" none | "json" | "form" (multipart) | "bytes" (raw octets) + const char* fields; // comma list of top level body fields; "*" = open schema + const char* required; // comma list of required body fields + const char* query; // comma list of query parameters + const char* response; // "json" | "bytes" | "text" | "none" + const char* source; // "openapi:" | "sdk:" | "docs:" +}; + +class CallTable { + public: + CallTable(const char* provider, const CallSpec* rows, std::size_t count) + : provider_(provider), rows_(rows), count_(count) {} + + const char* provider() const { return provider_; } + std::size_t size() const { return count_; } + const CallSpec& at(std::size_t i) const { return rows_[i]; } + const CallSpec* find(const std::string& id) const; + // Throws llm::ApiError::invalid for an unknown id. + const CallSpec& get(const std::string& id) const; + + private: + const char* provider_; + const CallSpec* rows_; + std::size_t count_; +}; + +// Helpers over the comma lists. +std::vector csv_list(const char* csv); +bool csv_has(const char* csv, const std::string& name); + +clients::Method method_of(const CallSpec& spec); +std::size_t placeholder_count(const CallSpec& spec); +// True for DELETE and for ids that cancel, disable, remove, revoke or purge. +bool is_destructive(const CallSpec& spec); + +// Fills `{placeholders}` in order. Every segment must be a plain id +// (llm::Service::valid_id); a placeholder named *email* takes an address and +// is percent-encoded. Wrong count or bad segment: llm::ApiError::invalid. +std::string render_path(const std::string& vendor, const CallSpec& spec, + std::initializer_list segments); + +} // namespace molo::audio diff --git a/src/vendors/audio/caps.h b/src/vendors/audio/caps.h new file mode 100644 index 0000000..3adcbbc --- /dev/null +++ b/src/vendors/audio/caps.h @@ -0,0 +1,44 @@ +#pragma once + +// Static ceilings for the audio vendor package. Never autotuned. Provider +// limits that are documented are cited in SPEC.md; everything here is our own +// ceiling and sits at or below the provider's. + +#include + +namespace molo::audio { + +struct AudioCaps { + // Text and ids. + static constexpr std::size_t kMaxTextChars = 100000; // one synthesis input + static constexpr std::size_t kMaxChunkChars = 5000; // one chunk of the splitter + static constexpr std::size_t kMaxChunks = 64; // chunks per splitter call + static constexpr std::size_t kMaxFieldChars = 8192; // names, prompts, urls + static constexpr std::size_t kMaxListItems = 1000; // items in a request list + static constexpr std::size_t kMaxQueryChars = 8000; // rendered query string + // Payloads. + static constexpr std::size_t kMaxJsonBodyBytes = 8u * 1024u * 1024u; + static constexpr std::size_t kMaxAudioInBytes = 128u * 1024u * 1024u; // upload / inline audio + static constexpr std::size_t kMaxAudioOutBytes = 64u * 1024u * 1024u; // synthesized audio kept + static constexpr std::size_t kErrorProbeBytes = 64u * 1024u; // json error sniff on streams + // Paging and polling. + static constexpr std::size_t kMaxPages = 20; + static constexpr int kMaxPollSeconds = 1800; + static constexpr int kMinPollIntervalMs = 500; // floor: never poll faster + static constexpr int kMaxPollIntervalMs = 15000; + // WebSocket sessions. + static constexpr int kConnectTimeoutMs = 10000; + static constexpr int kHandshakeTimeoutMs = 10000; + static constexpr int kIoTimeoutMs = 5000; + static constexpr int kPollSliceMs = 200; + static constexpr int kMaxSessionSeconds = 1800; + static constexpr int kMaxWaitMs = 120000; // one next()/collect() wait + static constexpr std::size_t kMaxEventsPerCollect = 4096; + static constexpr std::size_t kMaxSocketFrameBytes = 4u * 1024u * 1024u; // one frame we send + static constexpr std::size_t kMaxStreamAudioBytes = 64u * 1024u * 1024u; // audio kept per session + // Provider specific, all documented in the module SPEC. + static constexpr std::size_t kMaxKeywords = 100; + static constexpr std::size_t kMaxSpeakers = 64; +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/cartesia/calls.cpp b/src/vendors/audio/cartesia/calls.cpp new file mode 100644 index 0000000..5ec4eb0 --- /dev/null +++ b/src/vendors/audio/cartesia/calls.cpp @@ -0,0 +1,72 @@ +#include "vendors/audio/cartesia/calls.h" + +namespace molo::audio::cartesia { + +// One row per REST call. tools/check_spec.py parses the block between the +// markers: {id, method, path, host, body, fields, required, query, response, source}. +// CALLS-BEGIN +static const CallSpec kRows[] = { + {"get_status", "GET", "/", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /"}, + {"agents_retrieve", "GET", "/agents/{agent_id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/{agent_id}"}, + {"agents_update", "PATCH", "/agents/{agent_id}", "api", "json", "description,name,tts_language,tts_voice", "", "", "json", "sdk:cartesia-python api.md patch /agents/{agent_id}"}, + {"agents_list", "GET", "/agents", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents"}, + {"agents_delete", "DELETE", "/agents/{agent_id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /agents/{agent_id}"}, + {"agents_list_phone_numbers", "GET", "/agents/{agent_id}/phone-numbers", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/{agent_id}/phone-numbers"}, + {"agents_list_templates", "GET", "/agents/templates", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/templates"}, + {"calls_retrieve", "GET", "/agents/calls/{call_id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/calls/{call_id}"}, + {"calls_list", "GET", "/agents/calls", "api", "-", "", "", "agent_id,ending_before,expand,limit,starting_after", "json", "sdk:cartesia-python api.md get /agents/calls"}, + {"calls_download_audio", "GET", "/agents/calls/{call_id}/audio", "api", "-", "", "", "", "bytes", "sdk:cartesia-python api.md get /agents/calls/{call_id}/audio"}, + {"metrics_create", "POST", "/agents/metrics", "api", "json", "display_name,name,prompt", "name,prompt", "", "json", "sdk:cartesia-python api.md post /agents/metrics"}, + {"metrics_retrieve", "GET", "/agents/metrics/{metric_id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/metrics/{metric_id}"}, + {"metrics_list", "GET", "/agents/metrics", "api", "-", "", "", "limit,starting_after", "json", "sdk:cartesia-python api.md get /agents/metrics"}, + {"metrics_add_to_agent", "POST", "/agents/{agent_id}/metrics/{metric_id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md post /agents/{agent_id}/metrics/{metric_id}"}, + {"metrics_remove_from_agent", "DELETE", "/agents/{agent_id}/metrics/{metric_id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /agents/{agent_id}/metrics/{metric_id}"}, + {"metrics_results_list", "GET", "/agents/metrics/results", "api", "-", "", "", "agent_id,call_id,deployment_id,end_date,ending_before,limit,metric_id,start_date,starting_after", "json", "sdk:cartesia-python api.md get /agents/metrics/results"}, + {"metrics_results_export", "GET", "/agents/metrics/results/export", "api", "-", "", "", "agent_id,call_id,deployment_id,end_date,metric_id,start_date", "text", "sdk:cartesia-python api.md get /agents/metrics/results/export"}, + {"deployments_retrieve", "GET", "/agents/deployments/{deployment_id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/deployments/{deployment_id}"}, + {"deployments_list", "GET", "/agents/{agent_id}/deployments", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /agents/{agent_id}/deployments"}, + {"access_token_create", "POST", "/access-token", "api", "json", "expires_in,grants", "", "", "json", "sdk:cartesia-python api.md post /access-token"}, + {"datasets_create", "POST", "/datasets/", "api", "json", "description,name", "description,name", "", "json", "sdk:cartesia-python api.md post /datasets/"}, + {"datasets_retrieve", "GET", "/datasets/{id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /datasets/{id}"}, + {"datasets_update", "PATCH", "/datasets/{id}", "api", "json", "description,name", "description,name", "", "none", "sdk:cartesia-python api.md patch /datasets/{id}"}, + {"datasets_list", "GET", "/datasets/", "api", "-", "", "", "ending_before,limit,starting_after", "json", "sdk:cartesia-python api.md get /datasets/"}, + {"datasets_delete", "DELETE", "/datasets/{id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /datasets/{id}"}, + {"datasets_files_list", "GET", "/datasets/{id}/files", "api", "-", "", "", "ending_before,limit,starting_after", "json", "sdk:cartesia-python api.md get /datasets/{id}/files"}, + {"datasets_files_delete", "DELETE", "/datasets/{id}/files/{file_id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /datasets/{id}/files/{fileID}"}, + {"datasets_files_upload", "POST", "/datasets/{id}/files", "api", "form", "file,purpose", "file", "", "none", "sdk:cartesia-python api.md post /datasets/{id}/files"}, + {"fine_tunes_create", "POST", "/fine-tunes/", "api", "json", "accent,dataset,description,language,model_id,name", "dataset,description,language,model_id,name", "", "json", "sdk:cartesia-python api.md post /fine-tunes/"}, + {"fine_tunes_retrieve", "GET", "/fine-tunes/{id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /fine-tunes/{id}"}, + {"fine_tunes_list", "GET", "/fine-tunes/", "api", "-", "", "", "ending_before,limit,starting_after", "json", "sdk:cartesia-python api.md get /fine-tunes/"}, + {"fine_tunes_delete", "DELETE", "/fine-tunes/{id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /fine-tunes/{id}"}, + {"fine_tunes_list_voices", "GET", "/fine-tunes/{id}/voices", "api", "-", "", "", "ending_before,limit,starting_after", "json", "sdk:cartesia-python api.md get /fine-tunes/{id}/voices"}, + {"pronunciation_dicts_create", "POST", "/pronunciation-dicts/", "api", "json", "items,name", "name", "", "json", "sdk:cartesia-python api.md post /pronunciation-dicts/"}, + {"pronunciation_dicts_retrieve", "GET", "/pronunciation-dicts/{id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /pronunciation-dicts/{id}"}, + {"pronunciation_dicts_update", "PATCH", "/pronunciation-dicts/{id}", "api", "json", "items,name", "", "", "json", "sdk:cartesia-python api.md patch /pronunciation-dicts/{id}"}, + {"pronunciation_dicts_list", "GET", "/pronunciation-dicts/", "api", "-", "", "", "ending_before,limit,starting_after", "json", "sdk:cartesia-python api.md get /pronunciation-dicts/"}, + {"pronunciation_dicts_delete", "DELETE", "/pronunciation-dicts/{id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /pronunciation-dicts/{id}"}, + {"stt_transcribe", "POST", "/stt", "api", "form", "file,model,encoding,sample_rate,language,timestamp_granularities[]", "file,model", "", "json", "sdk:cartesia-python api.md post /stt"}, + {"tts_generate", "POST", "/tts/bytes", "api", "json", "model_id,output_format,transcript,voice,generation_config,language,locale,normalization,pronunciation_dict_id,save,speed", "model_id,output_format,transcript,voice", "", "bytes", "sdk:cartesia-python api.md post /tts/bytes"}, + {"tts_generate_sse", "POST", "/tts/sse", "api", "json", "model_id,output_format,transcript,voice,add_phoneme_timestamps,add_timestamps,context_id,generation_config,language,locale,normalization,pronunciation_dict_id,speed,use_normalized_timestamps", "model_id,output_format,transcript,voice", "", "bytes", "sdk:cartesia-python api.md post /tts/sse"}, + {"tts_infill", "POST", "/infill/bytes", "api", "form", "language,left_audio,model_id,output_format[container],output_format[encoding],output_format[sample_rate],output_format[bit_rate],right_audio,transcript,voice_id", "left_audio,right_audio,transcript,voice_id", "", "bytes", "sdk:cartesia-python api.md post /infill/bytes"}, + {"voice_changer_generate", "POST", "/voice-changer/bytes", "api", "form", "clip,output_format[bit_rate],output_format[container],output_format[encoding],output_format[sample_rate],voice[id]", "clip,voice[id]", "", "bytes", "sdk:cartesia-python api.md post /voice-changer/bytes"}, + {"voice_changer_generate_sse", "POST", "/voice-changer/sse", "api", "form", "clip,output_format[bit_rate],output_format[container],output_format[encoding],output_format[sample_rate],voice[id]", "clip,voice[id]", "", "bytes", "sdk:cartesia-python api.md post /voice-changer/sse"}, + {"voices_update", "PATCH", "/voices/{id}", "api", "json", "accent,description,gender,name", "", "", "json", "sdk:cartesia-python api.md patch /voices/{id}"}, + {"voices_list", "GET", "/voices", "api", "-", "", "", "ending_before,expand,gender,is_owner,limit,q,starting_after", "json", "sdk:cartesia-python api.md get /voices"}, + {"voices_delete", "DELETE", "/voices/{id}", "api", "-", "", "", "", "none", "sdk:cartesia-python api.md delete /voices/{id}"}, + {"voices_add_accents", "PATCH", "/voices/{id}/accents", "api", "json", "accents", "accents", "", "json", "sdk:cartesia-python api.md patch /voices/{id}/accents"}, + {"voices_clone", "POST", "/voices/clone", "api", "form", "accent,base_voice_id,clip,description,language,name", "clip,language,name", "", "json", "sdk:cartesia-python api.md post /voices/clone"}, + {"voices_delete_accent", "DELETE", "/voices/{id}/accents/{accent_id}", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md delete /voices/{id}/accents/{accent_id}"}, + {"voices_get", "GET", "/voices/{id}", "api", "-", "", "", "expand", "json", "sdk:cartesia-python api.md get /voices/{id}"}, + {"voices_list_accents", "GET", "/accents", "api", "-", "", "", "", "json", "sdk:cartesia-python api.md get /accents"}, + {"voices_localize", "POST", "/voices/localize", "api", "json", "accent,description,dialect,language,name,original_speaker_gender,tagline,voice_id", "accent,name,voice_id", "", "json", "sdk:cartesia-python api.md post /voices/localize"}, +}; +// CALLS-END + +static_assert(sizeof(kRows) / sizeof(kRows[0]) == 53, "cartesia call table size"); + +const CallTable& calls() { + static const CallTable table("cartesia", kRows, sizeof(kRows) / sizeof(kRows[0])); + return table; +} + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/calls.h b/src/vendors/audio/cartesia/calls.h new file mode 100644 index 0000000..d712f79 --- /dev/null +++ b/src/vendors/audio/cartesia/calls.h @@ -0,0 +1,39 @@ +#pragma once + +// The Cartesia call table. Cartesia's docs host answers machine fetches with a +// bot-check page, so the rows come from the official Python SDK (generated +// from Cartesia's OpenAPI document): api.md for method and path, the +// *_params.py files for fields. Version 2026-08-14. + +#include "vendors/audio/call_spec.h" + +namespace molo::audio::cartesia { + +const CallTable& calls(); + +struct CartesiaCalls { + static constexpr const char* kStatus = "get_status"; + static constexpr const char* kTts = "tts_generate"; + static constexpr const char* kTtsSse = "tts_generate_sse"; + static constexpr const char* kInfill = "tts_infill"; + static constexpr const char* kStt = "stt_transcribe"; + static constexpr const char* kVoices = "voices_list"; + static constexpr const char* kVoice = "voices_get"; + static constexpr const char* kVoiceUpdate = "voices_update"; + static constexpr const char* kVoiceDelete = "voices_delete"; + static constexpr const char* kVoiceClone = "voices_clone"; + static constexpr const char* kVoiceLocalize = "voices_localize"; + static constexpr const char* kAccents = "voices_list_accents"; + static constexpr const char* kVoiceChanger = "voice_changer_generate"; + static constexpr const char* kAccessToken = "access_token_create"; + static constexpr const char* kAgents = "agents_list"; + static constexpr const char* kAgent = "agents_retrieve"; + static constexpr const char* kCalls = "calls_list"; + static constexpr const char* kCall = "calls_retrieve"; + static constexpr const char* kCallAudio = "calls_download_audio"; + static constexpr const char* kPronunciationList = "pronunciation_dicts_list"; + static constexpr const char* kFineTunes = "fine_tunes_list"; + static constexpr const char* kDatasets = "datasets_list"; +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/client.cpp b/src/vendors/audio/cartesia/client.cpp new file mode 100644 index 0000000..19cd00f --- /dev/null +++ b/src/vendors/audio/cartesia/client.cpp @@ -0,0 +1,35 @@ +#include "vendors/audio/cartesia/client.h" + +#include "llm/config.h" + +namespace molo::audio::cartesia { + +llm::ApiClientConfig CartesiaEndpoints::config() { + llm::ApiClientConfig c; + c.vendor = kVendor; + c.base_url = kBase; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kBearer; + c.default_headers["Cartesia-Version"] = kVersion; + return c; +} + +std::shared_ptr CartesiaEndpoints::from_env() { + auto hosts = std::make_shared(); + hosts->add("api", HostFactory::from_env(config())); + return hosts; +} + +std::shared_ptr CartesiaEndpoints::with(AudioHost host) { + auto hosts = std::make_shared(); + hosts->add("api", std::move(host)); + return hosts; +} + +std::string CartesiaEndpoints::key() { return llm::Config::get(kKeyEnv); } + +AudioSocket::Headers CartesiaEndpoints::ws_headers(const std::string& key) { + return {{"Authorization", "Bearer " + key}, {"Cartesia-Version", kVersion}}; +} + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/client.h b/src/vendors/audio/cartesia/client.h new file mode 100644 index 0000000..80f3633 --- /dev/null +++ b/src/vendors/audio/cartesia/client.h @@ -0,0 +1,31 @@ +#pragma once + +// Cartesia endpoints and credentials. +// REST https://api.cartesia.ai Authorization: Bearer $CARTESIA_API_KEY, Cartesia-Version: 2026-08-14 +// WebSocket wss://api.cartesia.ai same headers, plus ?cartesia_version=2026-08-14 +// The API version is a required header (an unversioned POST answers 400, +// observed 2026-10-25) and is pinned here, not taken from configuration. + +#include +#include + +#include "vendors/audio/hosts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::cartesia { + +struct CartesiaEndpoints { + static constexpr const char* kVendor = "cartesia"; + static constexpr const char* kKeyEnv = "CARTESIA_API_KEY"; + static constexpr const char* kBase = "https://api.cartesia.ai"; + static constexpr const char* kWsBase = "wss://api.cartesia.ai"; + static constexpr const char* kVersion = "2026-08-14"; + + static llm::ApiClientConfig config(); + static std::shared_ptr from_env(); + static std::shared_ptr with(AudioHost host); // one host called "api" + static std::string key(); + static AudioSocket::Headers ws_headers(const std::string& key); +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/speech.cpp b/src/vendors/audio/cartesia/speech.cpp new file mode 100644 index 0000000..c300f7c --- /dev/null +++ b/src/vendors/audio/cartesia/speech.cpp @@ -0,0 +1,79 @@ +#include "vendors/audio/cartesia/speech.h" + +#include + +namespace molo::audio::cartesia { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("cartesia", why); } +} // namespace + +CartesiaSynthesizer::CartesiaSynthesizer(std::shared_ptr hosts) : tts_(hosts), voices_(hosts) {} + +CartesiaTtsRequest CartesiaSynthesizer::map(const SpeechRequest& r) const { + validate(r); + if (r.voice.empty()) invalid("a voice id is required (list voices first)"); + CartesiaTtsRequest t; + if (!r.model.empty()) t.model_id = r.model; + t.transcript = r.text; + t.voice_id = r.voice; + t.language = r.language.substr(0, 2); + t.output_format.container = r.format == "pcm" ? "raw" : r.format; + t.output_format.encoding = "pcm_s16le"; + if (r.sample_rate != 0) t.output_format.sample_rate = r.sample_rate; + if (r.format == "mp3" && r.sample_rate == 0) t.output_format.sample_rate = 44100; + if (r.speed > 0) t.gen_speed = std::clamp(r.speed, CartesiaCaps::kMinGenSpeed, CartesiaCaps::kMaxGenSpeed); + return t; +} + +SpeechResult CartesiaSynthesizer::synthesize(const SpeechRequest& r) { + CartesiaAudio a = tts_.synthesize(map(r)); + SpeechResult out; + out.audio = std::move(a.bytes); + out.content_type = a.content_type.empty() ? content_type_for(r.format) : a.content_type; + out.format = r.format; + out.provider = name(); + return out; +} + +Json CartesiaSynthesizer::voices() { return voices_.list(); } + +CartesiaRecognizer::CartesiaRecognizer(std::shared_ptr hosts) : stt_(std::move(hosts)) {} + +CartesiaSttRequest CartesiaRecognizer::map(const TranscribeRequest& r) const { + validate(r); + if (r.audio.empty()) invalid("Cartesia batch transcription takes audio bytes, not a URL"); + CartesiaSttRequest s; + s.audio = r.audio; + s.filename = r.filename; + s.content_type = r.content_type; + if (!r.model.empty()) s.model = r.model; + s.language = r.language.substr(0, 2); + s.word_timestamps = true; + return s; +} + +Transcript CartesiaRecognizer::fold(const Json& doc) { + Transcript t; + t.provider = "cartesia"; + t.raw = doc; + if (!doc.is_object()) return t; + t.text = llm::Service::string_at(doc, "text"); + t.language = llm::Service::string_at(doc, "language"); + if (const Json* d = doc.find("duration"); d && d->is_number()) t.duration = d->as_number(); + if (const Json* words = doc.find("words"); words && words->is_array()) { + for (const Json& w : words->as_array()) { + if (!w.is_object()) continue; + TranscriptWord word; + word.text = llm::Service::string_at(w, "word"); + if (const Json* x = w.find("start"); x && x->is_number()) word.start = x->as_number(); + if (const Json* x = w.find("end"); x && x->is_number()) word.end = x->as_number(); + t.words.push_back(word); + } + } + return t; +} + +Transcript CartesiaRecognizer::transcribe(const TranscribeRequest& r) { return fold(stt_.transcribe(map(r))); } + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/speech.h b/src/vendors/audio/cartesia/speech.h new file mode 100644 index 0000000..fd3a87e --- /dev/null +++ b/src/vendors/audio/cartesia/speech.h @@ -0,0 +1,43 @@ +#pragma once + +// Cartesia behind the neutral speech seam. + +#include + +#include "vendors/audio/cartesia/stt.h" +#include "vendors/audio/cartesia/tts.h" +#include "vendors/audio/cartesia/voices.h" +#include "vendors/audio/speech.h" + +namespace molo::audio::cartesia { + +class CartesiaSynthesizer : public SpeechSynthesizer { + public: + explicit CartesiaSynthesizer(std::shared_ptr hosts = CartesiaEndpoints::from_env()); + std::string name() const override { return "cartesia"; } + bool configured() const override { return tts_.configured(); } + std::size_t max_chars() const override { return CartesiaCaps::kMaxTranscriptChars; } + std::vector formats() const override { return {"mp3", "wav", "pcm"}; } + SpeechResult synthesize(const SpeechRequest& request) override; + Json voices() override; + CartesiaTtsRequest map(const SpeechRequest& request) const; + + private: + CartesiaTts tts_; + CartesiaVoices voices_; +}; + +class CartesiaRecognizer : public SpeechRecognizer { + public: + explicit CartesiaRecognizer(std::shared_ptr hosts = CartesiaEndpoints::from_env()); + std::string name() const override { return "cartesia"; } + bool configured() const override { return stt_.configured(); } + Transcript transcribe(const TranscribeRequest& request) override; // audio bytes only (no URL input) + CartesiaSttRequest map(const TranscribeRequest& request) const; + static Transcript fold(const Json& doc); + + private: + CartesiaStt stt_; +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/stt.cpp b/src/vendors/audio/cartesia/stt.cpp new file mode 100644 index 0000000..ef53e5e --- /dev/null +++ b/src/vendors/audio/cartesia/stt.cpp @@ -0,0 +1,177 @@ +#include "vendors/audio/cartesia/stt.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::cartesia { + +using C = CartesiaCalls; + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("cartesia", why); } + +bool valid_encoding(const std::string& e) { + for (const char* ok : {"pcm_s16le", "pcm_s32le", "pcm_f16le", "pcm_f32le", "pcm_mulaw", "pcm_alaw"}) + if (e == ok) return true; + return false; +} + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} +} // namespace + +clients::body::MultipartBody CartesiaStt::build(const CartesiaSttRequest& r) const { + if (r.audio.empty() || r.audio.size() > CartesiaCaps::kMaxFileBytes) bad("audio must be 1 byte..100 MiB"); + require_text("model", r.model, 128); + require_text("language", r.language, 16, false); + if (!r.encoding.empty() && !valid_encoding(r.encoding)) bad("unknown encoding"); + if (r.sample_rate < 0 || r.sample_rate > 96000) bad("sample_rate must be 0 or up to 96000"); + clients::body::MultipartBody form; + form.add_file("file", r.filename, r.content_type, r.audio); + form.add_text("model", r.model); + if (!r.language.empty()) form.add_text("language", r.language); + if (!r.encoding.empty()) form.add_text("encoding", r.encoding); + if (r.sample_rate > 0) form.add_text("sample_rate", std::to_string(r.sample_rate)); + if (r.word_timestamps) form.add_text("timestamp_granularities[]", "word"); + return form; +} + +Json CartesiaStt::transcribe(const CartesiaSttRequest& r) { return call_form(C::kStt, {}, build(r)); } + +Vocabulary CartesiaSttSocket::auto_vocabulary() { + // stt_auto_finalize_websocket_request / _response in the Python SDK types. + return Vocabulary("type", {"close", "config"}, + {"connected", "turn.start", "turn.update", "turn.eager_end", "turn.resume", "turn.end", "error"}); +} +Vocabulary CartesiaSttSocket::manual_vocabulary() { + return Vocabulary("type", {}, {"transcript", "flush_done", "done", "error"}); +} + +CartesiaSttSocket::CartesiaSttSocket(std::string api_key, SttSocketParams p, bool auto_finalize, std::string url, + SocketOptions socket) + : auto_(auto_finalize) { + if (!valid_encoding(p.encoding)) invalid("unknown encoding"); + if (p.sample_rate < 8000 || p.sample_rate > 96000) invalid("sample_rate must be 8000..96000"); + if (p.model.empty() || p.model.size() > 128) invalid("model is required"); + if (p.keyterms.size() > AudioCaps::kMaxKeywords) invalid("too many keyterms"); + Query q; + q.add("encoding", p.encoding); + q.add("model", p.model); + q.add_int("sample_rate", p.sample_rate); + for (const auto& k : p.keyterms) { + if (k.empty() || k.size() > 100) invalid("keyterms are 1..100 characters"); + q.add("keyterm", k); + } + if (auto_finalize) { + if (p.turn_eager_end_threshold >= 0) q.add("turn_eager_end_threshold", std::to_string(p.turn_eager_end_threshold)); + if (p.turn_end_threshold >= 0) q.add("turn_end_threshold", std::to_string(p.turn_end_threshold)); + if (p.turn_start_threshold >= 0) q.add("turn_start_threshold", std::to_string(p.turn_start_threshold)); + if (p.turn_end_timeout_ms >= 0) q.add("turn_end_timeout_ms", std::to_string(static_cast(p.turn_end_timeout_ms))); + } else { + if (!p.language.empty()) q.add("language", p.language); + if (p.max_silence_duration_secs >= 0) q.add("max_silence_duration_secs", std::to_string(p.max_silence_duration_secs)); + if (p.min_volume >= 0) q.add("min_volume", std::to_string(p.min_volume)); + } + std::string base = url.empty() ? std::string(CartesiaEndpoints::kWsBase) + (auto_finalize ? "/stt/turns/websocket" : "/stt/websocket") : url; + socket_ = std::make_unique("cartesia", base + q.str(), CartesiaEndpoints::ws_headers(api_key), + auto_finalize ? auto_vocabulary() : manual_vocabulary(), std::move(socket)); +} + +void CartesiaSttSocket::finalize() { + if (auto_) invalid("finalize belongs to the manual-finalize socket"); + socket_->send_text("finalize"); +} + +void CartesiaSttSocket::end() { + if (auto_) { + Json m = Json::object(); + m.set("type", Json("close")); + socket_->send_json(m); + } else { + socket_->send_text("close"); + } +} + +void CartesiaSttSocket::configure_turn(double eager, double end_t, double start_t, double timeout_ms) { + if (!auto_) invalid("turn configuration belongs to the auto-finalize socket"); + Json turn = Json::object(); + if (eager >= 0) { + if (eager < 0.3 || eager > 0.6) invalid("eager_end_threshold must be 0.3..0.6"); + turn.set("eager_end_threshold", Json(eager)); + } + if (end_t >= 0) { + if (end_t < 0.05 || end_t > 0.5) invalid("end_threshold must be 0.05..0.5"); + turn.set("end_threshold", Json(end_t)); + } + if (start_t >= 0) { + if (start_t < 0.5 || start_t > 0.9) invalid("start_threshold must be 0.5..0.9"); + turn.set("start_threshold", Json(start_t)); + } + if (timeout_ms >= 0) { + if (timeout_ms < 640 || timeout_ms > 11200) invalid("end_timeout_ms must be 640..11200"); + turn.set("end_timeout_ms", Json(timeout_ms)); + } + Json m = Json::object(); + m.set("type", Json("config")); + m.set("turn", turn); + socket_->send_json(m); +} + +SttEvent CartesiaSttSocket::decode(const SocketEvent& ev) { + SttEvent out; + if (ev.kind != SocketEvent::Kind::kText || !ev.json_ok) return out; + out.raw = ev.json; + out.type = ev.type; + out.request_id = str_at(ev.json, "request_id"); + if (ev.type.rfind("turn.", 0) == 0) { + out.transcript = str_at(ev.json, "transcript"); + out.is_final = ev.type == "turn.end"; + } else if (ev.type == "transcript") { + out.transcript = str_at(ev.json, "text"); + if (const Json* f = ev.json.find("is_final"); f && f->is_bool()) out.is_final = f->as_bool(); + } + return out; +} + +SttSocketResult CartesiaSttSocket::collect(int max_wait_ms) { + SttSocketResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto raw = socket_->next(max_wait_ms); + if (!raw) return out; + ++out.events; + if (raw->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + SttEvent ev = decode(*raw); + if (!ev.request_id.empty()) out.request_id = ev.request_id; + if (ev.type == "error") { + out.error = str_at(ev.raw, "message"); + if (out.error.empty()) out.error = str_at(ev.raw, "title"); + if (out.error.empty()) out.error = "stream error"; + return out; + } + if (ev.type == "turn.end") { + out.finals.push_back(ev.transcript); + out.transcript += (out.transcript.empty() ? "" : " ") + ev.transcript; + out.partial.clear(); + } else if (ev.type == "turn.update" || ev.type == "turn.eager_end") { + out.partial = ev.transcript; + } else if (ev.type == "transcript") { + if (ev.is_final) { + // Manual mode deltas concatenate exactly as received (the API says: do not trim or add whitespace). + out.finals.push_back(ev.transcript); + out.transcript += ev.transcript; + } else { + out.partial = ev.transcript; + } + } else if (ev.type == "done") { + out.done = true; + return out; + } + } + return out; +} + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/stt.h b/src/vendors/audio/cartesia/stt.h new file mode 100644 index 0000000..d5ae423 --- /dev/null +++ b/src/vendors/audio/cartesia/stt.h @@ -0,0 +1,93 @@ +#pragma once + +// Cartesia speech to text: batch POST /stt (multipart) and the two realtime +// WebSockets: /stt/turns/websocket (auto finalize, turn events) and +// /stt/websocket (manual finalize). + +#include +#include +#include +#include + +#include "vendors/audio/cartesia/tts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::cartesia { + +struct CartesiaSttRequest { + std::string audio; // required bytes + std::string filename = "audio"; + std::string content_type = "application/octet-stream"; + std::string model = "ink-whisper"; // ink-whisper or a dated snapshot + std::string language; // ISO 639-1 code, "" = detect + std::string encoding; // for raw audio: pcm_s16le pcm_s32le pcm_f16le pcm_f32le pcm_mulaw pcm_alaw + int sample_rate = 0; + bool word_timestamps = false; // timestamp_granularities[]=word +}; + +class CartesiaStt : public CartesiaService { + public: + using CartesiaService::CartesiaService; + clients::body::MultipartBody build(const CartesiaSttRequest& request) const; + Json transcribe(const CartesiaSttRequest& request); +}; + +struct SttSocketParams { + std::string encoding = "pcm_s16le"; // required by the SDK signature + std::string model = "ink-whisper"; + int sample_rate = 16000; + std::vector keyterms; + // auto-finalize only (/stt/turns/websocket) + double turn_eager_end_threshold = -1, turn_end_threshold = -1, turn_start_threshold = -1; + double turn_end_timeout_ms = -1; + // manual-finalize only (/stt/websocket) + std::string language; + double max_silence_duration_secs = -1, min_volume = -1; +}; + +struct SttEvent { + std::string type; // connected turn.start turn.update turn.eager_end turn.resume turn.end transcript flush_done done error + std::string transcript; // turn.* events: transcript; manual: text delta + bool is_final = false; + std::string request_id; + Json raw; +}; + +struct SttSocketResult { + std::vector finals; // turn.end transcripts (auto) or is_final text deltas (manual) + std::string transcript; // auto: joined with a space; manual: concatenated exactly (no added whitespace) + std::string partial; + std::string request_id; + std::string error; + bool done = false; + bool closed = false; + std::size_t events = 0; +}; + +class CartesiaSttSocket { + public: + // auto_finalize true = /stt/turns/websocket, false = /stt/websocket. + CartesiaSttSocket(std::string api_key, SttSocketParams params, bool auto_finalize, std::string url = "", + SocketOptions socket = {}); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void send_audio(const std::string& audio) { socket_->send_binary(audio); } + void finalize(); // manual: the text frame "finalize" + void end(); // auto: {"type":"close"}; manual: the text frame "close" + void configure_turn(double eager_end, double end_threshold, double start_threshold, double end_timeout_ms); // auto only + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + static SttEvent decode(const SocketEvent& ev); + SttSocketResult collect(int max_wait_ms); + + static Vocabulary auto_vocabulary(); + static Vocabulary manual_vocabulary(); + + private: + bool auto_; + std::unique_ptr socket_; +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/tts.cpp b/src/vendors/audio/cartesia/tts.cpp new file mode 100644 index 0000000..abd36e5 --- /dev/null +++ b/src/vendors/audio/cartesia/tts.cpp @@ -0,0 +1,117 @@ +#include "vendors/audio/cartesia/tts.h" + +#include "llm/sse.h" +#include "security/crypto/Base64.h" + +namespace molo::audio::cartesia { + +using C = CartesiaCalls; + +namespace { +bool in(int v, std::initializer_list ok) { + for (int x : ok) + if (x == v) return true; + return false; +} +} // namespace + +Json OutputFormat::to_json(const CartesiaService& svc) const { + if (!in(sample_rate, {8000, 16000, 22050, 24000, 44100, 48000})) + svc.bad("sample_rate must be 8000, 16000, 22050, 24000, 44100 or 48000"); + Json o = Json::object(); + o.set("container", Json(container)); + o.set("sample_rate", Json(sample_rate)); + if (container == "mp3") { + if (!in(bit_rate, {32000, 64000, 96000, 128000, 192000})) + svc.bad("mp3 bit_rate must be 32000, 64000, 96000, 128000 or 192000"); + o.set("bit_rate", Json(bit_rate)); + } else if (container == "raw" || container == "wav") { + if (encoding != "pcm_f32le" && encoding != "pcm_s16le" && encoding != "pcm_mulaw" && encoding != "pcm_alaw") + svc.bad("encoding must be pcm_f32le, pcm_s16le, pcm_mulaw or pcm_alaw"); + o.set("encoding", Json(encoding)); + } else { + svc.bad("container must be raw, wav or mp3"); + } + return o; +} + +Json CartesiaTts::build(const CartesiaTtsRequest& r, bool sse) const { + require_text("model_id", r.model_id, 128); + require_text("transcript", r.transcript, CartesiaCaps::kMaxTranscriptChars); + check_id("voice", r.voice_id); + Json o = Json::object(); + o.set("model_id", Json(r.model_id)); + o.set("transcript", Json(r.transcript)); + Json voice = Json::object(); + voice.set("id", Json(r.voice_id)); + o.set("voice", voice); + o.set("output_format", r.output_format.to_json(*this)); + if (!r.language.empty()) { + require_text("language", r.language, 16); + o.set("language", Json(r.language)); + } + if (!r.speed.empty()) { + if (r.speed != "slow" && r.speed != "normal" && r.speed != "fast") bad("speed must be slow, normal or fast"); + o.set("speed", Json(r.speed)); + } + Json gen = Json::object(); + if (!r.emotion.empty()) { + require_text("emotion", r.emotion, 64); + gen.set("emotion", Json(r.emotion)); + } + if (r.gen_speed >= 0) { + require_range("generation speed", r.gen_speed, CartesiaCaps::kMinGenSpeed, CartesiaCaps::kMaxGenSpeed); + gen.set("speed", Json(r.gen_speed)); + } + if (r.volume >= 0) { + require_range("volume", r.volume, CartesiaCaps::kMinVolume, CartesiaCaps::kMaxVolume); + gen.set("volume", Json(r.volume)); + } + if (!gen.as_object().empty()) o.set("generation_config", gen); + if (!r.pronunciation_dict_id.empty()) { + check_id("pronunciation dictionary", r.pronunciation_dict_id); + o.set("pronunciation_dict_id", Json(r.pronunciation_dict_id)); + } + if (!r.normalization.empty()) { + require_text("normalization", r.normalization, 16); + o.set("normalization", Json(r.normalization)); + } + check_body(table_.get(sse ? C::kTtsSse : C::kTts), &o); + return o; +} + +CartesiaAudio CartesiaTts::synthesize(const CartesiaTtsRequest& r) { + Json body = build(r); + llm::RawResponse raw = call_raw(C::kTts, {}, &body); + return CartesiaAudio{raw.body, raw.content_type}; +} + +ByteStreamer::Result CartesiaTts::stream(const CartesiaTtsRequest& r, + const std::function& sink) { + Json body = build(r, true); + llm::SseDecoder decoder("cartesia", [&](const llm::SseEvent& ev) -> bool { + const Json& d = ev.data; + if (!d.is_object()) return true; + const Json* type = d.find("type"); + std::string t = type && type->is_string() ? type->as_string() : ev.event; + if (t == "error") { + const Json* m = d.find("message"); + throw llm::ApiError("cartesia", 502, "stream_error", m && m->is_string() ? m->as_string() : "stream error"); + } + if (t == "done") return false; + if (t != "chunk") return true; + const Json* data = d.find("data"); + if (!data || !data->is_string()) return true; + std::string raw; + if (!security::crypto::Base64::decode(data->as_string(), raw)) + throw llm::ApiError("cartesia", 502, "BAD_RESPONSE", "audio chunk is not valid base64"); + return raw.empty() ? true : sink(raw); + }); + ByteStreamer::Result res = call_stream(C::kTtsSse, {}, &body, Query(), [&](const std::string& chunk) { + return decoder.feed(chunk); + }); + decoder.finish(); + return res; +} + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/tts.h b/src/vendors/audio/cartesia/tts.h new file mode 100644 index 0000000..f0073bb --- /dev/null +++ b/src/vendors/audio/cartesia/tts.h @@ -0,0 +1,68 @@ +#pragma once + +// Cartesia text to speech over HTTP: /tts/bytes (buffered audio) and +// /tts/sse (base64 chunks in server-sent events). WebSocket contexts are in +// TtsSocket. + +#include +#include + +#include "vendors/audio/audio_service.h" +#include "vendors/audio/cartesia/calls.h" +#include "vendors/audio/cartesia/client.h" + +namespace molo::audio::cartesia { + +struct CartesiaCaps { + static constexpr std::size_t kMaxTranscriptChars = 20000; // our ceiling for one request + static constexpr double kMinGenSpeed = 0.6, kMaxGenSpeed = 1.5; // generation_config.speed (SDK docstring) + static constexpr double kMinVolume = 0.5, kMaxVolume = 2.0; // generation_config.volume + static constexpr std::size_t kMaxFileBytes = 100u * 1024u * 1024u; + static constexpr int kMaxListLimit = 100; +}; + +class CartesiaService : public AudioService { + public: + explicit CartesiaService(std::shared_ptr hosts) : AudioService(calls(), std::move(hosts)) {} + CartesiaService() : CartesiaService(CartesiaEndpoints::from_env()) {} +}; + +// Audio container and encoding of a synthesis result. +struct OutputFormat { + std::string container = "wav"; // raw | wav | mp3 + std::string encoding = "pcm_s16le"; // raw/wav: pcm_f32le pcm_s16le pcm_mulaw pcm_alaw + int sample_rate = 44100; // 8000 16000 22050 24000 44100 48000 + int bit_rate = 128000; // mp3: 32000 64000 96000 128000 192000 + Json to_json(const CartesiaService& svc) const; +}; + +struct CartesiaTtsRequest { + std::string model_id = "sonic-3"; // sonic-3.5 sonic-3 sonic-latest or a dated snapshot + std::string transcript; // required + std::string voice_id; // required + std::string language; // "" = model default + OutputFormat output_format; + std::string speed; // "", slow, normal, fast (model speed) + // generation_config (sonic-3 only); negative = unset + std::string emotion; + double gen_speed = -1; // 0.6..1.5 + double volume = -1; // 0.5..2.0 + std::string pronunciation_dict_id; + std::string normalization; // "", auto, off, or a locale code +}; + +struct CartesiaAudio { + std::string bytes; + std::string content_type; +}; + +class CartesiaTts : public CartesiaService { + public: + using CartesiaService::CartesiaService; + Json build(const CartesiaTtsRequest& request, bool sse = false) const; + CartesiaAudio synthesize(const CartesiaTtsRequest& request); + // Streams decoded audio chunks from /tts/sse; `sink` returns false to stop. + ByteStreamer::Result stream(const CartesiaTtsRequest& request, const std::function& sink); +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/tts_socket.cpp b/src/vendors/audio/cartesia/tts_socket.cpp new file mode 100644 index 0000000..cf3be07 --- /dev/null +++ b/src/vendors/audio/cartesia/tts_socket.cpp @@ -0,0 +1,127 @@ +#include "vendors/audio/cartesia/tts_socket.h" + +#include "security/crypto/Base64.h" +#include "vendors/audio/caps.h" + +namespace molo::audio::cartesia { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("cartesia", why); } + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} +} // namespace + +Vocabulary TtsSocket::vocabulary() { + // Client messages (generation request, cancel) have no `type` field; server types from websocket_response.py. + return Vocabulary("type", {}, {"chunk", "flush_done", "done", "timestamps", "phoneme_timestamps", "error"}); +} + +TtsSocket::TtsSocket(std::string api_key, std::string url, SocketOptions socket) { + std::string base = url.empty() ? std::string(CartesiaEndpoints::kWsBase) + "/tts/websocket" : url; + base = AudioSocket::with_query(base, std::string("cartesia_version=") + CartesiaEndpoints::kVersion); + socket_ = std::make_unique("cartesia", base, CartesiaEndpoints::ws_headers(api_key), vocabulary(), + std::move(socket)); +} + +Json TtsSocket::build(const TtsSocketRequest& r) { + if (!llm::Service::valid_id(r.context_id)) invalid("bad context id"); + if (r.model_id.empty() || r.model_id.size() > 128) invalid("model_id is required"); + if (r.transcript.empty() || r.transcript.size() > CartesiaCaps::kMaxTranscriptChars) + invalid("transcript must be 1.." + std::to_string(CartesiaCaps::kMaxTranscriptChars) + " characters"); + if (r.transcript.find('\0') != std::string::npos) invalid("transcript has a NUL byte"); + if (!llm::Service::valid_id(r.voice_id)) invalid("bad voice id"); + bool rate_ok = false; + for (int v : {8000, 16000, 22050, 24000, 44100, 48000}) rate_ok = rate_ok || v == r.sample_rate; + if (!rate_ok) invalid("sample_rate must be 8000, 16000, 22050, 24000, 44100 or 48000"); + if (r.encoding != "pcm_f32le" && r.encoding != "pcm_s16le" && r.encoding != "pcm_mulaw" && r.encoding != "pcm_alaw") + invalid("encoding must be pcm_f32le, pcm_s16le, pcm_mulaw or pcm_alaw"); + Json o = Json::object(); + o.set("context_id", Json(r.context_id)); + o.set("model_id", Json(r.model_id)); + o.set("transcript", Json(r.transcript)); + Json voice = Json::object(); + voice.set("id", Json(r.voice_id)); + o.set("voice", voice); + Json fmt = Json::object(); + fmt.set("container", Json("raw")); + fmt.set("encoding", Json(r.encoding)); + fmt.set("sample_rate", Json(r.sample_rate)); + o.set("output_format", fmt); + if (!r.language.empty()) { + if (r.language.size() > 16) invalid("language is too long"); + o.set("language", Json(r.language)); + } + if (r.continue_ >= 0) o.set("continue", Json(r.continue_ == 1)); + if (r.flush >= 0) o.set("flush", Json(r.flush == 1)); + if (r.add_timestamps >= 0) o.set("add_timestamps", Json(r.add_timestamps == 1)); + if (r.add_phoneme_timestamps >= 0) o.set("add_phoneme_timestamps", Json(r.add_phoneme_timestamps == 1)); + if (r.max_buffer_delay_ms >= 0) { + if (r.max_buffer_delay_ms > 5000) invalid("max_buffer_delay_ms must be 0..5000"); + o.set("max_buffer_delay_ms", Json(r.max_buffer_delay_ms)); + } + if (!r.speed.empty()) { + if (r.speed != "slow" && r.speed != "normal" && r.speed != "fast") invalid("speed must be slow, normal or fast"); + o.set("speed", Json(r.speed)); + } + return o; +} + +void TtsSocket::generate(const TtsSocketRequest& r) { socket_->send_json(build(r), true); } + +void TtsSocket::cancel(const std::string& context_id) { + if (!llm::Service::valid_id(context_id)) invalid("bad context id"); + Json m = Json::object(); + m.set("context_id", Json(context_id)); + m.set("cancel", Json(true)); + socket_->send_json(m, true); +} + +TtsSocketResult TtsSocket::collect(const std::string& context_id, int max_wait_ms, bool stop_at_flush) { + TtsSocketResult out; + out.context_id = context_id; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + if (ev->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + if (!ev->json_ok || !ev->json.is_object()) continue; + std::string ctx = str_at(ev->json, "context_id"); + if (!ctx.empty() && ctx != context_id) continue; // another context on this connection + if (ev->type == "chunk") { + std::string raw; + if (!security::crypto::Base64::decode(str_at(ev->json, "data"), raw)) + throw llm::ApiError("cartesia", 502, "BAD_RESPONSE", "audio chunk is not valid base64"); + if (out.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { + out.truncated = true; + return out; + } + out.audio += raw; + ++out.chunks; + if (const Json* f = ev->json.find("flush_id"); f && f->is_number()) out.flush_id = f->as_int(); + } else if (ev->type == "timestamps") { + if (const Json* w = ev->json.find("word_timestamps"); w) out.timestamps.push_back(*w); + } else if (ev->type == "phoneme_timestamps") { + if (const Json* w = ev->json.find("phoneme_timestamps"); w) out.phoneme_timestamps.push_back(*w); + } else if (ev->type == "flush_done") { + out.flush_done = true; + if (const Json* f = ev->json.find("flush_id"); f && f->is_number()) out.flush_id = f->as_int(); + if (stop_at_flush) return out; + } else if (ev->type == "done") { + out.done = true; + return out; + } else if (ev->type == "error") { + out.error = str_at(ev->json, "message"); + if (out.error.empty()) out.error = str_at(ev->json, "title"); + if (out.error.empty()) out.error = "stream error"; + return out; + } + } + return out; +} + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/tts_socket.h b/src/vendors/audio/cartesia/tts_socket.h new file mode 100644 index 0000000..e6f5947 --- /dev/null +++ b/src/vendors/audio/cartesia/tts_socket.h @@ -0,0 +1,71 @@ +#pragma once + +// Cartesia text to speech over WebSocket (wss://api.cartesia.ai/tts/websocket): +// several contexts on one connection, continuations (`continue`), flushes, +// cancel, word and phoneme timestamps. Message shapes: Python SDK +// types/generation_request.py, websocket_client_event.py, websocket_response.py. + +#include +#include +#include +#include +#include + +#include "vendors/audio/cartesia/tts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::cartesia { + +struct TtsSocketRequest { + std::string context_id; // required, you choose it: [A-Za-z0-9._:-]{1,256} + std::string model_id = "sonic-3"; + std::string transcript; // required, may be a partial transcript when continue_ is true + std::string voice_id; // required, same for every request of one context + std::string language; + // The websocket output is raw audio only. + std::string encoding = "pcm_s16le"; + int sample_rate = 44100; + int continue_ = -1; // -1 unset: whether more input follows on this context + int flush = -1; + int add_timestamps = -1; + int add_phoneme_timestamps = -1; + int max_buffer_delay_ms = -1; // 0..5000 + std::string speed; // "", slow, normal, fast +}; + +struct TtsSocketResult { + std::string context_id; + std::string audio; // decoded chunks of that context + std::size_t chunks = 0; + std::vector timestamps; // word_timestamps objects + std::vector phoneme_timestamps; + long flush_id = 0; // last flush_id seen + bool flush_done = false; + bool done = false; + bool truncated = false; + std::string error; + bool closed = false; +}; + +class TtsSocket { + public: + TtsSocket(std::string api_key, std::string url = "", SocketOptions socket = {}); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + static Json build(const TtsSocketRequest& request); // validates + void generate(const TtsSocketRequest& request); + void cancel(const std::string& context_id); + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + // Reads events of `context_id` until done / flush_done / error / close or silence. + TtsSocketResult collect(const std::string& context_id, int max_wait_ms, bool stop_at_flush = true); + + static Vocabulary vocabulary(); + + private: + std::unique_ptr socket_; +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/voices.cpp b/src/vendors/audio/cartesia/voices.cpp new file mode 100644 index 0000000..1df4d51 --- /dev/null +++ b/src/vendors/audio/cartesia/voices.cpp @@ -0,0 +1,90 @@ +#include "vendors/audio/cartesia/voices.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::cartesia { + +using C = CartesiaCalls; + +Json CartesiaVoices::list(const Json& q) { + if (q.is_object()) { + const Json* l = q.find("limit"); + if (l && l->is_number() && (l->as_int() < 1 || l->as_int() > CartesiaCaps::kMaxListLimit)) + bad("limit must be 1..100"); + } + return call(C::kVoices, {}, nullptr, query_from(C::kVoices, q)); +} + +VoiceListing CartesiaVoices::list_all(std::size_t max_pages, int limit, const std::string& q) { + if (max_pages == 0 || max_pages > AudioCaps::kMaxPages) bad("max_pages must be 1.." + std::to_string(AudioCaps::kMaxPages)); + VoiceListing out; + Json query = Json::object(); + query.set("limit", Json(limit)); + if (!q.empty()) query.set("q", Json(q)); + for (std::size_t page = 0; page < max_pages; ++page) { + Json doc = list(query); + ++out.pages; + const Json* data = doc.find("data"); + std::string last_id; + if (data && data->is_array()) { + for (const Json& v : data->as_array()) { + if (out.voices.as_array().size() >= AudioCaps::kMaxListItems) { + out.truncated = true; + return out; + } + out.voices.push(v); + last_id = string_at(v, "id"); + } + } + const Json* more = doc.find("has_more"); + if (!more || !more->is_bool() || !more->as_bool() || last_id.empty()) return out; + query.set("starting_after", Json(last_id)); + } + out.truncated = true; + return out; +} + +Json CartesiaVoices::get(const std::string& id, bool expand) { + check_id("voice", id); + Query q; + if (expand) q.add("expand", "preview_file_url"); + return call(C::kVoice, {id}, nullptr, q); +} + +Json CartesiaVoices::accents() { return call(C::kAccents); } + +Json CartesiaVoices::clone(const std::string& clip, const std::string& filename, const std::string& content_type, + const std::string& name, const std::string& language, const std::string& description, + bool confirm) { + if (clip.empty() || clip.size() > CartesiaCaps::kMaxFileBytes) bad("clip must be 1 byte..100 MiB"); + require_text("name", name, 200); + require_text("language", language, 16); + require_text("description", description, 2000, false); + require_confirm("cloning a voice", confirm); + clients::body::MultipartBody form; + form.add_file("clip", filename.empty() ? "clip" : filename, content_type, clip); + form.add_text("name", name); + form.add_text("language", language); + if (!description.empty()) form.add_text("description", description); + return call_form(C::kVoiceClone, {}, form); +} + +Json CartesiaVoices::localize(const Json& body, bool confirm) { + check_body(table_.get(C::kVoiceLocalize), &body); + require_confirm("localizing a voice", confirm); + return call(C::kVoiceLocalize, {}, &body); +} + +Json CartesiaVoices::update(const std::string& id, const Json& body, bool confirm) { + check_id("voice", id); + check_body(table_.get(C::kVoiceUpdate), &body); + require_confirm("updating a voice", confirm); + return call(C::kVoiceUpdate, {id}, &body); +} + +Json CartesiaVoices::remove(const std::string& id, bool confirm) { + check_id("voice", id); + return call_destructive(C::kVoiceDelete, {id}, nullptr, Query(), confirm); +} + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/cartesia/voices.h b/src/vendors/audio/cartesia/voices.h new file mode 100644 index 0000000..bbe9f23 --- /dev/null +++ b/src/vendors/audio/cartesia/voices.h @@ -0,0 +1,35 @@ +#pragma once + +// Voices: list, get, accents (read side) and clone, localize, update, delete +// (writes: confirm required). + +#include + +#include "vendors/audio/cartesia/tts.h" + +namespace molo::audio::cartesia { + +struct VoiceListing { + Json voices = Json::array(); + bool truncated = false; // the page or item cap stopped the walk while more existed + std::size_t pages = 0; +}; + +class CartesiaVoices : public CartesiaService { + public: + using CartesiaService::CartesiaService; + Json list(const Json& query_object = Json()); + // Walks starting_after cursors (id of the last voice) up to max_pages. + VoiceListing list_all(std::size_t max_pages, int limit = 100, const std::string& q = ""); + Json get(const std::string& voice_id, bool expand_preview = false); + Json accents(); + // Cloning needs a clip (bytes), name and language; costs credits. + Json clone(const std::string& clip, const std::string& filename, const std::string& content_type, + const std::string& name, const std::string& language, const std::string& description, + bool confirm); + Json localize(const Json& body, bool confirm); + Json update(const std::string& voice_id, const Json& body, bool confirm); + Json remove(const std::string& voice_id, bool confirm); +}; + +} // namespace molo::audio::cartesia diff --git a/src/vendors/audio/deepgram/calls.cpp b/src/vendors/audio/deepgram/calls.cpp new file mode 100644 index 0000000..cdbc743 --- /dev/null +++ b/src/vendors/audio/deepgram/calls.cpp @@ -0,0 +1,70 @@ +#include "vendors/audio/deepgram/calls.h" + +namespace molo::audio::deepgram { + +// One row per REST call. tools/check_spec.py parses the block between the +// markers: {id, method, path, host, body, fields, required, query, response, source}. +// CALLS-BEGIN +static const CallSpec kRows[] = { + {"tokens_grant", "POST", "/v1/auth/grant", "api", "json", "ttl_seconds", "", "", "json", "openapi:tokens_grant"}, + {"media_transcribe", "POST", "/v1/listen", "api", "json", "url", "url", "callback,callback_method,extra,sentiment,summarize,tag,topics,custom_topic,custom_topic_mode,intents,custom_intent,custom_intent_mode,detect_entities,detect_language,diarize,diarize_model,dictation,encoding,filler_words,keyterm,keywords,language,language_hint,measurements,model,multichannel,numerals,paragraphs,profanity_filter,punctuate,redact,replace,search,smart_format,utterances,utt_split,version,mip_opt_out", "json", "openapi:media_transcribe"}, + {"models_list_3", "GET", "/v1/models", "api", "-", "", "", "include_outdated", "json", "openapi:models_list_3"}, + {"models_get_2", "GET", "/v1/models/{model_id}", "api", "-", "", "", "", "json", "openapi:models_get_2"}, + {"projects_list", "GET", "/v1/projects", "api", "-", "", "", "", "json", "openapi:projects_list"}, + {"projects_get", "GET", "/v1/projects/{project_id}", "api", "-", "", "", "limit,page", "json", "openapi:projects_get"}, + {"projects_update", "PATCH", "/v1/projects/{project_id}", "api", "json", "name", "", "", "json", "openapi:projects_update"}, + {"projects_delete", "DELETE", "/v1/projects/{project_id}", "api", "-", "", "", "", "json", "openapi:projects_delete"}, + {"variables_list", "GET", "/v1/projects/{project_id}/agent-variables", "api", "-", "", "", "", "json", "openapi:variables_list"}, + {"variables_create", "POST", "/v1/projects/{project_id}/agent-variables", "api", "json", "key,value,api_version", "key,value", "", "json", "openapi:variables_create"}, + {"variables_get", "GET", "/v1/projects/{project_id}/agent-variables/{variable_id}", "api", "-", "", "", "", "json", "openapi:variables_get"}, + {"variables_update", "PATCH", "/v1/projects/{project_id}/agent-variables/{variable_id}", "api", "json", "value", "value", "", "json", "openapi:variables_update"}, + {"variables_delete", "DELETE", "/v1/projects/{project_id}/agent-variables/{variable_id}", "api", "-", "", "", "", "json", "openapi:variables_delete"}, + {"configurations_list", "GET", "/v1/projects/{project_id}/agents", "api", "-", "", "", "", "json", "openapi:configurations_list"}, + {"configurations_create", "POST", "/v1/projects/{project_id}/agents", "api", "json", "config,metadata,api_version", "config", "", "json", "openapi:configurations_create"}, + {"configurations_get", "GET", "/v1/projects/{project_id}/agents/{agent_id}", "api", "-", "", "", "", "json", "openapi:configurations_get"}, + {"configurations_update", "PUT", "/v1/projects/{project_id}/agents/{agent_id}", "api", "json", "metadata", "metadata", "", "json", "openapi:configurations_update"}, + {"configurations_delete", "DELETE", "/v1/projects/{project_id}/agents/{agent_id}", "api", "-", "", "", "", "json", "openapi:configurations_delete"}, + {"balances_list", "GET", "/v1/projects/{project_id}/balances", "api", "-", "", "", "", "json", "openapi:balances_list"}, + {"balances_get", "GET", "/v1/projects/{project_id}/balances/{balance_id}", "api", "-", "", "", "", "json", "openapi:balances_get"}, + {"breakdown_list", "GET", "/v1/projects/{project_id}/billing/breakdown", "api", "-", "", "", "start,end,accessor,deployment,tag,line_item,grouping", "json", "openapi:breakdown_list"}, + {"fields_list_2", "GET", "/v1/projects/{project_id}/billing/fields", "api", "-", "", "", "start,end", "json", "openapi:fields_list_2"}, + {"invites_list", "GET", "/v1/projects/{project_id}/invites", "api", "-", "", "", "", "json", "openapi:invites_list"}, + {"invites_create", "POST", "/v1/projects/{project_id}/invites", "api", "json", "email,scope", "email,scope", "", "json", "openapi:invites_create"}, + {"invites_delete", "DELETE", "/v1/projects/{project_id}/invites/{email}", "api", "-", "", "", "", "json", "openapi:invites_delete"}, + {"keys_list", "GET", "/v1/projects/{project_id}/keys", "api", "-", "", "", "status", "json", "openapi:keys_list"}, + {"keys_create", "POST", "/v1/projects/{project_id}/keys", "api", "json", "*", "", "", "json", "openapi:keys_create"}, + {"keys_get", "GET", "/v1/projects/{project_id}/keys/{key_id}", "api", "-", "", "", "", "json", "openapi:keys_get"}, + {"keys_delete", "DELETE", "/v1/projects/{project_id}/keys/{key_id}", "api", "-", "", "", "", "json", "openapi:keys_delete"}, + {"projects_leave", "DELETE", "/v1/projects/{project_id}/leave", "api", "-", "", "", "", "json", "openapi:projects_leave"}, + {"members_list", "GET", "/v1/projects/{project_id}/members", "api", "-", "", "", "", "json", "openapi:members_list"}, + {"members_delete", "DELETE", "/v1/projects/{project_id}/members/{member_id}", "api", "-", "", "", "", "json", "openapi:members_delete"}, + {"scopes_list", "GET", "/v1/projects/{project_id}/members/{member_id}/scopes", "api", "-", "", "", "", "json", "openapi:scopes_list"}, + {"scopes_update", "PUT", "/v1/projects/{project_id}/members/{member_id}/scopes", "api", "json", "scope", "scope", "", "json", "openapi:scopes_update"}, + {"models_list_2", "GET", "/v1/projects/{project_id}/models", "api", "-", "", "", "include_outdated", "json", "openapi:models_list_2"}, + {"models_get", "GET", "/v1/projects/{project_id}/models/{model_id}", "api", "-", "", "", "", "json", "openapi:models_get"}, + {"purchases_list", "GET", "/v1/projects/{project_id}/purchases", "api", "-", "", "", "limit", "json", "openapi:purchases_list"}, + {"requests_list", "GET", "/v1/projects/{project_id}/requests", "api", "-", "", "", "start,end,limit,page,accessor,request_id,deployment,endpoint,method,status", "json", "openapi:requests_list"}, + {"requests_get", "GET", "/v1/projects/{project_id}/requests/{request_id}", "api", "-", "", "", "", "json", "openapi:requests_get"}, + {"distributionCredentials_list", "GET", "/v1/projects/{project_id}/self-hosted/distribution/credentials", "api", "-", "", "", "", "json", "openapi:distributionCredentials_list"}, + {"distributionCredentials_create", "POST", "/v1/projects/{project_id}/self-hosted/distribution/credentials", "api", "json", "comment", "", "scopes,provider", "json", "openapi:distributionCredentials_create"}, + {"distributionCredentials_get", "GET", "/v1/projects/{project_id}/self-hosted/distribution/credentials/{distribution_credentials_id}", "api", "-", "", "", "", "json", "openapi:distributionCredentials_get"}, + {"distributionCredentials_delete", "DELETE", "/v1/projects/{project_id}/self-hosted/distribution/credentials/{distribution_credentials_id}", "api", "-", "", "", "", "json", "openapi:distributionCredentials_delete"}, + {"usage_get", "GET", "/v1/projects/{project_id}/usage", "api", "-", "", "", "start,end,accessor,alternatives,callback_method,callback,channels,custom_intent_mode,custom_intent,custom_topic_mode,custom_topic,deployment,detect_entities,detect_language,diarize,dictation,encoding,endpoint,extra,filler_words,intents,keyterm,keywords,language,measurements,method,model,multichannel,numerals,paragraphs,profanity_filter,punctuate,redact,replace,sample_rate,search,sentiment,smart_format,summarize,tag,topics,utt_split,utterances,version", "json", "openapi:usage_get"}, + {"breakdown_get", "GET", "/v1/projects/{project_id}/usage/breakdown", "api", "-", "", "", "start,end,grouping,accessor,alternatives,callback_method,callback,channels,custom_intent_mode,custom_intent,custom_topic_mode,custom_topic,deployment,detect_entities,detect_language,diarize,dictation,encoding,endpoint,extra,filler_words,intents,keyterm,keywords,language,measurements,method,model,multichannel,numerals,paragraphs,profanity_filter,punctuate,redact,replace,sample_rate,search,sentiment,smart_format,summarize,tag,topics,utt_split,utterances,version", "json", "openapi:breakdown_get"}, + {"fields_list", "GET", "/v1/projects/{project_id}/usage/fields", "api", "-", "", "", "start,end", "json", "openapi:fields_list"}, + {"text_analyze", "POST", "/v1/read", "api", "json", "url,text", "", "callback,callback_method,sentiment,summarize,tag,topics,custom_topic,custom_topic_mode,intents,custom_intent,custom_intent_mode,language", "json", "openapi:text_analyze"}, + {"audio_generate", "POST", "/v1/speak", "api", "json", "text", "text", "callback,callback_method,mip_opt_out,tag,bit_rate,container,encoding,model,sample_rate,speed", "bytes", "openapi:audio_generate"}, + {"audio_generate_2", "POST", "/v2/speak", "api", "json", "text", "text", "callback,callback_method,mip_opt_out,tag,bit_rate,container,encoding,expressivity,model,sample_rate,speed,priority", "bytes", "openapi:audio_generate_2"}, + {"agent_think_models", "GET", "/v1/agent/settings/think/models", "agent", "-", "", "", "", "json", "openapi:models_list"}, + {"media_transcribe_bytes", "POST", "/v1/listen", "api", "bytes", "", "", "callback,callback_method,extra,sentiment,summarize,tag,topics,custom_topic,custom_topic_mode,intents,custom_intent,custom_intent_mode,detect_entities,detect_language,diarize,diarize_model,dictation,encoding,filler_words,keyterm,keywords,language,language_hint,measurements,model,multichannel,numerals,paragraphs,profanity_filter,punctuate,redact,replace,search,smart_format,utterances,utt_split,version,mip_opt_out", "json", "guide:listen-pre-recorded (raw audio body; the Fern OpenAPI lists only the JSON url form)"}, +}; +// CALLS-END + +static_assert(sizeof(kRows) / sizeof(kRows[0]) == 51, "deepgram call table size"); + +const CallTable& calls() { + static const CallTable table("deepgram", kRows, sizeof(kRows) / sizeof(kRows[0])); + return table; +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/calls.h b/src/vendors/audio/deepgram/calls.h new file mode 100644 index 0000000..f1c6cde --- /dev/null +++ b/src/vendors/audio/deepgram/calls.h @@ -0,0 +1,38 @@ +#pragma once + +// The Deepgram call table (https://github.com/deepgram/deepgram-api-specs +// openapi.yml, Fern generated). Services refer to rows by the ids below. + +#include "vendors/audio/call_spec.h" + +namespace molo::audio::deepgram { + +const CallTable& calls(); + +struct DeepgramCalls { + static constexpr const char* kListenUrl = "media_transcribe"; + static constexpr const char* kListenBytes = "media_transcribe_bytes"; + static constexpr const char* kSpeak = "audio_generate"; + static constexpr const char* kSpeakV2 = "audio_generate_2"; + static constexpr const char* kRead = "text_analyze"; + static constexpr const char* kModels = "models_list_3"; + static constexpr const char* kModel = "models_get_2"; + static constexpr const char* kAgentModels = "agent_think_models"; + static constexpr const char* kGrant = "tokens_grant"; + static constexpr const char* kProjects = "projects_list"; + static constexpr const char* kProject = "projects_get"; + static constexpr const char* kKeys = "keys_list"; + static constexpr const char* kKeyGet = "keys_get"; + static constexpr const char* kKeyCreate = "keys_create"; + static constexpr const char* kKeyDelete = "keys_delete"; + static constexpr const char* kMembers = "members_list"; + static constexpr const char* kBalances = "balances_list"; + static constexpr const char* kRequests = "requests_list"; + static constexpr const char* kRequest = "requests_get"; + static constexpr const char* kUsageBreakdown = "breakdown_get"; + static constexpr const char* kUsageFields = "fields_list"; + static constexpr const char* kBillingBreakdown = "breakdown_list"; + static constexpr const char* kPurchases = "purchases_list"; +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/client.cpp b/src/vendors/audio/deepgram/client.cpp new file mode 100644 index 0000000..7a92174 --- /dev/null +++ b/src/vendors/audio/deepgram/client.cpp @@ -0,0 +1,44 @@ +#include "vendors/audio/deepgram/client.h" + +#include "llm/config.h" + +namespace molo::audio::deepgram { + +std::string DeepgramEndpoints::base_url() { + return allowlisted_base(kBaseEnv, kDefaultBase, {"https://api.deepgram.com", "https://api.eu.deepgram.com"}); +} + +llm::ApiClientConfig DeepgramEndpoints::config(const std::string& vendor_base) { + llm::ApiClientConfig c; + c.vendor = kVendor; + c.base_url = vendor_base; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "Authorization"; + return c; +} + +std::shared_ptr DeepgramEndpoints::from_env() { + auto hosts = std::make_shared(); + hosts->add("api", HostFactory::from_env(config(base_url()), kKeyPrefix)); + hosts->add("agent", HostFactory::from_env(config(kAgentBase), kKeyPrefix)); + return hosts; +} + +std::shared_ptr DeepgramEndpoints::with(AudioHost api, AudioHost agent) { + auto hosts = std::make_shared(); + hosts->add("api", std::move(api)); + hosts->add("agent", std::move(agent)); + return hosts; +} + +std::string DeepgramEndpoints::key() { return llm::Config::get(kKeyEnv); } + +AudioSocket::Headers DeepgramEndpoints::ws_headers(const std::string& key) { + return {{"Authorization", std::string(kKeyPrefix) + key}}; +} + +std::string DeepgramEndpoints::ws_base() { return to_ws_url(base_url()); } +std::string DeepgramEndpoints::agent_ws_base() { return to_ws_url(kAgentBase); } + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/client.h b/src/vendors/audio/deepgram/client.h new file mode 100644 index 0000000..ee26408 --- /dev/null +++ b/src/vendors/audio/deepgram/client.h @@ -0,0 +1,38 @@ +#pragma once + +// Deepgram endpoints and credentials. +// REST https://api.deepgram.com Authorization: Token $DEEPGRAM_API_KEY +// Agent https://agent.deepgram.com same key (agent settings catalogue) +// WS wss://api.deepgram.com same header; the agent socket is wss://agent.deepgram.com +// DEEPGRAM_BASE_URL may select the EU API host (https://api.eu.deepgram.com). + +#include +#include + +#include "vendors/audio/hosts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::deepgram { + +struct DeepgramEndpoints { + static constexpr const char* kVendor = "deepgram"; + static constexpr const char* kKeyEnv = "DEEPGRAM_API_KEY"; + static constexpr const char* kBaseEnv = "DEEPGRAM_BASE_URL"; + static constexpr const char* kDefaultBase = "https://api.deepgram.com"; + static constexpr const char* kAgentBase = "https://agent.deepgram.com"; + // The API key authenticates as "Token "; a temporary token from + // /v1/auth/grant authenticates as "Bearer " and is out of scope here. + static constexpr const char* kKeyPrefix = "Token "; + + static std::string base_url(); + static llm::ApiClientConfig config(const std::string& vendor_base); + static std::shared_ptr from_env(); + // Injected hosts (tests): "api" and "agent". + static std::shared_ptr with(AudioHost api, AudioHost agent); + static std::string key(); + static AudioSocket::Headers ws_headers(const std::string& key); // Authorization: Token + static std::string ws_base(); // wss://api... + static std::string agent_ws_base(); // wss://agent... +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/listen.cpp b/src/vendors/audio/deepgram/listen.cpp new file mode 100644 index 0000000..3a875db --- /dev/null +++ b/src/vendors/audio/deepgram/listen.cpp @@ -0,0 +1,87 @@ +#include "vendors/audio/deepgram/listen.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::deepgram { + +using C = DeepgramCalls; + +namespace { +void tri(Query& q, const char* name, int v) { + if (v >= 0) q.add_bool(name, v == 1); +} +} // namespace + +Query Listen::query(const ListenOptions& o, const char* call_id) const { + Query q; + if (!o.model.empty()) { + if (!valid_token(o.model)) bad("model has characters Deepgram does not accept"); + q.add("model", o.model); + } + if (!o.language.empty()) { + if (!valid_token(o.language)) bad("bad language"); + q.add("language", o.language); + } + tri(q, "detect_language", o.detect_language); + tri(q, "punctuate", o.punctuate); + tri(q, "smart_format", o.smart_format); + tri(q, "diarize", o.diarize); + tri(q, "utterances", o.utterances); + tri(q, "paragraphs", o.paragraphs); + tri(q, "numerals", o.numerals); + tri(q, "profanity_filter", o.profanity_filter); + tri(q, "filler_words", o.filler_words); + tri(q, "multichannel", o.multichannel); + tri(q, "dictation", o.dictation); + tri(q, "sentiment", o.sentiment); + tri(q, "topics", o.topics); + tri(q, "intents", o.intents); + tri(q, "detect_entities", o.detect_entities); + tri(q, "mip_opt_out", o.mip_opt_out); + if (!o.summarize.empty()) { + if (o.summarize != "v2") bad("summarize must be v2"); + q.add("summarize", o.summarize); + } + if (!o.encoding.empty()) { + if (!valid_token(o.encoding)) bad("bad encoding"); + q.add("encoding", o.encoding); + } + struct L { const char* name; const std::vector* values; }; + for (const L& l : {L{"keywords", &o.keywords}, L{"keyterm", &o.keyterms}, L{"search", &o.search}, + L{"replace", &o.replace}, L{"redact", &o.redact}, L{"tag", &o.tags}}) { + if (l.values->size() > DeepgramCaps::kMaxTerms) bad(std::string("too many ") + l.name); + for (const auto& v : *l.values) { + require_text(l.name, v, DeepgramCaps::kMaxTermChars); + q.add(l.name, v); + } + } + q.add_if("version", o.version); + q.add_if("callback", o.callback); + q.add_if("callback_method", o.callback_method); + if (o.extra.is_object()) + for (const auto& [k, v] : o.extra.as_object()) + if (!q.add_json(k, v)) bad("extra '" + k + "' must be a scalar or a list of scalars"); + check_query(table_.get(call_id), q); + return q; +} + +Json Listen::transcribe_url(const std::string& url, const ListenOptions& o) { + if (url.rfind("https://", 0) != 0 && url.rfind("http://", 0) != 0) bad("url must be http or https"); + require_text("url", url, AudioCaps::kMaxFieldChars); + Json body = Json::object(); + body.set("url", Json(url)); + return call(C::kListenUrl, {}, &body, query(o, C::kListenUrl)); +} + +Json Listen::transcribe_bytes(const std::string& audio, const std::string& content_type, + const ListenOptions& o) { + if (audio.size() > DeepgramCaps::kMaxFileBytes) bad("audio is over the byte cap"); + llm::RawResponse raw = call_bytes(C::kListenBytes, {}, audio, content_type, query(o, C::kListenBytes)); + try { + return Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError("deepgram", 502, "BAD_RESPONSE", e.what()); + } +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/listen.h b/src/vendors/audio/deepgram/listen.h new file mode 100644 index 0000000..c5572ae --- /dev/null +++ b/src/vendors/audio/deepgram/listen.h @@ -0,0 +1,39 @@ +#pragma once + +// Pre-recorded speech to text: POST /v1/listen with a URL (JSON body) or the +// audio bytes as the body. Options become query parameters; every name is +// checked against the call table. + +#include +#include + +#include "vendors/audio/deepgram/service.h" + +namespace molo::audio::deepgram { + +struct ListenOptions { + std::string model; // "" = server default; token charset + std::string language; // BCP-47 + int detect_language = -1; // tri state: -1 unset, 0 false, 1 true + int punctuate = -1, smart_format = -1, diarize = -1, utterances = -1, paragraphs = -1; + int numerals = -1, profanity_filter = -1, filler_words = -1, multichannel = -1, dictation = -1; + int sentiment = -1, topics = -1, intents = -1, detect_entities = -1, mip_opt_out = -1; + std::string summarize; // "", "v2" + std::string encoding; // headerless raw audio only + std::vector keywords, keyterms, search, replace, redact, tags; + std::string version, callback, callback_method; + Json extra; // any other query parameter in the table: object of scalars / lists +}; + +class Listen : public DeepgramService { + public: + using DeepgramService::DeepgramService; + Query query(const ListenOptions& options, const char* call_id) const; + // Remote media by URL. + Json transcribe_url(const std::string& url, const ListenOptions& options = {}); + // Audio bytes as the body; content_type like audio/wav, audio/mpeg, audio/*. + Json transcribe_bytes(const std::string& audio, const std::string& content_type, + const ListenOptions& options = {}); +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/manage.cpp b/src/vendors/audio/deepgram/manage.cpp new file mode 100644 index 0000000..f5d25a8 --- /dev/null +++ b/src/vendors/audio/deepgram/manage.cpp @@ -0,0 +1,100 @@ +#include "vendors/audio/deepgram/manage.h" + +namespace molo::audio::deepgram { + +using C = DeepgramCalls; + +Json Manage::projects() { return call(C::kProjects); } + +Json Manage::project(const std::string& id, const Json& q) { + check_id("project", id); + return call(C::kProject, {id}, nullptr, query_from(C::kProject, q)); +} + +Json Manage::models(bool outdated) { + Query q; + if (outdated) q.add_bool("include_outdated", true); + return call(C::kModels, {}, nullptr, q); +} + +Json Manage::model(const std::string& id) { + check_id("model", id); + return call(C::kModel, {id}); +} + +Json Manage::project_models(const std::string& id, bool outdated) { + check_id("project", id); + Query q; + if (outdated) q.add_bool("include_outdated", true); + return call("models_list_2", {id}, nullptr, q); +} + +Json Manage::agent_think_models() { return call(C::kAgentModels); } + +Json Manage::keys(const std::string& id, const std::string& status) { + check_id("project", id); + Query q; + if (!status.empty()) { + if (status != "active" && status != "expired") bad("status must be active or expired"); + q.add("status", status); + } + return call(C::kKeys, {id}, nullptr, q); +} + +Json Manage::key(const std::string& id, const std::string& key_id) { + check_id("project", id); + check_id("key", key_id); + return call(C::kKeyGet, {id, key_id}); +} + +Json Manage::create_key(const std::string& id, const Json& body, bool confirm) { + check_id("project", id); + check_body(table_.get(C::kKeyCreate), &body); + require_confirm("creating an API key", confirm); + return call(C::kKeyCreate, {id}, &body); +} + +Json Manage::delete_key(const std::string& id, const std::string& key_id, bool confirm) { + check_id("project", id); + check_id("key", key_id); + return call_destructive(C::kKeyDelete, {id, key_id}, nullptr, Query(), confirm); +} + +Json Manage::members(const std::string& id) { + check_id("project", id); + return call(C::kMembers, {id}); +} + +Json Manage::balances(const std::string& id) { + check_id("project", id); + return call(C::kBalances, {id}); +} + +Json Manage::requests(const std::string& id, const Json& q) { + check_id("project", id); + return call(C::kRequests, {id}, nullptr, query_from(C::kRequests, q)); +} + +Json Manage::usage_breakdown(const std::string& id, const Json& q) { + check_id("project", id); + return call(C::kUsageBreakdown, {id}, nullptr, query_from(C::kUsageBreakdown, q)); +} + +Json Manage::usage_fields(const std::string& id, const Json& q) { + check_id("project", id); + return call(C::kUsageFields, {id}, nullptr, query_from(C::kUsageFields, q)); +} + +Json Manage::billing_breakdown(const std::string& id, const Json& q) { + check_id("project", id); + return call(C::kBillingBreakdown, {id}, nullptr, query_from(C::kBillingBreakdown, q)); +} + +Json Manage::grant_token(int ttl) { + if (ttl < 1 || ttl > 3600) bad("ttl_seconds must be 1..3600"); + Json body = Json::object(); + body.set("ttl_seconds", Json(ttl)); + return call(C::kGrant, {}, &body); +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/manage.h b/src/vendors/audio/deepgram/manage.h new file mode 100644 index 0000000..6d08f59 --- /dev/null +++ b/src/vendors/audio/deepgram/manage.h @@ -0,0 +1,40 @@ +#pragma once + +// Management API: projects, keys, members, balances, requests, usage, +// billing, model catalogue, temporary tokens. Reads are plain calls; key +// creation and every delete need confirm. Further rows (invites, scopes, +// self-hosted credentials, agent variables and configurations) are in the +// call table and reached with call() / call_destructive(). + +#include + +#include "vendors/audio/deepgram/service.h" + +namespace molo::audio::deepgram { + +class Manage : public DeepgramService { + public: + using DeepgramService::DeepgramService; + + Json projects(); + Json project(const std::string& project_id, const Json& query_object = Json()); + Json models(bool include_outdated = false); + Json model(const std::string& model_id); + Json project_models(const std::string& project_id, bool include_outdated = false); + Json agent_think_models(); + Json keys(const std::string& project_id, const std::string& status = ""); + Json key(const std::string& project_id, const std::string& key_id); + // Creates a key (the secret is returned once): a write, confirm required. + Json create_key(const std::string& project_id, const Json& body, bool confirm); + Json delete_key(const std::string& project_id, const std::string& key_id, bool confirm); + Json members(const std::string& project_id); + Json balances(const std::string& project_id); + Json requests(const std::string& project_id, const Json& query_object = Json()); + Json usage_breakdown(const std::string& project_id, const Json& query_object = Json()); + Json usage_fields(const std::string& project_id, const Json& query_object = Json()); + Json billing_breakdown(const std::string& project_id, const Json& query_object = Json()); + // Short lived access token (JWT) for browser use; ttl 1..3600 seconds. + Json grant_token(int ttl_seconds); +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/read.cpp b/src/vendors/audio/deepgram/read.cpp new file mode 100644 index 0000000..a1ee2f4 --- /dev/null +++ b/src/vendors/audio/deepgram/read.cpp @@ -0,0 +1,53 @@ +#include "vendors/audio/deepgram/read.h" + +namespace molo::audio::deepgram { + +using C = DeepgramCalls; + +Json Read::build(const ReadRequest& r) const { + if (r.text.empty() == r.url.empty()) bad("give exactly one of text and url"); + Json body = Json::object(); + if (!r.text.empty()) { + require_text("text", r.text, DeepgramCaps::kMaxReadChars); + body.set("text", Json(r.text)); + } else { + if (r.url.rfind("https://", 0) != 0 && r.url.rfind("http://", 0) != 0) bad("url must be http or https"); + require_text("url", r.url, 8192); + body.set("url", Json(r.url)); + } + return body; +} + +Query Read::query(const ReadRequest& r) const { + Query q; + if (!r.language.empty()) { + if (!valid_token(r.language)) bad("bad language"); + q.add("language", r.language); + } + if (!r.summarize.empty()) { + if (r.summarize != "v2") bad("summarize must be v2"); + q.add("summarize", r.summarize); + } + if (r.topics >= 0) q.add_bool("topics", r.topics == 1); + if (r.intents >= 0) q.add_bool("intents", r.intents == 1); + if (r.sentiment >= 0) q.add_bool("sentiment", r.sentiment == 1); + if (r.custom_topics.size() > DeepgramCaps::kMaxTerms || r.custom_intents.size() > DeepgramCaps::kMaxTerms) + bad("too many custom topics or intents"); + for (const auto& t : r.custom_topics) { require_text("custom_topic", t, DeepgramCaps::kMaxTermChars); q.add("custom_topic", t); } + for (const auto& t : r.custom_intents) { require_text("custom_intent", t, DeepgramCaps::kMaxTermChars); q.add("custom_intent", t); } + for (const auto& [name, mode] : {std::pair{"custom_topic_mode", &r.custom_topic_mode}, + {"custom_intent_mode", &r.custom_intent_mode}}) { + if (mode->empty()) continue; + if (*mode != "extended" && *mode != "strict") bad(std::string(name) + " must be extended or strict"); + q.add(name, *mode); + } + check_query(table_.get(C::kRead), q); + return q; +} + +Json Read::analyze(const ReadRequest& r) { + Json body = build(r); + return call(C::kRead, {}, &body, query(r)); +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/read.h b/src/vendors/audio/deepgram/read.h new file mode 100644 index 0000000..0c82f1a --- /dev/null +++ b/src/vendors/audio/deepgram/read.h @@ -0,0 +1,30 @@ +#pragma once + +// Text intelligence: POST /v1/read (summaries, topics, intents, sentiment). + +#include +#include + +#include "vendors/audio/deepgram/service.h" + +namespace molo::audio::deepgram { + +struct ReadRequest { + std::string text; // exactly one of text and url + std::string url; + std::string language = "en"; // the API supports English only + std::string summarize; // "", v2 + int topics = -1, intents = -1, sentiment = -1; + std::vector custom_topics, custom_intents; + std::string custom_topic_mode, custom_intent_mode; // "", extended, strict +}; + +class Read : public DeepgramService { + public: + using DeepgramService::DeepgramService; + Json build(const ReadRequest& request) const; + Query query(const ReadRequest& request) const; + Json analyze(const ReadRequest& request); +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/service.cpp b/src/vendors/audio/deepgram/service.cpp new file mode 100644 index 0000000..c2eb148 --- /dev/null +++ b/src/vendors/audio/deepgram/service.cpp @@ -0,0 +1,14 @@ +#include "vendors/audio/deepgram/service.h" + +namespace molo::audio::deepgram { + +bool valid_token(const std::string& s) { + if (s.empty() || s.size() > 128) return false; + for (char c : s) + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || + c == '_' || c == '.' || c == ':')) + return false; + return true; +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/service.h b/src/vendors/audio/deepgram/service.h new file mode 100644 index 0000000..804edda --- /dev/null +++ b/src/vendors/audio/deepgram/service.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "vendors/audio/audio_service.h" +#include "vendors/audio/deepgram/calls.h" +#include "vendors/audio/deepgram/client.h" + +namespace molo::audio::deepgram { + +struct DeepgramCaps { + static constexpr std::size_t kMaxSpeakChars = 2000; // Deepgram TTS REST input, per the TTS guide + static constexpr std::size_t kMaxReadChars = 100000; // our ceiling for /v1/read text + static constexpr std::size_t kMaxFileBytes = 128u * 1024u * 1024u; // our ceiling (provider allows 2 GB) + static constexpr std::size_t kMaxTerms = 100; // keywords / keyterm / search / replace / redact entries + static constexpr std::size_t kMaxTermChars = 100; +}; + +class DeepgramService : public AudioService { + public: + explicit DeepgramService(std::shared_ptr hosts) + : AudioService(calls(), std::move(hosts)) {} + DeepgramService() : DeepgramService(DeepgramEndpoints::from_env()) {} +}; + +// A model name or other token Deepgram puts in a query: [A-Za-z0-9._:-]{1,128}. +bool valid_token(const std::string& s); + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/speak.cpp b/src/vendors/audio/deepgram/speak.cpp new file mode 100644 index 0000000..0ae84ef --- /dev/null +++ b/src/vendors/audio/deepgram/speak.cpp @@ -0,0 +1,59 @@ +#include "vendors/audio/deepgram/speak.h" + +namespace molo::audio::deepgram { + +using C = DeepgramCalls; + +Json Speak::build(const SpeakOptions& o) const { + require_text("text", o.text, DeepgramCaps::kMaxSpeakChars); + Json body = Json::object(); + body.set("text", Json(o.text)); + return body; +} + +Query Speak::query(const SpeakOptions& o, bool v2) const { + Query q; + if (!o.model.empty()) { + if (!valid_token(o.model)) bad("model has characters Deepgram does not accept"); + q.add("model", o.model); + } + if (!o.encoding.empty()) { + if (!valid_token(o.encoding)) bad("bad encoding"); + q.add("encoding", o.encoding); + } + if (!o.container.empty()) { + if (!valid_token(o.container)) bad("bad container"); + q.add("container", o.container); + } + if (o.sample_rate < 0 || o.sample_rate > 96000) bad("sample_rate must be 0 or up to 96000"); + if (o.sample_rate > 0) q.add_int("sample_rate", o.sample_rate); + if (o.bit_rate < 0 || o.bit_rate > 1000000) bad("bit_rate is out of range"); + if (o.bit_rate > 0) q.add_int("bit_rate", o.bit_rate); + if (o.speed < 0) bad("speed must not be negative"); + if (o.speed > 0) q.add("speed", std::to_string(o.speed)); + if (v2) { + q.add_if("expressivity", o.expressivity); + q.add_if("priority", o.priority); + } else if (!o.expressivity.empty() || !o.priority.empty()) { + bad("expressivity and priority belong to /v2/speak"); + } + q.add_if("tag", o.tag); + q.add_if("callback", o.callback); + q.add_if("callback_method", o.callback_method); + if (o.mip_opt_out >= 0) q.add_bool("mip_opt_out", o.mip_opt_out == 1); + check_query(table_.get(v2 ? C::kSpeakV2 : C::kSpeak), q); + return q; +} + +SpeakResult Speak::synthesize(const SpeakOptions& o, bool v2) { + Json body = build(o); + llm::RawResponse raw = call_raw(v2 ? C::kSpeakV2 : C::kSpeak, {}, &body, query(o, v2)); + return SpeakResult{raw.body, raw.content_type}; +} + +ByteStreamer::Result Speak::stream(const SpeakOptions& o, const ChunkSink& sink, bool v2) { + Json body = build(o); + return call_stream(v2 ? C::kSpeakV2 : C::kSpeak, {}, &body, query(o, v2), sink); +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/speak.h b/src/vendors/audio/deepgram/speak.h new file mode 100644 index 0000000..9c3d669 --- /dev/null +++ b/src/vendors/audio/deepgram/speak.h @@ -0,0 +1,40 @@ +#pragma once + +// Text to speech: POST /v1/speak (Aura) and POST /v2/speak (Flux TTS). +// The response body is the audio. + +#include + +#include "vendors/audio/deepgram/service.h" + +namespace molo::audio::deepgram { + +struct SpeakOptions { + std::string text; // 1..2000 characters + std::string model; // "" = server default, e.g. aura-2-thalia-en + std::string encoding; // linear16 mulaw alaw mp3 opus flac aac + std::string container; // wav ogg none + int sample_rate = 0; // 0 unset + int bit_rate = 0; // 0 unset + double speed = 0; // 0 unset (v1/v2 speed query parameter) + std::string expressivity; // v2 only + std::string priority; // v2 only + std::string tag, callback, callback_method; + int mip_opt_out = -1; +}; + +struct SpeakResult { + std::string audio; + std::string content_type; +}; + +class Speak : public DeepgramService { + public: + using DeepgramService::DeepgramService; + Json build(const SpeakOptions& options) const; + Query query(const SpeakOptions& options, bool v2) const; + SpeakResult synthesize(const SpeakOptions& options, bool v2 = false); + ByteStreamer::Result stream(const SpeakOptions& options, const ChunkSink& sink, bool v2 = false); +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/speech.cpp b/src/vendors/audio/deepgram/speech.cpp new file mode 100644 index 0000000..2ade0b5 --- /dev/null +++ b/src/vendors/audio/deepgram/speech.cpp @@ -0,0 +1,103 @@ +#include "vendors/audio/deepgram/speech.h" + +namespace molo::audio::deepgram { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("deepgram", why); } +} // namespace + +DeepgramSynthesizer::DeepgramSynthesizer(std::shared_ptr hosts) : speak_(hosts), manage_(hosts) {} + +SpeakOptions DeepgramSynthesizer::map(const SpeechRequest& r) const { + validate(r); + SpeakOptions o; + o.text = r.text; + o.model = !r.voice.empty() ? r.voice : r.model; // Aura voices are models: aura-2-thalia-en + if (r.format == "mp3") { + o.encoding = "mp3"; + if (r.sample_rate != 0) invalid("mp3 has a fixed sample rate"); + } else { + o.encoding = "linear16"; + o.container = r.format == "wav" ? "wav" : "none"; + if (r.sample_rate != 0) { + bool ok = false; + for (int v : {8000, 16000, 24000, 32000, 48000}) ok = ok || v == r.sample_rate; + if (!ok) invalid("linear16 sample_rate must be 8000, 16000, 24000, 32000 or 48000"); + o.sample_rate = r.sample_rate; + } + } + if (r.speed > 0) o.speed = r.speed; + return o; +} + +SpeechResult DeepgramSynthesizer::synthesize(const SpeechRequest& r) { + SpeakResult res = speak_.synthesize(map(r)); + SpeechResult out; + out.audio = std::move(res.audio); + out.content_type = res.content_type.empty() ? content_type_for(r.format) : res.content_type; + out.format = r.format; + out.provider = name(); + return out; +} + +Json DeepgramSynthesizer::voices() { return manage_.models(); } + +DeepgramRecognizer::DeepgramRecognizer(std::shared_ptr hosts) : listen_(std::move(hosts)) {} + +ListenOptions DeepgramRecognizer::map(const TranscribeRequest& r) const { + validate(r); + ListenOptions o; + o.model = r.model; + o.language = r.language; + if (r.language.empty()) o.detect_language = 1; + o.punctuate = 1; + o.smart_format = 1; + o.utterances = 1; + if (r.diarize) o.diarize = 1; + o.keyterms = r.keywords; + return o; +} + +Transcript DeepgramRecognizer::fold(const Json& doc) { + Transcript t; + t.provider = "deepgram"; + t.raw = doc; + if (!doc.is_object()) return t; + if (const Json* meta = doc.find("metadata"); meta && meta->is_object()) { + if (const Json* d = meta->find("duration"); d && d->is_number()) t.duration = d->as_number(); + } + const Json* results = doc.find("results"); + const Json* channels = results && results->is_object() ? results->find("channels") : nullptr; + if (!channels || !channels->is_array() || channels->as_array().empty()) return t; + const Json& ch = channels->as_array()[0]; + if (const Json* l = ch.find("detected_language"); l && l->is_string()) t.language = l->as_string(); + const Json* alts = ch.find("alternatives"); + if (!alts || !alts->is_array() || alts->as_array().empty()) return t; + const Json& alt = alts->as_array()[0]; + if (const Json* x = alt.find("transcript"); x && x->is_string()) t.text = x->as_string(); + if (const Json* words = alt.find("words"); words && words->is_array()) { + for (const Json& w : words->as_array()) { + if (!w.is_object()) continue; + TranscriptWord word; + const Json* pw = w.find("punctuated_word"); + const Json* raw = w.find("word"); + if (pw && pw->is_string()) word.text = pw->as_string(); + else if (raw && raw->is_string()) word.text = raw->as_string(); + if (const Json* x = w.find("start"); x && x->is_number()) word.start = x->as_number(); + if (const Json* x = w.find("end"); x && x->is_number()) word.end = x->as_number(); + if (const Json* x = w.find("confidence"); x && x->is_number()) word.confidence = x->as_number(); + if (const Json* x = w.find("speaker"); x && x->is_number()) word.speaker = std::to_string(x->as_int()); + t.words.push_back(word); + } + } + return t; +} + +Transcript DeepgramRecognizer::transcribe(const TranscribeRequest& r) { + ListenOptions o = map(r); + Json doc = !r.url.empty() ? listen_.transcribe_url(r.url, o) + : listen_.transcribe_bytes(r.audio, r.content_type, o); + return fold(doc); +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/speech.h b/src/vendors/audio/deepgram/speech.h new file mode 100644 index 0000000..f33f70d --- /dev/null +++ b/src/vendors/audio/deepgram/speech.h @@ -0,0 +1,43 @@ +#pragma once + +// Deepgram behind the neutral speech seam. + +#include + +#include "vendors/audio/deepgram/listen.h" +#include "vendors/audio/deepgram/manage.h" +#include "vendors/audio/deepgram/speak.h" +#include "vendors/audio/speech.h" + +namespace molo::audio::deepgram { + +class DeepgramSynthesizer : public SpeechSynthesizer { + public: + explicit DeepgramSynthesizer(std::shared_ptr hosts = DeepgramEndpoints::from_env()); + std::string name() const override { return "deepgram"; } + bool configured() const override { return speak_.configured(); } + std::size_t max_chars() const override { return DeepgramCaps::kMaxSpeakChars; } + std::vector formats() const override { return {"mp3", "wav", "pcm"}; } + SpeechResult synthesize(const SpeechRequest& request) override; + Json voices() override; // the model catalogue (tts section) + SpeakOptions map(const SpeechRequest& request) const; + + private: + Speak speak_; + Manage manage_; +}; + +class DeepgramRecognizer : public SpeechRecognizer { + public: + explicit DeepgramRecognizer(std::shared_ptr hosts = DeepgramEndpoints::from_env()); + std::string name() const override { return "deepgram"; } + bool configured() const override { return listen_.configured(); } + Transcript transcribe(const TranscribeRequest& request) override; + ListenOptions map(const TranscribeRequest& request) const; + static Transcript fold(const Json& doc); + + private: + Listen listen_; +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/streams.cpp b/src/vendors/audio/deepgram/streams.cpp new file mode 100644 index 0000000..538bbfa --- /dev/null +++ b/src/vendors/audio/deepgram/streams.cpp @@ -0,0 +1,325 @@ +#include "vendors/audio/deepgram/streams.h" + +namespace molo::audio::deepgram { + +namespace { + +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("deepgram", why); } + +std::string with_params(const std::string& base, const StreamParams& p, const std::vector& allowed) { + Query q; + for (const auto& [k, v] : p.pairs) { + bool ok = false; + for (const auto& a : allowed) ok = ok || a == k; + if (!ok) invalid("unknown stream parameter '" + k + "'"); + if (v.size() > 256) invalid("stream parameter value is too long"); + q.add(k, v); + } + std::string s = q.str(); + return s.empty() ? base : base + s; +} + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} + +double num_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_number() ? v->as_number() : 0.0; +} + +bool bool_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_bool() && v->as_bool(); +} + +Json typed(const char* type) { + Json m = Json::object(); + m.set("type", Json(type)); + return m; +} + +} // namespace + +const std::vector& StreamChannels::listen_v1_params() { + static const std::vector v = { + "callback", "callback_method", "channels", "detect_entities", "diarize", "diarize_model", "dictation", + "encoding", "endpointing", "extra", "interim_results", "keyterm", "keywords", "language", + "language_hint", "mip_opt_out", "model", "multichannel", "numerals", "profanity_filter", "punctuate", + "redact", "replace", "sample_rate", "search", "smart_format", "tag", "utterance_end_ms", "vad_events", + "version"}; + return v; +} +const std::vector& StreamChannels::listen_v2_params() { + static const std::vector v = { + "model", "encoding", "sample_rate", "eager_eot_threshold", "eot_threshold", "eot_timeout_ms", + "keyterm", "language_hint", "profanity_filter", "numerals", "redact", "mip_opt_out", "tag"}; + return v; +} +const std::vector& StreamChannels::speak_v1_params() { + static const std::vector v = {"encoding", "mip_opt_out", "model", "sample_rate", "speed"}; + return v; +} +const std::vector& StreamChannels::speak_v2_params() { + static const std::vector v = {"model", "encoding", "sample_rate", "speed", "expressivity", + "mip_opt_out", "tag"}; + return v; +} + +Vocabulary StreamChannels::listen_v1() { + return Vocabulary("type", {"KeepAlive", "Finalize", "CloseStream", "Configure"}, + {"Results", "Metadata", "UtteranceEnd", "SpeechStarted", "ConfigureSuccess", + "ConfigureFailure", "Error"}); +} +Vocabulary StreamChannels::listen_v2() { + return Vocabulary("type", {"CloseStream", "ForceEndTurn", "Configure"}, + {"Connected", "TurnInfo", "ConfigureSuccess", "ConfigureFailure", "Warning", "Error"}); +} +Vocabulary StreamChannels::speak_v1() { + return Vocabulary("type", {"Speak", "Flush", "Clear", "Close"}, + {"Metadata", "Flushed", "Cleared", "Warning"}); +} +Vocabulary StreamChannels::speak_v2() { + return Vocabulary("type", {"Speak", "Flush", "Interrupt", "Configure", "Close"}, + {"Connected", "SpeechStarted", "SpeechMetadata", "SpeechInterrupted", "Flushed", + "SessionMetadata", "ConfigureSuccess", "ConfigureFailure", "Warning", "Error"}); +} +Vocabulary StreamChannels::agent() { + return Vocabulary( + "type", + {"Settings", "UpdateListen", "UpdateThink", "UpdateSpeak", "InjectUserMessage", "InjectAgentMessage", + "FunctionCallResponse", "KeepAlive", "UpdatePrompt", "ForceEndTurn"}, + {"Welcome", "SettingsApplied", "ConversationText", "UserStartedSpeaking", "AgentThinking", + "FunctionCallRequest", "FunctionCallResponse", "FunctionCallCancelled", "AgentStartedSpeaking", + "AgentAudioDone", "Error", "Warning", "History", "PromptUpdated", "SpeakUpdated", "ThinkUpdated", + "ListenUpdated", "InjectionRefused", "LatencyReport"}); +} + +// ---- listen --------------------------------------------------------------- + +ListenStream::ListenStream(std::string api_key, StreamParams params, bool flux, std::string url, + SocketOptions socket) + : flux_(flux) { + std::string base = url.empty() ? DeepgramEndpoints::ws_base() + (flux ? "/v2/listen" : "/v1/listen") : url; + std::string full = with_params(base, params, flux ? StreamChannels::listen_v2_params() : StreamChannels::listen_v1_params()); + socket_ = std::make_unique("deepgram", full, DeepgramEndpoints::ws_headers(api_key), + flux ? StreamChannels::listen_v2() : StreamChannels::listen_v1(), + std::move(socket)); +} + +void ListenStream::keep_alive() { socket_->send_json(typed("KeepAlive")); } +void ListenStream::finalize() { + if (flux_) invalid("Finalize belongs to /v1/listen; use force_end_turn on /v2/listen"); + socket_->send_json(typed("Finalize")); +} +void ListenStream::force_end_turn() { + if (!flux_) invalid("ForceEndTurn belongs to /v2/listen; use finalize on /v1/listen"); + socket_->send_json(typed("ForceEndTurn")); +} +void ListenStream::close_stream() { socket_->send_json(typed("CloseStream")); } + +ListenEvent ListenStream::decode(const SocketEvent& ev) { + ListenEvent out; + if (ev.kind == SocketEvent::Kind::kClose) { + out.kind = ListenEvent::Kind::kClose; + return out; + } + if (ev.kind != SocketEvent::Kind::kText || !ev.json_ok) return out; + out.raw = ev.json; + const Json& j = ev.json; + if (ev.type == "Results") { + out.kind = ListenEvent::Kind::kResults; + out.is_final = bool_at(j, "is_final"); + out.speech_final = bool_at(j, "speech_final"); + if (const Json* ch = j.find("channel"); ch && ch->is_object()) { + if (const Json* alts = ch->find("alternatives"); alts && alts->is_array() && !alts->as_array().empty()) { + out.transcript = str_at(alts->as_array()[0], "transcript"); + out.confidence = num_at(alts->as_array()[0], "confidence"); + } + } + } else if (ev.type == "Metadata") { + out.kind = ListenEvent::Kind::kMetadata; + out.request_id = str_at(j, "request_id"); + } else if (ev.type == "UtteranceEnd") { + out.kind = ListenEvent::Kind::kUtteranceEnd; + } else if (ev.type == "SpeechStarted") { + out.kind = ListenEvent::Kind::kSpeechStarted; + } else if (ev.type == "TurnInfo") { + out.kind = ListenEvent::Kind::kTurn; + out.turn_event = str_at(j, "event"); + out.transcript = str_at(j, "transcript"); + out.confidence = num_at(j, "end_of_turn_confidence"); + out.is_final = out.turn_event == "EndOfTurn"; + } else if (ev.type == "Error" || ev.type == "ConfigureFailure") { + out.kind = ListenEvent::Kind::kError; + } + return out; +} + +ListenResult ListenStream::collect(int max_wait_ms) { + ListenResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + ++out.events; + ListenEvent d = decode(*ev); + switch (d.kind) { + case ListenEvent::Kind::kResults: + case ListenEvent::Kind::kTurn: + if (d.is_final) { + if (!d.transcript.empty()) { + out.finals.push_back(d.transcript); + out.transcript += (out.transcript.empty() ? "" : " ") + d.transcript; + } + out.interim.clear(); + } else { + out.interim = d.transcript; + } + break; + case ListenEvent::Kind::kUtteranceEnd: ++out.utterance_ends; break; + case ListenEvent::Kind::kMetadata: + out.request_id = d.request_id; + out.duration = num_at(d.raw, "duration"); + return out; + case ListenEvent::Kind::kError: + out.error = str_at(d.raw, "description"); + if (out.error.empty()) out.error = str_at(d.raw, "message"); + if (out.error.empty()) out.error = "stream error"; + return out; + case ListenEvent::Kind::kClose: out.closed = true; return out; + default: break; + } + } + return out; +} + +// ---- speak ---------------------------------------------------------------- + +SpeakStream::SpeakStream(std::string api_key, StreamParams params, bool v2, std::string url, + SocketOptions socket) + : v2_(v2) { + std::string base = url.empty() ? DeepgramEndpoints::ws_base() + (v2 ? "/v2/speak" : "/v1/speak") : url; + std::string full = with_params(base, params, v2 ? StreamChannels::speak_v2_params() : StreamChannels::speak_v1_params()); + socket_ = std::make_unique("deepgram", full, DeepgramEndpoints::ws_headers(api_key), + v2 ? StreamChannels::speak_v2() : StreamChannels::speak_v1(), + std::move(socket)); +} + +void SpeakStream::speak(const std::string& text) { + if (text.empty() || text.size() > DeepgramCaps::kMaxSpeakChars) invalid("text must be 1..2000 characters"); + if (text.find('\0') != std::string::npos) invalid("text has a NUL byte"); + Json m = typed("Speak"); + m.set("text", Json(text)); + socket_->send_json(m); +} +void SpeakStream::flush() { socket_->send_json(typed("Flush")); } +void SpeakStream::clear() { + if (v2_) invalid("Clear belongs to /v1/speak; use interrupt on /v2/speak"); + socket_->send_json(typed("Clear")); +} +void SpeakStream::interrupt(double playback_offset) { + if (!v2_) invalid("Interrupt belongs to /v2/speak; use clear on /v1/speak"); + if (playback_offset < 0 || playback_offset > 36000) invalid("playback_offset must be 0..36000"); + Json m = typed("Interrupt"); + m.set("playback_offset", Json(playback_offset)); + socket_->send_json(m); +} +void SpeakStream::end() { socket_->send_json(typed("Close")); } + +SpeakStreamResult SpeakStream::collect(int max_wait_ms) { + SpeakStreamResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + if (ev->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + if (ev->kind == SocketEvent::Kind::kBinary) { + if (out.audio.size() + ev->bytes.size() > AudioCaps::kMaxStreamAudioBytes) { + out.truncated = true; + return out; + } + out.audio += ev->bytes; + ++out.chunks; + continue; + } + if (!ev->json_ok) continue; + out.metadata.push_back(ev->json); + if (ev->type == "Flushed") { + out.flushed = true; + return out; + } + if (ev->type == "Cleared") { + out.cleared = true; + return out; + } + if (ev->type == "Warning") out.warning = str_at(ev->json, "description"); + if (ev->type == "Error") { + out.error = str_at(ev->json, "description"); + if (out.error.empty()) out.error = "stream error"; + return out; + } + } + return out; +} + +// ---- agent ---------------------------------------------------------------- + +AgentSession::AgentSession(std::string api_key, std::string url, SocketOptions socket) { + std::string full = url.empty() ? DeepgramEndpoints::agent_ws_base() + "/v1/agent/converse" : url; + socket_ = std::make_unique("deepgram", full, DeepgramEndpoints::ws_headers(api_key), + StreamChannels::agent(), std::move(socket)); +} + +void AgentSession::send_settings(const Json& settings) { + if (!settings.is_object()) invalid("settings must be a JSON object"); + Json m = settings; + m.set("type", Json("Settings")); + socket_->send_json(m); +} + +void AgentSession::inject_user_message(const std::string& content) { + if (content.empty() || content.size() > 20000) invalid("content must be 1..20000 characters"); + Json m = typed("InjectUserMessage"); + m.set("content", Json(content)); + socket_->send_json(m); +} + +void AgentSession::inject_agent_message(const std::string& message) { + if (message.empty() || message.size() > 20000) invalid("message must be 1..20000 characters"); + Json m = typed("InjectAgentMessage"); + m.set("message", Json(message)); + socket_->send_json(m); +} + +void AgentSession::keep_alive() { socket_->send_json(typed("KeepAlive")); } + +void AgentSession::send_function_result(const std::string& id, const std::string& name, const std::string& content) { + if (!llm::Service::valid_id(id)) invalid("bad function call id"); + if (name.empty() || name.size() > 256) invalid("name must be 1..256 characters"); + if (content.size() > 200000) invalid("content is over 200000 characters"); + Json m = typed("FunctionCallResponse"); + m.set("id", Json(id)); + m.set("name", Json(name)); + m.set("content", Json(content)); + socket_->send_json(m); +} + +std::optional AgentSession::next(int wait_ms) { + auto ev = socket_->next(wait_ms); + if (!ev) return std::nullopt; + AgentEvent out; + if (ev->kind == SocketEvent::Kind::kBinary) { + out.binary = true; + out.audio = ev->bytes; + out.type = "audio"; + } else { + out.type = ev->type; + out.json = ev->json; + } + return out; +} + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/deepgram/streams.h b/src/vendors/audio/deepgram/streams.h new file mode 100644 index 0000000..cd314cb --- /dev/null +++ b/src/vendors/audio/deepgram/streams.h @@ -0,0 +1,153 @@ +#pragma once + +// Deepgram WebSockets (AsyncAPI https://github.com/deepgram/deepgram-api-specs +// asyncapi.yml): live speech to text (/v1/listen and the Flux turn model +// /v2/listen), live text to speech (/v1/speak and /v2/speak) and the Voice +// Agent (wss://agent.deepgram.com/v1/agent/converse). + +#include +#include +#include +#include + +#include "vendors/audio/deepgram/service.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::deepgram { + +// Query parameters a channel accepts (copied from the AsyncAPI channel +// bindings; check_spec.py compares them). Used to refuse typos before connecting. +struct StreamChannels { + static const std::vector& listen_v1_params(); + static const std::vector& listen_v2_params(); + static const std::vector& speak_v1_params(); + static const std::vector& speak_v2_params(); + static Vocabulary listen_v1(); + static Vocabulary listen_v2(); + static Vocabulary speak_v1(); + static Vocabulary speak_v2(); + static Vocabulary agent(); +}; + +struct StreamParams { + // name -> value; each name must be one the channel accepts. Lists repeat the key. + std::vector> pairs; + StreamParams& add(const std::string& name, const std::string& value) { + pairs.emplace_back(name, value); + return *this; + } +}; + +struct ListenEvent { + enum class Kind { kOther, kResults, kMetadata, kUtteranceEnd, kSpeechStarted, kTurn, kError, kClose }; + Kind kind = Kind::kOther; + std::string transcript; // alternatives[0].transcript (v1) or transcript (v2 TurnInfo) + bool is_final = false; // v1 is_final, v2 EndOfTurn + bool speech_final = false; + double confidence = 0; + std::string turn_event; // v2: StartOfTurn Update EagerEndOfTurn TurnResumed EndOfTurn + std::string request_id; + Json raw; +}; + +struct ListenResult { + std::vector finals; // final transcripts in order + std::string transcript; // finals joined by a space + std::string interim; // last interim + std::size_t utterance_ends = 0; + std::string request_id; + double duration = 0; + std::string error; + bool closed = false; + std::size_t events = 0; +}; + +class ListenStream { + public: + ListenStream(std::string api_key, StreamParams params, bool flux = false, std::string url = "", + SocketOptions socket = {}); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void send_audio(const std::string& audio) { socket_->send_binary(audio); } + void keep_alive(); // {"type":"KeepAlive"} + void finalize(); // {"type":"Finalize"} (v1) + void force_end_turn(); // {"type":"ForceEndTurn"} (v2) + void close_stream(); // {"type":"CloseStream"} + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + static ListenEvent decode(const SocketEvent& ev); + // Reads until Metadata (end), error, close, or silence of `max_wait_ms`. + ListenResult collect(int max_wait_ms); + + private: + bool flux_; + std::unique_ptr socket_; +}; + +struct SpeakStreamResult { + std::string audio; + std::size_t chunks = 0; + bool flushed = false; + bool cleared = false; + bool truncated = false; + std::vector metadata; // JSON control messages in order + std::string warning; + std::string error; + bool closed = false; +}; + +class SpeakStream { + public: + SpeakStream(std::string api_key, StreamParams params, bool v2 = false, std::string url = "", + SocketOptions socket = {}); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void speak(const std::string& text); // {"type":"Speak","text":...}, 1..2000 chars + void flush(); // {"type":"Flush"} + void clear(); // {"type":"Clear"} (v1) + void interrupt(double playback_offset); // {"type":"Interrupt"} (v2) + void end(); // {"type":"Close"} + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + // Reads until Flushed/Cleared, error, close, or silence. + SpeakStreamResult collect(int max_wait_ms); + + private: + bool v2_; + std::unique_ptr socket_; +}; + +// Voice Agent: settings are a free-form object (the schema is large and +// provider owned); message names are enforced by the vocabulary. +struct AgentEvent { + std::string type; + Json json; + std::string audio; // binary frames carry agent audio + bool binary = false; +}; + +class AgentSession { + public: + AgentSession(std::string api_key, std::string url = "", SocketOptions socket = {}); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void send_settings(const Json& settings); // {"type":"Settings",...}; Settings fields as the spec has them + void send_audio(const std::string& audio) { socket_->send_binary(audio); } + void inject_user_message(const std::string& content); + void inject_agent_message(const std::string& message); + void keep_alive(); + void send_function_result(const std::string& id, const std::string& name, const std::string& content); + std::optional next(int wait_ms); + + private: + std::unique_ptr socket_; +}; + +} // namespace molo::audio::deepgram diff --git a/src/vendors/audio/elevenlabs/account.cpp b/src/vendors/audio/elevenlabs/account.cpp new file mode 100644 index 0000000..76dc30b --- /dev/null +++ b/src/vendors/audio/elevenlabs/account.cpp @@ -0,0 +1,41 @@ +#include "vendors/audio/elevenlabs/account.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +Json Account::models() { return call(C::kModels); } +Json Account::user() { return call(C::kUser); } +Json Account::subscription() { return call(C::kSubscription); } + +Json Account::history(const Json& q) { + if (q.is_object()) { + const Json* size = q.find("page_size"); + if (size && size->is_number() && (size->as_int() < 1 || size->as_int() > 1000)) + bad("page_size must be 1..1000"); + } + return call(C::kHistory, {}, nullptr, query_from(C::kHistory, q)); +} + +Json Account::history_item(const std::string& id) { + check_id("history item", id); + return call(C::kHistoryItem, {id}); +} + +AudioBlob Account::history_audio(const std::string& id) { + check_id("history item", id); + llm::RawResponse raw = call_raw(C::kHistoryAudio, {id}); + return AudioBlob{raw.body, raw.content_type}; +} + +Json Account::remove_history_item(const std::string& id, bool confirm) { + check_id("history item", id); + return call_destructive(C::kHistoryDelete, {id}, nullptr, Query(), confirm); +} + +Json Account::single_use_token(const std::string& type) { + check_id("token type", type); + return call(C::kSingleUseToken, {type}); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/account.h b/src/vendors/audio/elevenlabs/account.h new file mode 100644 index 0000000..2a8948d --- /dev/null +++ b/src/vendors/audio/elevenlabs/account.h @@ -0,0 +1,30 @@ +#pragma once + +// Account reads: generation history, models, user, subscription; plus +// short-lived single-use tokens for browser WebSockets. + +#include + +#include "vendors/audio/elevenlabs/service.h" +#include "vendors/audio/elevenlabs/tts.h" + +namespace molo::audio::elevenlabs { + +class Account : public ElevenService { + public: + using ElevenService::ElevenService; + + Json models(); + Json user(); + Json subscription(); + // History of generated speech, one page (page_size <= 1000 per the OpenAPI text). + Json history(const Json& query_object = Json()); + Json history_item(const std::string& history_item_id); + AudioBlob history_audio(const std::string& history_item_id); + Json remove_history_item(const std::string& history_item_id, bool confirm); + // token_type is a path segment (for example realtime_scribe or tts_websocket); + // the token is a credential, never log it. + Json single_use_token(const std::string& token_type); +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/agents.cpp b/src/vendors/audio/elevenlabs/agents.cpp new file mode 100644 index 0000000..59b5fd5 --- /dev/null +++ b/src/vendors/audio/elevenlabs/agents.cpp @@ -0,0 +1,113 @@ +#include "vendors/audio/elevenlabs/agents.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +Json Agents::list(const Json& q) { return call(C::kAgents, {}, nullptr, query_from(C::kAgents, q)); } + +Json Agents::list_all(std::size_t max_pages, int page_size) { + if (max_pages == 0 || max_pages > AudioCaps::kMaxPages) bad("max_pages must be 1.." + std::to_string(AudioCaps::kMaxPages)); + if (page_size < 1 || page_size > ElevenCaps::kMaxPageSize) bad("page_size must be 1..100"); + Json agents = Json::array(); + Json q = Json::object(); + q.set("page_size", Json(page_size)); + bool truncated = true; + std::size_t pages = 0; + for (; pages < max_pages;) { + Json doc = list(q); + ++pages; + if (const Json* a = doc.find("agents"); a && a->is_array()) + for (const Json& e : a->as_array()) { + if (agents.as_array().size() >= AudioCaps::kMaxListItems) { + Json out = Json::object(); + out.set("agents", agents); + out.set("truncated", Json(true)); + out.set("pages", Json(static_cast(pages))); + return out; + } + agents.push(e); + } + const Json* more = doc.find("has_more"); + const Json* cursor = doc.find("next_cursor"); + if (!more || !more->is_bool() || !more->as_bool() || !cursor || !cursor->is_string() || + cursor->as_string().empty()) { + truncated = false; + break; + } + q.set("cursor", Json(cursor->as_string())); + } + Json out = Json::object(); + out.set("agents", agents); + out.set("truncated", Json(truncated)); + out.set("pages", Json(static_cast(pages))); + return out; +} + +Json Agents::get(const std::string& id, const Json& q) { + check_id("agent", id); + return call(C::kAgentGet, {id}, nullptr, query_from(C::kAgentGet, q)); +} + +Json Agents::create(const Json& body, bool confirm, bool enable_versioning) { + check_body(table_.get(C::kAgentCreate), &body); + require_confirm("creating an agent", confirm); + Query q; + if (enable_versioning) q.add_bool("enable_versioning", true); + return call(C::kAgentCreate, {}, &body, q); +} + +Json Agents::patch(const std::string& id, const Json& body, bool confirm) { + check_id("agent", id); + check_body(table_.get(C::kAgentPatch), &body); + require_confirm("changing an agent", confirm); + return call(C::kAgentPatch, {id}, &body); +} + +Json Agents::remove(const std::string& id, bool confirm, bool force) { + check_id("agent", id); + Query q; + if (force) q.add_bool("force", true); + return call_destructive(C::kAgentDelete, {id}, nullptr, q, confirm); +} + +Json Agents::link(const std::string& id) { + check_id("agent", id); + return call(C::kAgentLink, {id}); +} + +Json Agents::signed_url(const std::string& id, const Json& q) { + Query query = query_from(C::kSignedUrl, q); + check_id("agent", id); + Query with_agent; + with_agent.add("agent_id", id); + for (const auto& [k, v] : query.pairs()) + if (k != "agent_id") with_agent.add(k, v); + return call(C::kSignedUrl, {}, nullptr, with_agent); +} + +Json Agents::conversations(const Json& q) { + return call(C::kConversations, {}, nullptr, query_from(C::kConversations, q)); +} + +Json Agents::conversation(const std::string& id, const Json& q) { + check_id("conversation", id); + return call(C::kConversationGet, {id}, nullptr, query_from(C::kConversationGet, q)); +} + +AudioBlob Agents::conversation_audio(const std::string& id) { + check_id("conversation", id); + llm::RawResponse raw = call_raw(C::kConversationAudio, {id}); + return AudioBlob{raw.body, raw.content_type}; +} + +Json Agents::remove_conversation(const std::string& id, bool confirm) { + check_id("conversation", id); + return call_destructive(C::kConversationDelete, {id}, nullptr, Query(), confirm); +} + +Json Agents::llms() { return call(C::kLlmList); } + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/agents.h b/src/vendors/audio/elevenlabs/agents.h new file mode 100644 index 0000000..1bfcc62 --- /dev/null +++ b/src/vendors/audio/elevenlabs/agents.h @@ -0,0 +1,39 @@ +#pragma once + +// Conversational AI agents (ElevenAgents): agent CRUD, signed URLs for the +// conversation WebSocket, conversation history and audio, the LLM catalogue. +// Writes (create, patch, delete) need confirm. Everything else under +// /v1/convai (tools, knowledge base, phone numbers, tests, MCP servers, ...) +// is in the call table and reached with call() / call_destructive(). + +#include + +#include "vendors/audio/elevenlabs/service.h" +#include "vendors/audio/elevenlabs/tts.h" + +namespace molo::audio::elevenlabs { + +class Agents : public ElevenService { + public: + using ElevenService::ElevenService; + + Json list(const Json& query_object = Json()); + // Walks cursors up to max_pages; `truncated` is set when the cap stopped it. + Json list_all(std::size_t max_pages, int page_size = 100); + Json get(const std::string& agent_id, const Json& query_object = Json()); + Json create(const Json& body, bool confirm, bool enable_versioning = false); + Json patch(const std::string& agent_id, const Json& body, bool confirm); + Json remove(const std::string& agent_id, bool confirm, bool force = false); + Json link(const std::string& agent_id); + // Signed wss URL for a conversation with a private agent (valid 15 minutes + // per the docs): the URL itself is a credential, never log it. + Json signed_url(const std::string& agent_id, const Json& query_object = Json()); + + Json conversations(const Json& query_object = Json()); + Json conversation(const std::string& conversation_id, const Json& query_object = Json()); + AudioBlob conversation_audio(const std::string& conversation_id); + Json remove_conversation(const std::string& conversation_id, bool confirm); + Json llms(); +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/calls.cpp b/src/vendors/audio/elevenlabs/calls.cpp new file mode 100644 index 0000000..32ae934 --- /dev/null +++ b/src/vendors/audio/elevenlabs/calls.cpp @@ -0,0 +1,344 @@ +#include "vendors/audio/elevenlabs/calls.h" + +namespace molo::audio::elevenlabs { + +// One row per REST call. tools/check_spec.py parses the block between the +// markers: {id, method, path, host, body, fields, required, query, response, source}. +// CALLS-BEGIN +static const CallSpec kRows[] = { + {"audio_isolation", "POST", "/v1/audio-isolation", "api", "form", "audio,file_format,preview_b64", "audio", "", "bytes", "openapi:audio_isolation"}, + {"get_audio_isolation_history", "GET", "/v1/audio-isolation/history", "api", "-", "", "", "page_size,page,search", "json", "openapi:get_audio_isolation_history"}, + {"delete_audio_isolation_history_item", "DELETE", "/v1/audio-isolation/history/{history_item_id}", "api", "-", "", "", "", "none", "openapi:delete_audio_isolation_history_item"}, + {"audio_isolation_stream", "POST", "/v1/audio-isolation/stream", "api", "form", "audio,file_format", "audio", "", "bytes", "openapi:audio_isolation_stream"}, + {"list_chat_response_tests_route", "GET", "/v1/convai/agent-testing", "api", "-", "", "", "cursor,page_size,search,parent_folder_id,types,include_folders,sort_mode,sharing_mode,created_by_user_id", "json", "openapi:list_chat_response_tests_route"}, + {"agent_testing_bulk_move_route", "POST", "/v1/convai/agent-testing/bulk-move", "api", "json", "entity_ids,move_to", "entity_ids", "", "json", "openapi:agent_testing_bulk_move_route"}, + {"create_agent_response_test_route", "POST", "/v1/convai/agent-testing/create", "api", "json", "from_conversation_metadata,dynamic_variables,chat_history,conversation_initiation_source,environment,type,success_condition,success_examples,failure_examples,name,parent_folder_id,tool_call_parameters,check_any_tool_matches,success_conditions,simulation_scenario,simulation_max_turns,simulation_environment,tool_mock_config,tool_mock_overrides,evaluation_model,simulated_user_model", "", "", "json", "openapi:create_agent_response_test_route"}, + {"create_agent_test_folder_route", "POST", "/v1/convai/agent-testing/folders", "api", "json", "name,parent_folder_id", "name", "", "json", "openapi:create_agent_test_folder_route"}, + {"get_agent_test_folder_route", "GET", "/v1/convai/agent-testing/folders/{folder_id}", "api", "-", "", "", "", "json", "openapi:get_agent_test_folder_route"}, + {"update_agent_test_folder_route", "PATCH", "/v1/convai/agent-testing/folders/{folder_id}", "api", "json", "name", "name", "", "json", "openapi:update_agent_test_folder_route"}, + {"delete_agent_test_folder_route", "DELETE", "/v1/convai/agent-testing/folders/{folder_id}", "api", "-", "", "", "force", "none", "openapi:delete_agent_test_folder_route"}, + {"get_agent_response_tests_summaries_route", "POST", "/v1/convai/agent-testing/summaries", "api", "json", "test_ids", "test_ids", "", "json", "openapi:get_agent_response_tests_summaries_route"}, + {"get_agent_response_test_route", "GET", "/v1/convai/agent-testing/{test_id}", "api", "-", "", "", "", "json", "openapi:get_agent_response_test_route"}, + {"update_agent_response_test_route", "PUT", "/v1/convai/agent-testing/{test_id}", "api", "json", "from_conversation_metadata,dynamic_variables,chat_history,conversation_initiation_source,environment,type,success_condition,success_examples,failure_examples,name,parent_folder_id,tool_call_parameters,check_any_tool_matches,success_conditions,simulation_scenario,simulation_max_turns,simulation_environment,tool_mock_config,tool_mock_overrides,evaluation_model,simulated_user_model", "", "", "json", "openapi:update_agent_response_test_route"}, + {"delete_chat_response_test_route", "DELETE", "/v1/convai/agent-testing/{test_id}", "api", "-", "", "", "", "json", "openapi:delete_chat_response_test_route"}, + {"get_agent_knowledge_base_size", "GET", "/v1/convai/agent/{agent_id}/knowledge-base/size", "api", "-", "", "", "", "json", "openapi:get_agent_knowledge_base_size"}, + {"get_agent_llm_expected_cost_calculation", "POST", "/v1/convai/agent/{agent_id}/llm-usage/calculate", "api", "json", "prompt_length,number_of_pages,rag_enabled,service_tier", "", "", "json", "openapi:get_agent_llm_expected_cost_calculation"}, + {"get_agents_route", "GET", "/v1/convai/agents", "api", "-", "", "", "page_size,search,archived,show_only_owned_agents,created_by_user_id,tags,sort_direction,sort_by,cursor", "json", "openapi:get_agents_route"}, + {"create_agent_route", "POST", "/v1/convai/agents/create", "api", "json", "conversation_config,platform_settings,workflow,name,tags", "conversation_config", "enable_versioning", "json", "openapi:create_agent_route"}, + {"get_agent_summaries_route", "GET", "/v1/convai/agents/summaries", "api", "-", "", "", "agent_ids", "json", "openapi:get_agent_summaries_route"}, + {"get_agent_route", "GET", "/v1/convai/agents/{agent_id}", "api", "-", "", "", "version_id,branch_id,include_procedure_generated_workflow_elements", "json", "openapi:get_agent_route"}, + {"patch_agent_settings_route", "PATCH", "/v1/convai/agents/{agent_id}", "api", "json", "conversation_config,platform_settings,workflow,name,tags,version_description,procedures", "", "enable_versioning_if_not_enabled,branch_id,include_procedure_generated_workflow_elements", "json", "openapi:patch_agent_settings_route"}, + {"delete_agent_route", "DELETE", "/v1/convai/agents/{agent_id}", "api", "-", "", "", "force", "none", "openapi:delete_agent_route"}, + {"post_agent_avatar_route", "POST", "/v1/convai/agents/{agent_id}/avatar", "api", "form", "avatar_file", "avatar_file", "", "json", "openapi:post_agent_avatar_route"}, + {"get_branches_route", "GET", "/v1/convai/agents/{agent_id}/branches", "api", "-", "", "", "include_archived,limit,include_commit_status", "json", "openapi:get_branches_route"}, + {"create_branch_route", "POST", "/v1/convai/agents/{agent_id}/branches", "api", "json", "parent_version_id,name,description,conversation_config,platform_settings,workflow,include_draft", "parent_version_id,name,description", "", "json", "openapi:create_branch_route"}, + {"get_branch_route", "GET", "/v1/convai/agents/{agent_id}/branches/{branch_id}", "api", "-", "", "", "", "json", "openapi:get_branch_route"}, + {"update_branch_route", "PATCH", "/v1/convai/agents/{agent_id}/branches/{branch_id}", "api", "json", "name,description,is_archived,protection_status,auto_rebase", "", "", "json", "openapi:update_branch_route"}, + {"list_procedures_route", "GET", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures", "api", "-", "", "", "agent_version_id", "json", "openapi:list_procedures_route"}, + {"create_procedure_route", "POST", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures", "api", "json", "name,content,type,trigger,folder_parent_id", "", "", "json", "openapi:create_procedure_route"}, + {"compile_procedures_route", "POST", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures/compile", "api", "-", "", "", "", "json", "openapi:compile_procedures_route"}, + {"get_procedure_route", "GET", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures/{procedure_id}", "api", "-", "", "", "version_id,agent_version_id", "json", "openapi:get_procedure_route"}, + {"remove_procedure_route", "DELETE", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures/{procedure_id}", "api", "-", "", "", "", "json", "openapi:remove_procedure_route"}, + {"get_procedure_draft_route", "GET", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures/{procedure_id}/draft", "api", "-", "", "", "", "json", "openapi:get_procedure_draft_route"}, + {"update_procedure_draft_route", "PATCH", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures/{procedure_id}/draft", "api", "json", "name,content,type,trigger", "name,content,type", "", "json", "openapi:update_procedure_draft_route"}, + {"delete_procedure_draft_route", "DELETE", "/v1/convai/agents/{agent_id}/branches/{branch_id}/procedures/{procedure_id}/draft", "api", "-", "", "", "", "json", "openapi:delete_procedure_draft_route"}, + {"rebase_branch_onto_main", "POST", "/v1/convai/agents/{agent_id}/branches/{branch_id}/rebase", "api", "json", "*", "", "source_branch_id", "json", "openapi:rebase_branch_onto_main"}, + {"rebase_preview_route", "GET", "/v1/convai/agents/{agent_id}/branches/{branch_id}/rebase-preview", "api", "-", "", "", "source_branch_id,include_procedure_generated_workflow_elements", "json", "openapi:rebase_preview_route"}, + {"merge_branch_into_target", "POST", "/v1/convai/agents/{agent_id}/branches/{source_branch_id}/merge", "api", "json", "archive_source_branch,force", "", "target_branch_id", "json", "openapi:merge_branch_into_target"}, + {"merge_preview_route", "GET", "/v1/convai/agents/{agent_id}/branches/{source_branch_id}/merge-preview", "api", "-", "", "", "target_branch_id,force,include_procedure_generated_workflow_elements", "json", "openapi:merge_preview_route"}, + {"list_agent_deployments_route", "GET", "/v1/convai/agents/{agent_id}/deployments", "api", "-", "", "", "page,page_size", "json", "openapi:list_agent_deployments_route"}, + {"create_agent_deployment_route", "POST", "/v1/convai/agents/{agent_id}/deployments", "api", "json", "deployment_request", "deployment_request", "", "json", "openapi:create_agent_deployment_route"}, + {"create_agent_draft_route", "POST", "/v1/convai/agents/{agent_id}/drafts", "api", "json", "conversation_config,platform_settings,workflow,name,tags", "conversation_config,platform_settings,workflow,name", "branch_id", "json", "openapi:create_agent_draft_route"}, + {"delete_agent_draft_route", "DELETE", "/v1/convai/agents/{agent_id}/drafts", "api", "-", "", "", "branch_id", "json", "openapi:delete_agent_draft_route"}, + {"duplicate_agent_route", "POST", "/v1/convai/agents/{agent_id}/duplicate", "api", "json", "name", "", "", "json", "openapi:duplicate_agent_route"}, + {"post_agent_hold_audio_route", "POST", "/v1/convai/agents/{agent_id}/hold-audio", "api", "form", "hold_audio_file", "hold_audio_file", "", "json", "openapi:post_agent_hold_audio_route"}, + {"delete_agent_hold_audio_route", "DELETE", "/v1/convai/agents/{agent_id}/hold-audio", "api", "-", "", "", "", "json", "openapi:delete_agent_hold_audio_route"}, + {"query_agent_knowledge_base_rag_route", "POST", "/v1/convai/agents/{agent_id}/knowledge-base/rag-query", "api", "json", "query,use_agent_defaults,max_documents_length,max_retrieved_rag_chunks_count", "query", "branch_id", "json", "openapi:query_agent_knowledge_base_rag_route"}, + {"get_agent_link_route", "GET", "/v1/convai/agents/{agent_id}/link", "api", "-", "", "", "", "json", "openapi:get_agent_link_route"}, + {"list_merge_proposals_route", "GET", "/v1/convai/agents/{agent_id}/merge-proposals", "api", "-", "", "", "status,source_branch_id,target_branch_id,search,page_size,cursor", "json", "openapi:list_merge_proposals_route"}, + {"create_merge_proposal_route", "POST", "/v1/convai/agents/{agent_id}/merge-proposals", "api", "json", "source_branch_id,target_branch_id,title,description,requested_reviewer_user_ids,triage_ticket_id", "source_branch_id,target_branch_id,title", "", "json", "openapi:create_merge_proposal_route"}, + {"get_merge_proposal_route", "GET", "/v1/convai/agents/{agent_id}/merge-proposals/{merge_proposal_id}", "api", "-", "", "", "", "json", "openapi:get_merge_proposal_route"}, + {"update_merge_proposal_route", "PATCH", "/v1/convai/agents/{agent_id}/merge-proposals/{merge_proposal_id}", "api", "json", "title,description,requested_reviewer_user_ids,close", "", "", "json", "openapi:update_merge_proposal_route"}, + {"add_merge_proposal_comment_route", "POST", "/v1/convai/agents/{agent_id}/merge-proposals/{merge_proposal_id}/comments", "api", "json", "body", "body", "", "json", "openapi:add_merge_proposal_comment_route"}, + {"accept_merge_proposal_route", "POST", "/v1/convai/agents/{agent_id}/merge-proposals/{merge_proposal_id}/merge", "api", "json", "archive_source_branch,force", "", "", "json", "openapi:accept_merge_proposal_route"}, + {"submit_merge_proposal_review_route", "POST", "/v1/convai/agents/{agent_id}/merge-proposals/{merge_proposal_id}/reviews", "api", "json", "state,comment", "state", "", "json", "openapi:submit_merge_proposal_review_route"}, + {"run_agent_test_suite_route", "POST", "/v1/convai/agents/{agent_id}/run-tests", "api", "json", "name,tests,agent_config_override,branch_id,repeat_count", "tests", "", "json", "openapi:run_agent_test_suite_route"}, + {"run_conversation_simulation_route", "POST", "/v1/convai/agents/{agent_id}/simulate-conversation", "api", "json", "simulation_specification,extra_evaluation_criteria,new_turns_limit", "simulation_specification", "", "json", "openapi:run_conversation_simulation_route"}, + {"run_conversation_simulation_route_stream", "POST", "/v1/convai/agents/{agent_id}/simulate-conversation/stream", "api", "json", "simulation_specification,extra_evaluation_criteria,new_turns_limit", "simulation_specification", "", "none", "openapi:run_conversation_simulation_route_stream"}, + {"get_agent_topics_route", "GET", "/v1/convai/agents/{agent_id}/topics", "api", "-", "", "", "page_size,sort_by,sort_direction,from_unix_secs,to_unix_secs,include_evaluation_criteria,cursor", "json", "openapi:get_agent_topics_route"}, + {"list_agent_conversation_tickets_route", "GET", "/v1/convai/agents/{agent_id}/triage-tickets", "api", "-", "", "", "page_size,conversation_id,status,sources,priorities,sort_by,owner_user_id,assignee_user_id,issue_type,label,merged_into_ticket_id,search,cursor", "json", "openapi:list_agent_conversation_tickets_route"}, + {"create_manual_agent_ticket_route", "POST", "/v1/convai/agents/{agent_id}/triage-tickets", "api", "json", "title,qa_comment,priority", "qa_comment", "", "json", "openapi:create_manual_agent_ticket_route"}, + {"get_assignable_users_route", "GET", "/v1/convai/agents/{agent_id}/triage-tickets/assignable-users", "api", "-", "", "", "", "json", "openapi:get_assignable_users_route"}, + {"get_version_metadata_route", "GET", "/v1/convai/agents/{agent_id}/versions/{version_id}", "api", "-", "", "", "", "json", "openapi:get_version_metadata_route"}, + {"get_agent_widget_route", "GET", "/v1/convai/agents/{agent_id}/widget", "api", "-", "", "", "conversation_signature", "json", "openapi:get_agent_widget_route"}, + {"get_live_count", "GET", "/v1/convai/analytics/live-count", "api", "-", "", "", "agent_id,agent_ids", "json", "openapi:get_live_count"}, + {"create_batch_call", "POST", "/v1/convai/batch-calling/submit", "api", "json", "call_name,agent_id,recipients,scheduled_time_unix,agent_phone_number_id,whatsapp_params,timezone,branch_id,environment,telephony_call_config,target_concurrency_limit", "call_name,agent_id,recipients", "", "json", "openapi:create_batch_call"}, + {"get_workspace_batch_calls", "GET", "/v1/convai/batch-calling/workspace", "api", "-", "", "", "limit,last_doc,agent_id", "json", "openapi:get_workspace_batch_calls"}, + {"get_batch_call", "GET", "/v1/convai/batch-calling/{batch_id}", "api", "-", "", "", "", "json", "openapi:get_batch_call"}, + {"delete_batch_call", "DELETE", "/v1/convai/batch-calling/{batch_id}", "api", "-", "", "", "", "none", "openapi:delete_batch_call"}, + {"cancel_batch_call", "POST", "/v1/convai/batch-calling/{batch_id}/cancel", "api", "-", "", "", "", "json", "openapi:cancel_batch_call"}, + {"export_batch_call", "GET", "/v1/convai/batch-calling/{batch_id}/export", "api", "-", "", "", "limit", "text", "openapi:export_batch_call"}, + {"retry_batch_call", "POST", "/v1/convai/batch-calling/{batch_id}/retry", "api", "-", "", "", "", "json", "openapi:retry_batch_call"}, + {"get_conversation_signed_link", "GET", "/v1/convai/conversation/get-signed-url", "api", "-", "", "", "agent_id,include_conversation_id,branch_id,version_id,environment,debug_events_request", "json", "openapi:get_conversation_signed_link"}, + {"get_signed_url_deprecated", "GET", "/v1/convai/conversation/get_signed_url", "api", "-", "", "", "agent_id,include_conversation_id,branch_id,version_id,environment,debug_events_request", "json", "openapi:get_signed_url_deprecated"}, + {"get_livekit_token", "GET", "/v1/convai/conversation/token", "api", "-", "", "", "agent_id,participant_name,branch_id,version_id,environment,debug_events_request", "json", "openapi:get_livekit_token"}, + {"get_conversation_histories_route", "GET", "/v1/convai/conversations", "api", "-", "", "", "cursor,agent_id,visited_agent_ids,visited_agent_branch_ids,triggered_procedure_ids,call_successful,call_start_before_unix,call_start_after_unix,call_duration_min_secs,call_duration_max_secs,rating_max,rating_min,has_feedback_comment,user_id,evaluation_params,data_collection_params,dynamic_variable_params,data_collection_ids,evaluation_criteria_ids,tool_names,tool_names_successful,tool_names_errored,include_invalid_tool_calls,main_languages,page_size,summary_mode,search,conversation_initiation_source,text_only,conversation_product_type,branch_id,version_id,parent_conversation_id,topic_ids,exclude_statuses,tag_ids,workflow_node_entered_id,termination_reasons,guardrail_types,custom_guardrail_names,sort_direction", "json", "openapi:get_conversation_histories_route"}, + {"smart_search_conversation_messages_route", "GET", "/v1/convai/conversations/messages/smart-search", "api", "-", "", "", "text_query,agent_id,branch_id,page_size,cursor", "json", "openapi:smart_search_conversation_messages_route"}, + {"text_search_conversation_messages_route", "GET", "/v1/convai/conversations/messages/text-search", "api", "-", "", "", "text_query,agent_id,visited_agent_ids,visited_agent_branch_ids,triggered_procedure_ids,call_successful,call_start_before_unix,call_start_after_unix,call_duration_min_secs,call_duration_max_secs,rating_max,rating_min,has_feedback_comment,user_id,evaluation_params,data_collection_params,dynamic_variable_params,tool_names,tool_names_successful,tool_names_errored,include_invalid_tool_calls,main_languages,exclude_statuses,termination_reasons,page_size,summary_mode,conversation_initiation_source,text_only,conversation_product_type,branch_id,version_id,topic_ids,sort_by,cursor", "json", "openapi:text_search_conversation_messages_route"}, + {"resolve_conversation_reference_route", "GET", "/v1/convai/conversations/resolve", "api", "-", "", "", "agent_id,reference", "json", "openapi:resolve_conversation_reference_route"}, + {"get_conversation_history_route", "GET", "/v1/convai/conversations/{conversation_id}", "api", "-", "", "", "format", "json", "openapi:get_conversation_history_route"}, + {"delete_conversation_route", "DELETE", "/v1/convai/conversations/{conversation_id}", "api", "-", "", "", "", "json", "openapi:delete_conversation_route"}, + {"run_conversation_evaluations", "POST", "/v1/convai/conversations/{conversation_id}/analysis/evaluations/run", "api", "json", "evaluation_id,scope", "evaluation_id", "", "json", "openapi:run_conversation_evaluations"}, + {"run_conversation_analysis", "POST", "/v1/convai/conversations/{conversation_id}/analysis/run", "api", "-", "", "", "", "json", "openapi:run_conversation_analysis"}, + {"get_conversation_audio_route", "GET", "/v1/convai/conversations/{conversation_id}/audio", "api", "-", "", "", "", "bytes", "openapi:get_conversation_audio_route"}, + {"post_conversation_feedback_route", "POST", "/v1/convai/conversations/{conversation_id}/feedback", "api", "json", "feedback", "", "", "json", "openapi:post_conversation_feedback_route"}, + {"upload_file_route", "POST", "/v1/convai/conversations/{conversation_id}/files", "api", "form", "file", "file", "", "json", "openapi:upload_file_route"}, + {"cancel_file_upload_route", "DELETE", "/v1/convai/conversations/{conversation_id}/files/{file_id}", "api", "-", "", "", "", "json", "openapi:cancel_file_upload_route"}, + {"get_conversation_sip_messages", "GET", "/v1/convai/conversations/{conversation_id}/sip-messages", "api", "-", "", "", "page_size,cursor", "json", "openapi:get_conversation_sip_messages"}, + {"get_conversation_summary_route", "GET", "/v1/convai/conversations/{conversation_id}/summary", "api", "-", "", "", "max_messages", "json", "openapi:get_conversation_summary_route"}, + {"assign_conversation_tags_route", "POST", "/v1/convai/conversations/{conversation_id}/tags", "api", "json", "tag_ids", "tag_ids", "", "none", "openapi:assign_conversation_tags_route"}, + {"unassign_conversation_tag_route", "DELETE", "/v1/convai/conversations/{conversation_id}/tags/{tag_id}", "api", "-", "", "", "", "none", "openapi:unassign_conversation_tag_route"}, + {"list_environment_variables", "GET", "/v1/convai/environment-variables", "api", "-", "", "", "cursor,page_size,label,environment,type", "json", "openapi:list_environment_variables"}, + {"create_environment_variable", "POST", "/v1/convai/environment-variables", "api", "json", "type,label,values", "", "", "json", "openapi:create_environment_variable"}, + {"get_environment_variable", "GET", "/v1/convai/environment-variables/{env_var_id}", "api", "-", "", "", "", "json", "openapi:get_environment_variable"}, + {"update_environment_variable", "PATCH", "/v1/convai/environment-variables/{env_var_id}", "api", "json", "values", "values", "", "json", "openapi:update_environment_variable"}, + {"list_environments", "GET", "/v1/convai/environments", "api", "-", "", "", "", "json", "openapi:list_environments"}, + {"create_environment", "POST", "/v1/convai/environments", "api", "json", "name", "name", "", "json", "openapi:create_environment"}, + {"delete_environment", "DELETE", "/v1/convai/environments/{environment}", "api", "-", "", "", "", "none", "openapi:delete_environment"}, + {"handle_exotel_outbound_call", "POST", "/v1/convai/exotel/outbound-call", "api", "json", "agent_id,agent_phone_number_id,to_number,conversation_initiation_client_data,telephony_call_config", "agent_id,agent_phone_number_id,to_number", "", "json", "openapi:handle_exotel_outbound_call"}, + {"get_knowledge_base_list_route", "GET", "/v1/convai/knowledge-base", "api", "-", "", "", "page_size,search,show_only_owned_documents,created_by_user_id,types,parent_folder_id,ancestor_folder_id,folders_first,sort_direction,sort_by,cursor", "json", "openapi:get_knowledge_base_list_route"}, + {"add_documentation_to_knowledge_base", "POST", "/v1/convai/knowledge-base", "api", "form", "name,url,file", "", "agent_id", "json", "openapi:add_documentation_to_knowledge_base"}, + {"post_knowledge_base_bulk_delete_route", "POST", "/v1/convai/knowledge-base/bulk-delete", "api", "json", "document_ids,force", "document_ids", "", "json", "openapi:post_knowledge_base_bulk_delete_route"}, + {"post_knowledge_base_bulk_move_route", "POST", "/v1/convai/knowledge-base/bulk-move", "api", "json", "document_ids,move_to", "document_ids", "", "none", "openapi:post_knowledge_base_bulk_move_route"}, + {"list_crawl_jobs_route", "GET", "/v1/convai/knowledge-base/crawl", "api", "-", "", "", "include_job_ids,page_size,cursor", "json", "openapi:list_crawl_jobs_route"}, + {"create_crawl_job_route", "POST", "/v1/convai/knowledge-base/crawl", "api", "json", "url,max_depth,max_pages,pattern,exclude_patterns,sitemap_urls,parent_folder_id,enable_auto_sync,auto_remove,auto_discover,minimum_frequency_days", "url", "", "json", "openapi:create_crawl_job_route"}, + {"get_crawl_job_route", "GET", "/v1/convai/knowledge-base/crawl/{crawl_job_id}", "api", "-", "", "", "", "json", "openapi:get_crawl_job_route"}, + {"cancel_crawl_job_route", "POST", "/v1/convai/knowledge-base/crawl/{crawl_job_id}/cancel", "api", "-", "", "", "", "json", "openapi:cancel_crawl_job_route"}, + {"get_knowledge_base_bulk_dependent_agents_route", "POST", "/v1/convai/knowledge-base/dependent-agents", "api", "json", "document_ids", "document_ids", "dependent_type,page_size,cursor", "json", "openapi:get_knowledge_base_bulk_dependent_agents_route"}, + {"create_file_document_route", "POST", "/v1/convai/knowledge-base/file", "api", "form", "file,name,parent_folder_id", "file", "", "json", "openapi:create_file_document_route"}, + {"create_folder_route", "POST", "/v1/convai/knowledge-base/folder", "api", "json", "name,parent_folder_id,enable_auto_sync,auto_remove,minimum_frequency_days", "name", "", "json", "openapi:create_folder_route"}, + {"get_rag_index_overview", "GET", "/v1/convai/knowledge-base/rag-index", "api", "-", "", "", "", "json", "openapi:get_rag_index_overview"}, + {"get_or_create_rag_indexes", "POST", "/v1/convai/knowledge-base/rag-index", "api", "json", "items", "items", "", "json", "openapi:get_or_create_rag_indexes"}, + {"search_knowledge_base_content_route", "GET", "/v1/convai/knowledge-base/search", "api", "-", "", "", "query,page_size,types,cursor", "json", "openapi:search_knowledge_base_content_route"}, + {"get_agent_knowledge_base_summaries_route", "GET", "/v1/convai/knowledge-base/summaries", "api", "-", "", "", "document_ids", "json", "openapi:get_agent_knowledge_base_summaries_route"}, + {"create_text_document_route", "POST", "/v1/convai/knowledge-base/text", "api", "json", "text,name,parent_folder_id", "text", "", "json", "openapi:create_text_document_route"}, + {"create_url_document_route", "POST", "/v1/convai/knowledge-base/url", "api", "json", "url,name,parent_folder_id,enable_auto_sync,auto_remove,minimum_frequency_days", "url", "", "json", "openapi:create_url_document_route"}, + {"post_knowledge_base_move_route", "POST", "/v1/convai/knowledge-base/{document_id}/move", "api", "json", "move_to", "", "", "none", "openapi:post_knowledge_base_move_route"}, + {"get_documentation_from_knowledge_base", "GET", "/v1/convai/knowledge-base/{documentation_id}", "api", "-", "", "", "agent_id", "json", "openapi:get_documentation_from_knowledge_base"}, + {"update_document_route", "PATCH", "/v1/convai/knowledge-base/{documentation_id}", "api", "json", "name,content", "", "", "json", "openapi:update_document_route"}, + {"delete_knowledge_base_document", "DELETE", "/v1/convai/knowledge-base/{documentation_id}", "api", "-", "", "", "force", "json", "openapi:delete_knowledge_base_document"}, + {"get_documentation_chunk_from_knowledge_base", "GET", "/v1/convai/knowledge-base/{documentation_id}/chunk/{chunk_id}", "api", "-", "", "", "embedding_model", "json", "openapi:get_documentation_chunk_from_knowledge_base"}, + {"get_documentation_chunks_from_knowledge_base", "GET", "/v1/convai/knowledge-base/{documentation_id}/chunks", "api", "-", "", "", "embedding_model,page_size,cursor", "json", "openapi:get_documentation_chunks_from_knowledge_base"}, + {"get_knowledge_base_content", "GET", "/v1/convai/knowledge-base/{documentation_id}/content", "api", "-", "", "", "", "text", "openapi:get_knowledge_base_content"}, + {"get_knowledge_base_dependent_agents", "GET", "/v1/convai/knowledge-base/{documentation_id}/dependent-agents", "api", "-", "", "", "dependent_type,page_size,cursor", "json", "openapi:get_knowledge_base_dependent_agents"}, + {"get_rag_indexes", "GET", "/v1/convai/knowledge-base/{documentation_id}/rag-index", "api", "-", "", "", "", "json", "openapi:get_rag_indexes"}, + {"rag_index_status", "POST", "/v1/convai/knowledge-base/{documentation_id}/rag-index", "api", "json", "model", "model", "", "json", "openapi:rag_index_status"}, + {"delete_rag_index", "DELETE", "/v1/convai/knowledge-base/{documentation_id}/rag-index/{rag_index_id}", "api", "-", "", "", "", "json", "openapi:delete_rag_index"}, + {"refresh_url_document_route", "POST", "/v1/convai/knowledge-base/{documentation_id}/refresh", "api", "-", "", "", "", "json", "openapi:refresh_url_document_route"}, + {"get_knowledge_base_source_file_url", "GET", "/v1/convai/knowledge-base/{documentation_id}/source-file-url", "api", "-", "", "", "", "json", "openapi:get_knowledge_base_source_file_url"}, + {"update_file_document_route", "PATCH", "/v1/convai/knowledge-base/{documentation_id}/update-file", "api", "form", "file", "file", "", "json", "openapi:update_file_document_route"}, + {"get_public_llm_expected_cost_calculation", "POST", "/v1/convai/llm-usage/calculate", "api", "json", "prompt_length,number_of_pages,rag_enabled,service_tier", "prompt_length,number_of_pages,rag_enabled", "", "json", "openapi:get_public_llm_expected_cost_calculation"}, + {"list_available_llms", "GET", "/v1/convai/llm/list", "api", "-", "", "", "", "json", "openapi:list_available_llms"}, + {"list_mcp_servers_route", "GET", "/v1/convai/mcp-servers", "api", "-", "", "", "", "json", "openapi:list_mcp_servers_route"}, + {"create_mcp_server_route", "POST", "/v1/convai/mcp-servers", "api", "json", "config", "config", "", "json", "openapi:create_mcp_server_route"}, + {"get_mcp_route", "GET", "/v1/convai/mcp-servers/{mcp_server_id}", "api", "-", "", "", "", "json", "openapi:get_mcp_route"}, + {"update_mcp_server_config_route", "PATCH", "/v1/convai/mcp-servers/{mcp_server_id}", "api", "json", "approval_policy,force_pre_tool_speech,pre_tool_speech,disable_interruptions,interruption_mode,tool_call_sound,tool_call_sound_behavior,execution_mode,response_timeout_secs,request_headers,request_meta,disable_compression,secret_token,auth_connection", "", "", "json", "openapi:update_mcp_server_config_route"}, + {"delete_mcp_server_route", "DELETE", "/v1/convai/mcp-servers/{mcp_server_id}", "api", "-", "", "", "", "json", "openapi:delete_mcp_server_route"}, + {"update_mcp_server_approval_policy_route", "PATCH", "/v1/convai/mcp-servers/{mcp_server_id}/approval-policy", "api", "json", "approval_policy", "approval_policy", "", "json", "openapi:update_mcp_server_approval_policy_route"}, + {"add_mcp_server_tool_approval_route", "POST", "/v1/convai/mcp-servers/{mcp_server_id}/tool-approvals", "api", "json", "tool_name,tool_description,input_schema,approval_policy", "tool_name,tool_description", "", "json", "openapi:add_mcp_server_tool_approval_route"}, + {"remove_mcp_server_tool_approval_route", "DELETE", "/v1/convai/mcp-servers/{mcp_server_id}/tool-approvals/{tool_name}", "api", "-", "", "", "", "json", "openapi:remove_mcp_server_tool_approval_route"}, + {"add_mcp_tool_config_override_route", "POST", "/v1/convai/mcp-servers/{mcp_server_id}/tool-configs", "api", "json", "force_pre_tool_speech,pre_tool_speech,disable_interruptions,interruption_mode,tool_call_sound,tool_call_sound_behavior,execution_mode,response_timeout_secs,assignments,input_overrides,response_mocks,tool_name", "tool_name", "environment", "json", "openapi:add_mcp_tool_config_override_route"}, + {"get_mcp_tool_config_override_route", "GET", "/v1/convai/mcp-servers/{mcp_server_id}/tool-configs/{tool_name}", "api", "-", "", "", "", "json", "openapi:get_mcp_tool_config_override_route"}, + {"update_mcp_tool_config_override_route", "PATCH", "/v1/convai/mcp-servers/{mcp_server_id}/tool-configs/{tool_name}", "api", "json", "force_pre_tool_speech,pre_tool_speech,disable_interruptions,interruption_mode,tool_call_sound,tool_call_sound_behavior,execution_mode,response_timeout_secs,assignments,input_overrides,response_mocks", "", "environment", "json", "openapi:update_mcp_tool_config_override_route"}, + {"remove_mcp_tool_config_override_route", "DELETE", "/v1/convai/mcp-servers/{mcp_server_id}/tool-configs/{tool_name}", "api", "-", "", "", "", "json", "openapi:remove_mcp_tool_config_override_route"}, + {"list_mcp_server_tools_route", "GET", "/v1/convai/mcp-servers/{mcp_server_id}/tools", "api", "-", "", "", "environment", "json", "openapi:list_mcp_server_tools_route"}, + {"list_phone_numbers_route", "GET", "/v1/convai/phone-numbers", "api", "-", "", "", "provider,agent_id,branch_id", "json", "openapi:list_phone_numbers_route"}, + {"create_phone_number_route", "POST", "/v1/convai/phone-numbers", "api", "json", "phone_number,label,supports_inbound,supports_outbound,provider,agent_id,sid,token,account_auth_token,region_config,enable_sms,account_sid,api_key,api_token,api_subdomain,app_id,applet_url,inbound_trunk_config,outbound_trunk_config", "", "", "json", "openapi:create_phone_number_route"}, + {"get_phone_number_route", "GET", "/v1/convai/phone-numbers/{phone_number_id}", "api", "-", "", "", "", "json", "openapi:get_phone_number_route"}, + {"update_phone_number_route", "PATCH", "/v1/convai/phone-numbers/{phone_number_id}", "api", "json", "agent_id,label,inbound_trunk_config,outbound_trunk_config,livekit_stack,store_sip_messages,environment,branch_id", "", "", "json", "openapi:update_phone_number_route"}, + {"delete_phone_number_route", "DELETE", "/v1/convai/phone-numbers/{phone_number_id}", "api", "-", "", "", "", "json", "openapi:delete_phone_number_route"}, + {"list_sip_messages", "GET", "/v1/convai/phone-numbers/{phone_number_id}/sip-messages", "api", "-", "", "", "page_size,cursor", "json", "openapi:list_sip_messages"}, + {"create_realtime_session", "POST", "/v1/convai/realtime/client_secrets", "api", "json", "session,expires_after", "", "", "json", "openapi:create_realtime_session"}, + {"get_secrets_route", "GET", "/v1/convai/secrets", "api", "-", "", "", "page_size,dependency_limit,search,cursor", "json", "openapi:get_secrets_route"}, + {"create_secret_route", "POST", "/v1/convai/secrets", "api", "json", "type,name,value", "type,name,value", "", "json", "openapi:create_secret_route"}, + {"get_secret_route", "GET", "/v1/convai/secrets/{secret_id}", "api", "-", "", "", "", "json", "openapi:get_secret_route"}, + {"update_secret_route", "PATCH", "/v1/convai/secrets/{secret_id}", "api", "json", "type,name,value", "type,name,value", "", "json", "openapi:update_secret_route"}, + {"delete_secret_route", "DELETE", "/v1/convai/secrets/{secret_id}", "api", "-", "", "", "", "none", "openapi:delete_secret_route"}, + {"get_secret_dependencies_route", "GET", "/v1/convai/secrets/{secret_id}/dependencies/{resource_type}", "api", "-", "", "", "page_size,cursor", "json", "openapi:get_secret_dependencies_route"}, + {"get_settings_route", "GET", "/v1/convai/settings", "api", "-", "", "", "", "json", "openapi:get_settings_route"}, + {"update_settings_route", "PATCH", "/v1/convai/settings", "api", "json", "conversation_initiation_client_data_webhook,webhooks,can_use_mcp_servers,rag_retention_period_days,conversation_embedding_retention_days,default_livekit_stack", "", "", "json", "openapi:update_settings_route"}, + {"get_dashboard_settings_route", "GET", "/v1/convai/settings/dashboard", "api", "-", "", "", "", "json", "openapi:get_dashboard_settings_route"}, + {"update_dashboard_settings_route", "PATCH", "/v1/convai/settings/dashboard", "api", "json", "charts", "", "", "json", "openapi:update_dashboard_settings_route"}, + {"handle_sip_trunk_outbound_call", "POST", "/v1/convai/sip-trunk/outbound-call", "api", "json", "agent_id,agent_phone_number_id,to_number,conversation_initiation_client_data,telephony_call_config", "agent_id,agent_phone_number_id,to_number", "", "json", "openapi:handle_sip_trunk_outbound_call"}, + {"list_conversation_tags_route", "GET", "/v1/convai/tags", "api", "-", "", "", "page_size,cursor", "json", "openapi:list_conversation_tags_route"}, + {"create_conversation_tag_route", "POST", "/v1/convai/tags", "api", "json", "title,description", "title", "", "json", "openapi:create_conversation_tag_route"}, + {"get_conversation_tag_route", "GET", "/v1/convai/tags/{tag_id}", "api", "-", "", "", "", "json", "openapi:get_conversation_tag_route"}, + {"update_conversation_tag_route", "PATCH", "/v1/convai/tags/{tag_id}", "api", "json", "title,description", "", "", "json", "openapi:update_conversation_tag_route"}, + {"delete_conversation_tag_route", "DELETE", "/v1/convai/tags/{tag_id}", "api", "-", "", "", "", "none", "openapi:delete_conversation_tag_route"}, + {"list_test_invocations_route", "GET", "/v1/convai/test-invocations", "api", "-", "", "", "agent_id,branch_id,test_id,start_time,end_time,version_id,page_size,search,cursor", "json", "openapi:list_test_invocations_route"}, + {"get_test_invocation_route", "GET", "/v1/convai/test-invocations/{test_invocation_id}", "api", "-", "", "", "", "json", "openapi:get_test_invocation_route"}, + {"update_test_invocation_route", "PATCH", "/v1/convai/test-invocations/{test_invocation_id}", "api", "json", "name", "name", "", "json", "openapi:update_test_invocation_route"}, + {"cancel_test_invocation_route", "POST", "/v1/convai/test-invocations/{test_invocation_id}/cancel", "api", "-", "", "", "", "none", "openapi:cancel_test_invocation_route"}, + {"resubmit_tests_route", "POST", "/v1/convai/test-invocations/{test_invocation_id}/resubmit", "api", "json", "test_run_ids,agent_config_override,agent_id,branch_id", "test_run_ids,agent_id", "", "json", "openapi:resubmit_tests_route"}, + {"get_tools_route", "GET", "/v1/convai/tools", "api", "-", "", "", "search,page_size,show_only_owned_documents,created_by_user_id,types,sort_direction,sort_by,used_by_agent_id,cursor", "json", "openapi:get_tools_route"}, + {"add_tool_route", "POST", "/v1/convai/tools", "api", "json", "tool_config,response_mocks", "tool_config", "", "json", "openapi:add_tool_route"}, + {"get_tool_route", "GET", "/v1/convai/tools/{tool_id}", "api", "-", "", "", "environment", "json", "openapi:get_tool_route"}, + {"update_tool_route", "PATCH", "/v1/convai/tools/{tool_id}", "api", "json", "tool_config,response_mocks", "tool_config", "", "json", "openapi:update_tool_route"}, + {"delete_tool_route", "DELETE", "/v1/convai/tools/{tool_id}", "api", "-", "", "", "force", "json", "openapi:delete_tool_route"}, + {"get_tool_dependent_agents_route", "GET", "/v1/convai/tools/{tool_id}/dependent-agents", "api", "-", "", "", "cursor,page_size", "json", "openapi:get_tool_dependent_agents_route"}, + {"get_tool_executions_route", "GET", "/v1/convai/tools/{tool_id}/executions", "api", "-", "", "", "cursor,page_size,is_error,agent_id,branch_id,start_time,end_time", "json", "openapi:get_tool_executions_route"}, + {"list_workspace_conversation_tickets_route", "GET", "/v1/convai/triage-tickets", "api", "-", "", "", "page_size,agent_id,status,sources,priorities,sort_by,owner_user_id,assignee_user_id,issue_type,label,merged_into_ticket_id,search,cursor", "json", "openapi:list_workspace_conversation_tickets_route"}, + {"create_agent_conversation_ticket_route", "POST", "/v1/convai/triage-tickets", "api", "json", "conversation_id,title,qa_comment,turn_comments,priority", "conversation_id", "", "json", "openapi:create_agent_conversation_ticket_route"}, + {"get_agent_conversation_ticket_route", "GET", "/v1/convai/triage-tickets/{agentqa_ticket_id}", "api", "-", "", "", "", "json", "openapi:get_agent_conversation_ticket_route"}, + {"update_agent_conversation_ticket_route", "PATCH", "/v1/convai/triage-tickets/{agentqa_ticket_id}", "api", "json", "title,status,assignee_user_id,priority", "", "", "json", "openapi:update_agent_conversation_ticket_route"}, + {"delete_agent_conversation_ticket_route", "DELETE", "/v1/convai/triage-tickets/{agentqa_ticket_id}", "api", "-", "", "", "", "none", "openapi:delete_agent_conversation_ticket_route"}, + {"add_ticket_comment_route", "POST", "/v1/convai/triage-tickets/{agentqa_ticket_id}/comments", "api", "json", "comment", "comment", "", "json", "openapi:add_ticket_comment_route"}, + {"merge_agent_conversation_tickets_route", "POST", "/v1/convai/triage-tickets/{agentqa_ticket_id}/merge", "api", "json", "source_ticket_ids", "source_ticket_ids", "", "json", "openapi:merge_agent_conversation_tickets_route"}, + {"add_turn_comment_route", "POST", "/v1/convai/triage-tickets/{agentqa_ticket_id}/turn-comments", "api", "json", "turn_index,comment", "turn_index,comment", "", "json", "openapi:add_turn_comment_route"}, + {"handle_twilio_outbound_call", "POST", "/v1/convai/twilio/outbound-call", "api", "json", "agent_id,agent_phone_number_id,to_number,conversation_initiation_client_data,call_recording_enabled,telephony_call_config", "agent_id,agent_phone_number_id,to_number", "", "json", "openapi:handle_twilio_outbound_call"}, + {"register_twilio_call", "POST", "/v1/convai/twilio/register-call", "api", "json", "agent_id,from_number,to_number,direction,conversation_initiation_client_data", "agent_id,from_number,to_number", "", "text", "openapi:register_twilio_call"}, + {"get_conversation_users_route", "GET", "/v1/convai/users", "api", "-", "", "", "agent_id,branch_id,call_start_before_unix,call_start_after_unix,search,page_size,sort_by,sort_direction,cursor", "json", "openapi:get_conversation_users_route"}, + {"list_phone_numbers_page_route", "GET", "/v1/convai/v2/phone-numbers", "api", "-", "", "", "page_size,search,label,phone_number,provider,supports_outbound,agent_id,branch_id,sort_by,sort_direction,cursor", "json", "openapi:list_phone_numbers_page_route"}, + {"list_whatsapp_accounts", "GET", "/v1/convai/whatsapp-accounts", "api", "-", "", "", "agent_id", "json", "openapi:list_whatsapp_accounts"}, + {"get_whatsapp_account", "GET", "/v1/convai/whatsapp-accounts/{phone_number_id}", "api", "-", "", "", "", "json", "openapi:get_whatsapp_account"}, + {"update_whatsapp_account", "PATCH", "/v1/convai/whatsapp-accounts/{phone_number_id}", "api", "json", "assigned_agent_id,enable_messaging,enable_audio_message_response,enable_typing_indicator", "", "", "json", "openapi:update_whatsapp_account"}, + {"delete_whatsapp_account", "DELETE", "/v1/convai/whatsapp-accounts/{phone_number_id}", "api", "-", "", "", "", "json", "openapi:delete_whatsapp_account"}, + {"whatsapp_outbound_call", "POST", "/v1/convai/whatsapp/outbound-call", "api", "json", "whatsapp_phone_number_id,whatsapp_user_id,whatsapp_call_permission_request_template_name,whatsapp_call_permission_request_template_language_code,agent_id,conversation_initiation_client_data", "whatsapp_phone_number_id,whatsapp_user_id,whatsapp_call_permission_request_template_name,whatsapp_call_permission_request_template_language_code,agent_id", "", "json", "openapi:whatsapp_outbound_call"}, + {"whatsapp_outbound_message", "POST", "/v1/convai/whatsapp/outbound-message", "api", "json", "whatsapp_phone_number_id,whatsapp_user_id,template_name,template_language_code,template_params,agent_id,conversation_initiation_client_data", "whatsapp_phone_number_id,whatsapp_user_id,template_name,template_language_code,template_params,agent_id", "", "json", "openapi:whatsapp_outbound_message"}, + {"list_dubs", "GET", "/v1/dubbing", "api", "-", "", "", "cursor,page_size,dubbing_status,dubbing_statuses,dubbing_models,target_language_codes,creation_sources,filter_by_creator,order_by,order_direction", "json", "openapi:list_dubs"}, + {"create_dubbing", "POST", "/v1/dubbing", "api", "form", "file,csv_file,foreground_audio_file,background_audio_file,name,source_url,source_lang,target_lang,target_accent,num_speakers,watermark,start_time,end_time,highest_resolution,drop_background_audio,use_profanity_filter,dubbing_studio,disable_voice_cloning,mode,csv_fps", "", "", "json", "openapi:create_dubbing"}, + {"dubbing_project_list", "GET", "/v1/dubbing/project", "api", "-", "", "", "cursor,page_size,status,sort_direction", "json", "openapi:dubbing_project_list"}, + {"dubbing_project_create", "POST", "/v1/dubbing/project", "api", "form", "file,source_url,reference,source_language,model_id,keyterms,webhook_ids,target_language,transcript", "", "", "json", "openapi:dubbing_project_create"}, + {"dubbing_project_get", "GET", "/v1/dubbing/project/{project_id}", "api", "-", "", "", "", "json", "openapi:dubbing_project_get"}, + {"dubbing_project_delete", "DELETE", "/v1/dubbing/project/{project_id}", "api", "-", "", "", "", "none", "openapi:dubbing_project_delete"}, + {"dubbing_language_list", "GET", "/v1/dubbing/project/{project_id}/language", "api", "-", "", "", "cursor,page_size,status", "json", "openapi:dubbing_language_list"}, + {"dubbing_language_create", "POST", "/v1/dubbing/project/{project_id}/language", "api", "json", "target_language,voice_settings,translations", "target_language", "", "json", "openapi:dubbing_language_create"}, + {"dubbing_language_get", "GET", "/v1/dubbing/project/{project_id}/language/{language_id}", "api", "-", "", "", "", "json", "openapi:dubbing_language_get"}, + {"dubbing_language_delete", "DELETE", "/v1/dubbing/project/{project_id}/language/{language_id}", "api", "-", "", "", "", "none", "openapi:dubbing_language_delete"}, + {"dubbing_target_transcript_get", "GET", "/v1/dubbing/project/{project_id}/language/{language_id}/transcript", "api", "-", "", "", "", "json", "openapi:dubbing_target_transcript_get"}, + {"dubbing_target_transcript_regenerate", "POST", "/v1/dubbing/project/{project_id}/language/{language_id}/transcript/regenerate", "api", "-", "", "", "", "json", "openapi:dubbing_target_transcript_regenerate"}, + {"dubbing_target_transcript_segment_update", "PATCH", "/v1/dubbing/project/{project_id}/language/{language_id}/transcript/segment/{segment_id}", "api", "json", "translation", "", "", "json", "openapi:dubbing_target_transcript_segment_update"}, + {"dubbing_target_transcript_segments_update", "PATCH", "/v1/dubbing/project/{project_id}/language/{language_id}/transcript/segments", "api", "json", "segments", "segments", "", "json", "openapi:dubbing_target_transcript_segments_update"}, + {"dubbing_transcript_get", "GET", "/v1/dubbing/project/{project_id}/transcript", "api", "-", "", "", "", "json", "openapi:dubbing_transcript_get"}, + {"dubbing_transcript_segment_add", "POST", "/v1/dubbing/project/{project_id}/transcript/segment", "api", "json", "text,speaker_id,start_s,end_s", "text,speaker_id,start_s,end_s", "", "json", "openapi:dubbing_transcript_segment_add"}, + {"dubbing_transcript_segment_update", "PATCH", "/v1/dubbing/project/{project_id}/transcript/segment/{segment_id}", "api", "json", "text,speaker_id,start_s,end_s", "", "", "json", "openapi:dubbing_transcript_segment_update"}, + {"dubbing_transcript_segment_delete", "DELETE", "/v1/dubbing/project/{project_id}/transcript/segment/{segment_id}", "api", "-", "", "", "", "json", "openapi:dubbing_transcript_segment_delete"}, + {"dubbing_transcript_segments_update", "PATCH", "/v1/dubbing/project/{project_id}/transcript/segments", "api", "json", "segments", "segments", "", "json", "openapi:dubbing_transcript_segments_update"}, + {"get_dubbing_resource", "GET", "/v1/dubbing/resource/{dubbing_id}", "api", "-", "", "", "", "json", "openapi:get_dubbing_resource"}, + {"dub", "POST", "/v1/dubbing/resource/{dubbing_id}/dub", "api", "json", "segments,languages", "segments,languages", "", "json", "openapi:dub"}, + {"add_language", "POST", "/v1/dubbing/resource/{dubbing_id}/language", "api", "json", "language", "language", "", "json", "openapi:add_language"}, + {"migrate_segments", "POST", "/v1/dubbing/resource/{dubbing_id}/migrate-segments", "api", "json", "segment_ids,speaker_id", "segment_ids,speaker_id", "", "json", "openapi:migrate_segments"}, + {"render", "POST", "/v1/dubbing/resource/{dubbing_id}/render/{language}", "api", "json", "render_type,normalize_volume", "render_type", "", "json", "openapi:render"}, + {"delete_segment", "DELETE", "/v1/dubbing/resource/{dubbing_id}/segment/{segment_id}", "api", "-", "", "", "", "json", "openapi:delete_segment"}, + {"update_segment_language", "PATCH", "/v1/dubbing/resource/{dubbing_id}/segment/{segment_id}/{language}", "api", "json", "start_time,end_time,text", "", "", "json", "openapi:update_segment_language"}, + {"create_speaker", "POST", "/v1/dubbing/resource/{dubbing_id}/speaker", "api", "json", "speaker_name,voice_id,voice_stability,voice_similarity,voice_style", "", "", "json", "openapi:create_speaker"}, + {"update_speaker", "PATCH", "/v1/dubbing/resource/{dubbing_id}/speaker/{speaker_id}", "api", "json", "speaker_name,voice_id,voice_stability,voice_similarity,voice_style,languages", "", "", "json", "openapi:update_speaker"}, + {"create_clip", "POST", "/v1/dubbing/resource/{dubbing_id}/speaker/{speaker_id}/segment", "api", "json", "start_time,end_time,text,translations", "start_time,end_time", "", "json", "openapi:create_clip"}, + {"get_similar_voices_for_speaker", "GET", "/v1/dubbing/resource/{dubbing_id}/speaker/{speaker_id}/similar-voices", "api", "-", "", "", "", "json", "openapi:get_similar_voices_for_speaker"}, + {"transcribe", "POST", "/v1/dubbing/resource/{dubbing_id}/transcribe", "api", "json", "segments", "segments", "", "json", "openapi:transcribe"}, + {"translate", "POST", "/v1/dubbing/resource/{dubbing_id}/translate", "api", "json", "segments,languages", "segments,languages", "", "json", "openapi:translate"}, + {"get_dubbed_metadata", "GET", "/v1/dubbing/{dubbing_id}", "api", "-", "", "", "", "json", "openapi:get_dubbed_metadata"}, + {"delete_dubbing", "DELETE", "/v1/dubbing/{dubbing_id}", "api", "-", "", "", "", "json", "openapi:delete_dubbing"}, + {"get_dubbed_file", "GET", "/v1/dubbing/{dubbing_id}/audio/{language_code}", "api", "-", "", "", "", "bytes", "openapi:get_dubbed_file"}, + {"get_dubbed_transcript_file", "GET", "/v1/dubbing/{dubbing_id}/transcript/{language_code}", "api", "-", "", "", "format_type", "json", "openapi:get_dubbed_transcript_file"}, + {"get_dubbing_transcripts", "GET", "/v1/dubbing/{dubbing_id}/transcripts/{language_code}/format/{format_type}", "api", "-", "", "", "", "json", "openapi:get_dubbing_transcripts"}, + {"forced_alignment", "POST", "/v1/forced-alignment", "api", "form", "file,text", "file,text", "", "json", "openapi:forced_alignment"}, + {"get_speech_history", "GET", "/v1/history", "api", "-", "", "", "page_size,start_after_history_item_id,voice_id,model_id,date_before_unix,date_after_unix,sort_direction,search,source", "json", "openapi:get_speech_history"}, + {"download_speech_history_items", "POST", "/v1/history/download", "api", "json", "history_item_ids,output_format", "history_item_ids", "", "bytes", "openapi:download_speech_history_items"}, + {"get_speech_history_item_by_id", "GET", "/v1/history/{history_item_id}", "api", "-", "", "", "", "json", "openapi:get_speech_history_item_by_id"}, + {"delete_speech_history_item", "DELETE", "/v1/history/{history_item_id}", "api", "-", "", "", "", "json", "openapi:delete_speech_history_item"}, + {"get_audio_full_from_speech_history_item", "GET", "/v1/history/{history_item_id}/audio", "api", "-", "", "", "", "bytes", "openapi:get_audio_full_from_speech_history_item"}, + {"get_models", "GET", "/v1/models", "api", "-", "", "", "", "json", "openapi:get_models"}, + {"music_generate", "POST", "/v1/music", "api", "json", "prompt,generation_mode,music_prompt,lyrics_text,composition_plan,music_length_ms,model_id,seed,force_instrumental,finetune_id,finetune_strength,use_phonetic_names,respect_sections_durations,store_for_inpainting,sign_with_c2pa", "", "output_format,enable_logging", "bytes", "openapi:generate"}, + {"compose_detailed", "POST", "/v1/music/detailed", "api", "json", "prompt,generation_mode,music_prompt,lyrics_text,composition_plan,music_length_ms,model_id,seed,force_instrumental,finetune_id,finetune_strength,use_phonetic_names,respect_sections_durations,store_for_inpainting,with_timestamps,with_waveform_visual,sign_with_c2pa,model_style_prefix", "", "output_format,enable_logging", "json", "openapi:compose_detailed"}, + {"compose_detailed_stream", "POST", "/v1/music/detailed/stream", "api", "json", "prompt,generation_mode,music_prompt,lyrics_text,composition_plan,music_length_ms,model_id,seed,force_instrumental,finetune_id,finetune_strength,use_phonetic_names,store_for_inpainting,with_timestamps,with_waveform_visual", "", "output_format,enable_logging", "bytes", "openapi:compose_detailed_stream"}, + {"get_finetunes", "GET", "/v1/music/finetunes", "api", "-", "", "", "cursor,page_size,visibility,created_by,sort,sort_direction", "json", "openapi:get_finetunes"}, + {"create_finetune", "POST", "/v1/music/finetunes", "api", "form", "name,primary_genre,files,tags,visibility,model_id", "name,primary_genre", "", "json", "openapi:create_finetune"}, + {"get_finetune", "GET", "/v1/music/finetunes/{finetune_id}", "api", "-", "", "", "", "json", "openapi:get_finetune"}, + {"update_finetune", "PATCH", "/v1/music/finetunes/{finetune_id}", "api", "json", "name,tags,primary_genre,visibility", "", "", "json", "openapi:update_finetune"}, + {"delete_finetune", "DELETE", "/v1/music/finetunes/{finetune_id}", "api", "-", "", "", "", "json", "openapi:delete_finetune"}, + {"compose_plan", "POST", "/v1/music/plan", "api", "json", "prompt,music_length_ms,source_composition_plan,model_id", "prompt", "enable_logging", "json", "openapi:compose_plan"}, + {"separate_song_stems", "POST", "/v1/music/stem-separation", "api", "form", "file,stem_variation_id,sign_with_c2pa", "file", "output_format,enable_logging", "bytes", "openapi:separate_song_stems"}, + {"stream_compose", "POST", "/v1/music/stream", "api", "json", "prompt,generation_mode,music_prompt,lyrics_text,composition_plan,music_length_ms,model_id,seed,force_instrumental,finetune_id,finetune_strength,use_phonetic_names,store_for_inpainting", "", "output_format,enable_logging", "bytes", "openapi:stream_compose"}, + {"upload_song", "POST", "/v1/music/upload", "api", "form", "file,extract_composition_plan,with_timestamps,with_waveform_visual", "file", "", "json", "openapi:upload_song"}, + {"video_to_music", "POST", "/v1/music/video-to-music", "api", "form", "videos,description,tags,model_id,add_sfx,separate_tracks,use_audio_as_reference,sign_with_c2pa", "videos", "output_format,enable_logging", "bytes", "openapi:video_to_music"}, + {"get_pronunciation_dictionaries_metadata", "GET", "/v1/pronunciation-dictionaries", "api", "-", "", "", "cursor,page_size,sort,sort_direction,include_archived", "json", "openapi:get_pronunciation_dictionaries_metadata"}, + {"add_from_file", "POST", "/v1/pronunciation-dictionaries/add-from-file", "api", "form", "name,file,description,workspace_access", "name", "", "json", "openapi:add_from_file"}, + {"add_from_rules", "POST", "/v1/pronunciation-dictionaries/add-from-rules", "api", "json", "rules,name,description,workspace_access", "rules,name", "", "json", "openapi:add_from_rules"}, + {"get_pronunciation_dictionary_version_pls", "GET", "/v1/pronunciation-dictionaries/{dictionary_id}/{version_id}/download", "api", "-", "", "", "", "text", "openapi:get_pronunciation_dictionary_version_pls"}, + {"get_pronunciation_dictionary_metadata", "GET", "/v1/pronunciation-dictionaries/{pronunciation_dictionary_id}", "api", "-", "", "", "", "json", "openapi:get_pronunciation_dictionary_metadata"}, + {"patch_pronunciation_dictionary", "PATCH", "/v1/pronunciation-dictionaries/{pronunciation_dictionary_id}", "api", "json", "archived,name", "", "", "json", "openapi:patch_pronunciation_dictionary"}, + {"add_rules", "POST", "/v1/pronunciation-dictionaries/{pronunciation_dictionary_id}/add-rules", "api", "json", "rules", "rules", "", "json", "openapi:add_rules"}, + {"remove_rules", "POST", "/v1/pronunciation-dictionaries/{pronunciation_dictionary_id}/remove-rules", "api", "json", "rule_strings", "rule_strings", "", "json", "openapi:remove_rules"}, + {"set_rules", "POST", "/v1/pronunciation-dictionaries/{pronunciation_dictionary_id}/set-rules", "api", "json", "rules", "rules", "", "json", "openapi:set_rules"}, + {"get_library_voices", "GET", "/v1/shared-voices", "api", "-", "", "", "page_size,category,gender,age,accent,language,locale,search,use_cases,descriptives,featured,min_notice_period_days,include_custom_rates,include_live_moderated,reader_app_enabled,owner_id,sort,page", "json", "openapi:get_library_voices"}, + {"get_similar_library_voices", "POST", "/v1/similar-voices", "api", "form", "audio_file,similarity_threshold,top_k", "", "", "json", "openapi:get_similar_library_voices"}, + {"get_single_use_token", "POST", "/v1/single-use-token/{token_type}", "api", "-", "", "", "", "json", "openapi:get_single_use_token"}, + {"sound_generation", "POST", "/v1/sound-generation", "api", "json", "text,loop,duration_seconds,prompt_influence,model_id", "text", "output_format,enable_logging", "bytes", "openapi:sound_generation"}, + {"speech_to_text", "POST", "/v1/speech-to-text", "api", "form", "model_id,file,language_code,transcript_edit,tag_audio_events,num_speakers,timestamps_granularity,diarize,diarization_threshold,additional_formats,file_format,cloud_storage_url,source_url,webhook,webhook_id,temperature,seed,use_multi_channel,multichannel_output_style,webhook_metadata,entity_detection,no_verbatim,use_speaker_library,detect_speaker_roles,entity_redaction,entity_redaction_mode,keyterms", "model_id", "token,enable_logging", "json", "openapi:speech_to_text"}, + {"get_transcript_by_id", "GET", "/v1/speech-to-text/transcripts/{transcription_id}", "api", "-", "", "", "", "json", "openapi:get_transcript_by_id"}, + {"delete_transcript_by_id", "DELETE", "/v1/speech-to-text/transcripts/{transcription_id}", "api", "-", "", "", "", "json", "openapi:delete_transcript_by_id"}, + {"text_to_dialogue", "POST", "/v1/text-to-dialogue", "api", "json", "inputs,model_id,language_code,settings,previous_text,future_text,pronunciation_dictionary_locators,seed,apply_text_normalization,use_pvc_as_ivc,previous_request_ids,next_request_ids,audio_effects", "inputs", "output_format,enable_logging", "bytes", "openapi:text_to_dialogue"}, + {"text_to_dialogue_stream", "POST", "/v1/text-to-dialogue/stream", "api", "json", "inputs,model_id,language_code,settings,previous_text,future_text,pronunciation_dictionary_locators,seed,apply_text_normalization,use_pvc_as_ivc,previous_request_ids,next_request_ids,audio_effects", "inputs", "output_format,enable_logging", "bytes", "openapi:text_to_dialogue_stream"}, + {"text_to_dialogue_stream_with_timestamps", "POST", "/v1/text-to-dialogue/stream/with-timestamps", "api", "json", "inputs,model_id,language_code,settings,pronunciation_dictionary_locators,seed,apply_text_normalization,use_pvc_as_ivc,previous_request_ids,next_request_ids,previous_text,future_text,audio_effects", "inputs", "output_format,enable_logging", "bytes", "openapi:text_to_dialogue_stream_with_timestamps"}, + {"text_to_dialogue_full_with_timestamps", "POST", "/v1/text-to-dialogue/with-timestamps", "api", "json", "inputs,model_id,language_code,settings,pronunciation_dictionary_locators,seed,apply_text_normalization,use_pvc_as_ivc,previous_request_ids,next_request_ids,previous_text,future_text,audio_effects", "inputs", "output_format,enable_logging", "json", "openapi:text_to_dialogue_full_with_timestamps"}, + {"text_to_speech_full", "POST", "/v1/text-to-speech/{voice_id}", "api", "json", "text,model_id,language_code,voice_settings,pronunciation_dictionary_locators,seed,previous_text,next_text,previous_request_ids,next_request_ids,use_pvc_as_ivc,apply_text_normalization,apply_language_text_normalization,audio_effects", "text", "enable_logging,optimize_streaming_latency,output_format", "bytes", "openapi:text_to_speech_full"}, + {"text_to_speech_stream", "POST", "/v1/text-to-speech/{voice_id}/stream", "api", "json", "text,model_id,language_code,voice_settings,pronunciation_dictionary_locators,seed,previous_text,next_text,previous_request_ids,next_request_ids,use_pvc_as_ivc,apply_text_normalization,apply_language_text_normalization,audio_effects", "text", "enable_logging,optimize_streaming_latency,output_format", "bytes", "openapi:text_to_speech_stream"}, + {"text_to_speech_stream_with_timestamps", "POST", "/v1/text-to-speech/{voice_id}/stream/with-timestamps", "api", "json", "text,model_id,language_code,voice_settings,pronunciation_dictionary_locators,seed,previous_text,next_text,previous_request_ids,next_request_ids,use_pvc_as_ivc,apply_text_normalization,apply_language_text_normalization,audio_effects", "text", "enable_logging,optimize_streaming_latency,output_format", "bytes", "openapi:text_to_speech_stream_with_timestamps"}, + {"text_to_speech_full_with_timestamps", "POST", "/v1/text-to-speech/{voice_id}/with-timestamps", "api", "json", "text,model_id,language_code,voice_settings,pronunciation_dictionary_locators,seed,previous_text,next_text,previous_request_ids,next_request_ids,use_pvc_as_ivc,apply_text_normalization,apply_language_text_normalization,audio_effects", "text", "enable_logging,optimize_streaming_latency,output_format", "json", "openapi:text_to_speech_full_with_timestamps"}, + {"create_voice", "POST", "/v1/text-to-voice", "api", "json", "voice_name,voice_description,generated_voice_id,labels,played_not_selected_voice_ids", "voice_name,voice_description,generated_voice_id", "", "json", "openapi:create_voice"}, + {"text_to_voice", "POST", "/v1/text-to-voice/create-previews", "api", "json", "voice_description,text,auto_generate_text,loudness,quality,seed,guidance_scale,should_enhance", "voice_description", "output_format", "json", "openapi:text_to_voice"}, + {"text_to_voice_design", "POST", "/v1/text-to-voice/design", "api", "json", "voice_description,model_id,text,auto_generate_text,loudness,seed,guidance_scale,stream_previews,should_enhance,remixing_session_id,remixing_session_iteration_id,quality,reference_audio_base64,prompt_strength", "voice_description", "output_format", "json", "openapi:text_to_voice_design"}, + {"text_to_voice_preview_stream", "GET", "/v1/text-to-voice/{generated_voice_id}/stream", "api", "-", "", "", "", "bytes", "openapi:text_to_voice_preview_stream"}, + {"text_to_voice_remix", "POST", "/v1/text-to-voice/{voice_id}/remix", "api", "json", "voice_description,text,auto_generate_text,loudness,seed,guidance_scale,stream_previews,remixing_session_id,remixing_session_iteration_id,prompt_strength", "voice_description", "output_format", "json", "openapi:text_to_voice_remix"}, + {"usage_characters", "GET", "/v1/usage/character-stats", "api", "-", "", "", "start_unix,end_unix,include_workspace_metrics,breakdown_type,aggregation_interval,aggregation_bucket_size,metric", "json", "openapi:usage_characters"}, + {"get_user_info", "GET", "/v1/user", "api", "-", "", "", "", "json", "openapi:get_user_info"}, + {"get_user_subscription_info", "GET", "/v1/user/subscription", "api", "-", "", "", "", "json", "openapi:get_user_subscription_info"}, + {"speech_to_speech_full", "POST", "/v1/voice-changer/{voice_id}", "api", "form", "audio,model_id,voice_settings,seed,remove_background_noise,file_format", "audio", "enable_logging,optimize_streaming_latency,output_format", "bytes", "openapi:speech_to_speech_full"}, + {"speech_to_speech_stream", "POST", "/v1/voice-changer/{voice_id}/stream", "api", "form", "audio,model_id,voice_settings,seed,remove_background_noise,file_format", "audio", "enable_logging,optimize_streaming_latency,output_format", "bytes", "openapi:speech_to_speech_stream"}, + {"get_voices", "GET", "/v1/voices", "api", "-", "", "", "show_legacy", "json", "openapi:get_voices"}, + {"get_voice_accents", "GET", "/v1/voices/accents", "api", "-", "", "", "language,model_id", "json", "openapi:get_voice_accents"}, + {"add_voice", "POST", "/v1/voices/add", "api", "form", "name,files,remove_background_noise,description,labels", "name,files", "", "json", "openapi:add_voice"}, + {"add_sharing_voice", "POST", "/v1/voices/add/{public_user_id}/{voice_id}", "api", "json", "new_name,bookmarked", "new_name", "", "json", "openapi:add_sharing_voice"}, + {"get_user_voice_collections", "GET", "/v1/voices/collections", "api", "-", "", "", "", "json", "openapi:get_user_voice_collections"}, + {"create_or_update_user_voice_collection", "POST", "/v1/voices/collections", "api", "json", "title,icon", "title,icon", "", "json", "openapi:create_or_update_user_voice_collection"}, + {"get_user_voice_collection", "GET", "/v1/voices/collections/{collection_id}", "api", "-", "", "", "", "json", "openapi:get_user_voice_collection"}, + {"update_user_voice_collection", "PATCH", "/v1/voices/collections/{collection_id}", "api", "json", "title,icon", "", "", "json", "openapi:update_user_voice_collection"}, + {"delete_user_voice_collection", "DELETE", "/v1/voices/collections/{collection_id}", "api", "-", "", "", "", "json", "openapi:delete_user_voice_collection"}, + {"add_voice_to_user_voice_collection", "POST", "/v1/voices/collections/{collection_id}/voice/{voice_id}", "api", "-", "", "", "", "json", "openapi:add_voice_to_user_voice_collection"}, + {"remove_voice_from_user_voice_collection", "DELETE", "/v1/voices/collections/{collection_id}/voice/{voice_id}", "api", "-", "", "", "", "json", "openapi:remove_voice_from_user_voice_collection"}, + {"create_pvc_voice", "POST", "/v1/voices/pvc", "api", "json", "name,language,description,labels", "name,language", "", "json", "openapi:create_pvc_voice"}, + {"edit_pvc_voice", "POST", "/v1/voices/pvc/{voice_id}", "api", "json", "name,language,description,labels", "", "", "json", "openapi:edit_pvc_voice"}, + {"get_pvc_voice_captcha", "GET", "/v1/voices/pvc/{voice_id}/captcha", "api", "-", "", "", "", "none", "openapi:get_pvc_voice_captcha"}, + {"verify_pvc_voice_captcha", "POST", "/v1/voices/pvc/{voice_id}/captcha", "api", "form", "recording", "recording", "", "json", "openapi:verify_pvc_voice_captcha"}, + {"add_pvc_voice_samples", "POST", "/v1/voices/pvc/{voice_id}/samples", "api", "form", "files,remove_background_noise", "files", "", "json", "openapi:add_pvc_voice_samples"}, + {"edit_pvc_voice_sample", "POST", "/v1/voices/pvc/{voice_id}/samples/{sample_id}", "api", "json", "remove_background_noise,selected_speaker_ids,trim_start_time,trim_end_time,file_name", "", "", "json", "openapi:edit_pvc_voice_sample"}, + {"delete_pvc_voice_sample", "DELETE", "/v1/voices/pvc/{voice_id}/samples/{sample_id}", "api", "-", "", "", "", "json", "openapi:delete_pvc_voice_sample"}, + {"get_pvc_sample_audio", "GET", "/v1/voices/pvc/{voice_id}/samples/{sample_id}/audio", "api", "-", "", "", "remove_background_noise", "json", "openapi:get_pvc_sample_audio"}, + {"start_speaker_separation", "POST", "/v1/voices/pvc/{voice_id}/samples/{sample_id}/separate-speakers", "api", "-", "", "", "", "json", "openapi:start_speaker_separation"}, + {"get_pvc_sample_speakers", "GET", "/v1/voices/pvc/{voice_id}/samples/{sample_id}/speakers", "api", "-", "", "", "", "json", "openapi:get_pvc_sample_speakers"}, + {"get_speaker_audio", "GET", "/v1/voices/pvc/{voice_id}/samples/{sample_id}/speakers/{speaker_id}/audio", "api", "-", "", "", "", "json", "openapi:get_speaker_audio"}, + {"get_pvc_sample_visual_waveform", "GET", "/v1/voices/pvc/{voice_id}/samples/{sample_id}/waveform", "api", "-", "", "", "", "json", "openapi:get_pvc_sample_visual_waveform"}, + {"run_pvc_voice_training", "POST", "/v1/voices/pvc/{voice_id}/train", "api", "json", "model_id", "", "", "json", "openapi:run_pvc_voice_training"}, + {"request_pvc_manual_verification", "POST", "/v1/voices/pvc/{voice_id}/verification", "api", "form", "files,extra_text", "files", "", "json", "openapi:request_pvc_manual_verification"}, + {"get_voice_settings_default", "GET", "/v1/voices/settings/default", "api", "-", "", "", "", "json", "openapi:get_voice_settings_default"}, + {"get_voice_by_id", "GET", "/v1/voices/{voice_id}", "api", "-", "", "", "with_settings", "json", "openapi:get_voice_by_id"}, + {"delete_voice", "DELETE", "/v1/voices/{voice_id}", "api", "-", "", "", "", "json", "openapi:delete_voice"}, + {"edit_voice", "POST", "/v1/voices/{voice_id}/edit", "api", "form", "name,files,remove_background_noise,description,labels,moderate_metadata", "name", "", "json", "openapi:edit_voice"}, + {"replicate_voice_to_isolated_environment", "POST", "/v1/voices/{voice_id}/replicate-to-isolated-environment", "api", "json", "target_workspace_id,preserve_voice_id", "target_workspace_id", "", "json", "openapi:replicate_voice_to_isolated_environment"}, + {"delete_sample", "DELETE", "/v1/voices/{voice_id}/samples/{sample_id}", "api", "-", "", "", "", "json", "openapi:delete_sample"}, + {"get_audio_from_sample", "GET", "/v1/voices/{voice_id}/samples/{sample_id}/audio", "api", "-", "", "", "", "bytes", "openapi:get_audio_from_sample"}, + {"get_voice_settings", "GET", "/v1/voices/{voice_id}/settings", "api", "-", "", "", "", "json", "openapi:get_voice_settings"}, + {"edit_voice_settings", "POST", "/v1/voices/{voice_id}/settings/edit", "api", "json", "stability,use_speaker_boost,similarity_boost,style,speed,clone_strength", "", "", "json", "openapi:edit_voice_settings"}, + {"get_user_voices_v2", "GET", "/v2/voices", "api", "-", "", "", "next_page_token,page_size,search,sort,sort_direction,voice_type,category,fine_tuning_state,collection_id,gender,age,language,accent,use_cases,min_notice_period_days,include_custom_rates,include_live_moderated,high_quality,include_total_count,voice_ids", "json", "openapi:get_user_voices_v2"}, +}; +// CALLS-END + +static_assert(sizeof(kRows) / sizeof(kRows[0]) == 325, "elevenlabs call table size"); + +const CallTable& calls() { + static const CallTable table("elevenlabs", kRows, sizeof(kRows) / sizeof(kRows[0])); + return table; +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/calls.h b/src/vendors/audio/elevenlabs/calls.h new file mode 100644 index 0000000..84323b4 --- /dev/null +++ b/src/vendors/audio/elevenlabs/calls.h @@ -0,0 +1,72 @@ +#pragma once + +// The ElevenLabs call table (https://api.elevenlabs.io/openapi.json, one row +// per operation in the product areas this package covers). Services refer to +// rows by the ids in `ElevenCalls`; nothing builds a path from a literal. + +#include "vendors/audio/call_spec.h" + +namespace molo::audio::elevenlabs { + +const CallTable& calls(); + +struct ElevenCalls { + // Speech + static constexpr const char* kTts = "text_to_speech_full"; + static constexpr const char* kTtsStream = "text_to_speech_stream"; + static constexpr const char* kTtsTimestamps = "text_to_speech_full_with_timestamps"; + static constexpr const char* kTtsStreamTimestamps = "text_to_speech_stream_with_timestamps"; + static constexpr const char* kStt = "speech_to_text"; + static constexpr const char* kSttGet = "get_transcript_by_id"; + static constexpr const char* kSttDelete = "delete_transcript_by_id"; + static constexpr const char* kDialogue = "text_to_dialogue"; + static constexpr const char* kVoiceChanger = "speech_to_speech_full"; + static constexpr const char* kIsolation = "audio_isolation"; + static constexpr const char* kAlignment = "forced_alignment"; + // Voices + static constexpr const char* kVoices = "get_voices"; + static constexpr const char* kVoicesV2 = "get_user_voices_v2"; + static constexpr const char* kVoiceGet = "get_voice_by_id"; + static constexpr const char* kVoiceSettings = "get_voice_settings"; + static constexpr const char* kVoiceSettingsDefault = "get_voice_settings_default"; + static constexpr const char* kVoiceDelete = "delete_voice"; + static constexpr const char* kVoiceAdd = "add_voice"; + static constexpr const char* kSharedVoices = "get_library_voices"; + static constexpr const char* kVoiceDesign = "text_to_voice_design"; + static constexpr const char* kVoiceDesignSave = "create_voice"; + static constexpr const char* kVoiceDesignPreview = "text_to_voice_preview_stream"; + // Creative + static constexpr const char* kSfx = "sound_generation"; + static constexpr const char* kMusic = "music_generate"; + static constexpr const char* kMusicPlan = "compose_plan"; + static constexpr const char* kMusicStream = "stream_compose"; + // Dubbing + static constexpr const char* kDubCreate = "dubbing_project_create"; + static constexpr const char* kDubList = "dubbing_project_list"; + static constexpr const char* kDubGet = "dubbing_project_get"; + static constexpr const char* kDubDelete = "dubbing_project_delete"; + // Agents (conversational AI) + static constexpr const char* kAgents = "get_agents_route"; + static constexpr const char* kAgentGet = "get_agent_route"; + static constexpr const char* kAgentCreate = "create_agent_route"; + static constexpr const char* kAgentPatch = "patch_agent_settings_route"; + static constexpr const char* kAgentDelete = "delete_agent_route"; + static constexpr const char* kAgentLink = "get_agent_link_route"; + static constexpr const char* kSignedUrl = "get_conversation_signed_link"; + static constexpr const char* kConversations = "get_conversation_histories_route"; + static constexpr const char* kConversationGet = "get_conversation_history_route"; + static constexpr const char* kConversationDelete = "delete_conversation_route"; + static constexpr const char* kConversationAudio = "get_conversation_audio_route"; + static constexpr const char* kLlmList = "list_available_llms"; + // Account + static constexpr const char* kHistory = "get_speech_history"; + static constexpr const char* kHistoryItem = "get_speech_history_item_by_id"; + static constexpr const char* kHistoryAudio = "get_audio_full_from_speech_history_item"; + static constexpr const char* kHistoryDelete = "delete_speech_history_item"; + static constexpr const char* kModels = "get_models"; + static constexpr const char* kUser = "get_user_info"; + static constexpr const char* kSubscription = "get_user_subscription_info"; + static constexpr const char* kSingleUseToken = "get_single_use_token"; +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/client.cpp b/src/vendors/audio/elevenlabs/client.cpp new file mode 100644 index 0000000..213431c --- /dev/null +++ b/src/vendors/audio/elevenlabs/client.cpp @@ -0,0 +1,45 @@ +#include "vendors/audio/elevenlabs/client.h" + +#include "llm/config.h" + +namespace molo::audio::elevenlabs { + +std::string ElevenEndpoints::base_url() { + return allowlisted_base(kBaseEnv, kDefaultBase, + {"https://api.elevenlabs.io", "https://api.us.elevenlabs.io", + "https://api.eu.residency.elevenlabs.io", + "https://api.in.residency.elevenlabs.io", + "https://api.sg.residency.elevenlabs.io"}); +} + +llm::ApiClientConfig ElevenEndpoints::config(const std::string& base) { + llm::ApiClientConfig c; + c.vendor = kVendor; + c.base_url = base; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = kKeyHeader; + return c; +} + +std::shared_ptr ElevenEndpoints::from_env() { + auto hosts = std::make_shared(); + hosts->add("api", HostFactory::from_env(config(base_url()))); + return hosts; +} + +std::shared_ptr ElevenEndpoints::with(AudioHost host) { + auto hosts = std::make_shared(); + hosts->add("api", std::move(host)); + return hosts; +} + +std::string ElevenEndpoints::key() { return llm::Config::get(kKeyEnv); } + +AudioSocket::Headers ElevenEndpoints::ws_headers(const std::string& key) { + return {{kKeyHeader, key}}; +} + +std::string ElevenEndpoints::ws_base() { return to_ws_url(base_url()); } + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/client.h b/src/vendors/audio/elevenlabs/client.h new file mode 100644 index 0000000..c0a0405 --- /dev/null +++ b/src/vendors/audio/elevenlabs/client.h @@ -0,0 +1,35 @@ +#pragma once + +// ElevenLabs endpoints and credentials. +// REST https://api.elevenlabs.io header xi-api-key: $ELEVENLABS_API_KEY +// WebSocket wss://api.elevenlabs.io header xi-api-key +// Regional hosts (data residency) are chosen with ELEVENLABS_BASE_URL and only +// the hosts in the OpenAPI/AsyncAPI server lists are accepted. + +#include +#include + +#include "vendors/audio/hosts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::elevenlabs { + +struct ElevenEndpoints { + static constexpr const char* kVendor = "elevenlabs"; + static constexpr const char* kKeyEnv = "ELEVENLABS_API_KEY"; + static constexpr const char* kBaseEnv = "ELEVENLABS_BASE_URL"; + static constexpr const char* kDefaultBase = "https://api.elevenlabs.io"; + static constexpr const char* kKeyHeader = "xi-api-key"; + + static std::string base_url(); // allowlisted override or the default + static llm::ApiClientConfig config(const std::string& base = kDefaultBase); + // Real HTTPS transports, key from llm::Config. + static std::shared_ptr from_env(); + // Injected host (tests): one host called "api". + static std::shared_ptr with(AudioHost host); + static std::string key(); // "" when unset + static AudioSocket::Headers ws_headers(const std::string& key); + static std::string ws_base(); // wss://... of base_url() +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/convai_session.cpp b/src/vendors/audio/elevenlabs/convai_session.cpp new file mode 100644 index 0000000..04798fd --- /dev/null +++ b/src/vendors/audio/elevenlabs/convai_session.cpp @@ -0,0 +1,171 @@ +#include "vendors/audio/elevenlabs/convai_session.h" + +#include "security/crypto/Base64.h" + +namespace molo::audio::elevenlabs { + +namespace { + +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("elevenlabs", why); } + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} + +const Json* obj_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_object() ? v : nullptr; +} + +} // namespace + +Vocabulary ConvaiSession::vocabulary() { + // AsyncAPI channel /v1/convai/conversation. Audio uploads have no type field. + return Vocabulary( + "type", + {"conversation_initiation_client_data", "pong", "user_message", "user_activity", + "contextual_update", "client_tool_result", "mcp_tool_approval_result", "feedback", + "multimodal_message", "file_input"}, + {"conversation_initiation_metadata", "user_transcript", "agent_response", + "agent_response_correction", "agent_response_metadata", "agent_response_complete", + "agent_chat_response_part", "audio", "interruption", "ping", "client_tool_call", + "agent_tool_request", "agent_tool_response", "agent_tool_response_full_payload", "vad_score", + "context_usage", "mcp_tool_call", "mcp_connection_status", "queue_status", + "guardrail_triggered", "client_error"}); +} + +ConvaiSession::ConvaiSession(std::string api_key, ConvaiOptions options) : options_(std::move(options)) { + std::string url; + AudioSocket::Headers headers; + if (!options_.signed_url.empty()) { + if (options_.signed_url.rfind("wss://", 0) != 0 && options_.signed_url.rfind("ws://", 0) != 0) + invalid("signed_url must be a ws or wss URL"); + url = options_.signed_url; // signature stays inside the socket, never reported + } else { + if (!llm::Service::valid_id(options_.agent_id)) invalid("bad agent id"); + std::string base = options_.url.empty() ? ElevenEndpoints::ws_base() + "/v1/convai/conversation" : options_.url; + url = AudioSocket::with_query(base, "agent_id=" + Query::encode(options_.agent_id)); + if (!api_key.empty()) headers = ElevenEndpoints::ws_headers(api_key); + } + socket_ = std::make_unique("elevenlabs", url, headers, vocabulary(), options_.socket); +} + +std::string ConvaiSession::url() const { + if (!options_.signed_url.empty()) { + std::string u = options_.signed_url; + std::size_t q = u.find('?'); + return q == std::string::npos ? u : u.substr(0, q) + "?"; + } + return socket_->url(); +} + +void ConvaiSession::connect() { + socket_->connect(); + Json init = options_.initiation.is_object() ? options_.initiation : Json::object(); + init.set("type", Json("conversation_initiation_client_data")); + socket_->send_json(init); +} + +void ConvaiSession::send_audio(const std::string& pcm) { + if (pcm.empty() || pcm.size() > 1024u * 1024u) invalid("audio chunk must be 1 byte..1 MiB"); + Json m = Json::object(); + m.set("user_audio_chunk", Json(security::crypto::Base64::encode(pcm))); + socket_->send_json(m, true); +} + +void ConvaiSession::send_user_message(const std::string& text) { + if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters"); + Json m = Json::object(); + m.set("type", Json("user_message")); + m.set("text", Json(text)); + socket_->send_json(m); +} + +void ConvaiSession::send_user_activity() { + Json m = Json::object(); + m.set("type", Json("user_activity")); + socket_->send_json(m); +} + +void ConvaiSession::send_contextual_update(const std::string& text) { + if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters"); + Json m = Json::object(); + m.set("type", Json("contextual_update")); + m.set("text", Json(text)); + socket_->send_json(m); +} + +void ConvaiSession::send_tool_result(const std::string& id, const std::string& result, bool is_error) { + if (!llm::Service::valid_id(id)) invalid("bad tool_call_id"); + if (result.size() > 200000) invalid("result is over 200000 characters"); + Json m = Json::object(); + m.set("type", Json("client_tool_result")); + m.set("tool_call_id", Json(id)); + m.set("result", Json(result)); + m.set("is_error", Json(is_error)); + socket_->send_json(m); +} + +void ConvaiSession::fold(const SocketEvent& ev, ConvaiTurn& turn) { + if (ev.kind == SocketEvent::Kind::kClose) { + turn.closed = true; + return; + } + if (ev.kind != SocketEvent::Kind::kText || !ev.json_ok) return; + const Json& j = ev.json; + const std::string& t = ev.type; + if (t == "conversation_initiation_metadata") { + if (const Json* e = obj_at(j, "conversation_initiation_metadata_event")) { + turn.conversation_id = str_at(*e, "conversation_id"); + turn.agent_audio_format = str_at(*e, "agent_output_audio_format"); + turn.user_audio_format = str_at(*e, "user_input_audio_format"); + } + } else if (t == "user_transcript") { + if (const Json* e = obj_at(j, "user_transcription_event")) turn.user_transcripts.push_back(str_at(*e, "user_transcript")); + } else if (t == "agent_response") { + if (const Json* e = obj_at(j, "agent_response_event")) turn.agent_responses.push_back(str_at(*e, "agent_response")); + } else if (t == "audio") { + if (const Json* e = obj_at(j, "audio_event")) { + std::string raw; + if (!security::crypto::Base64::decode(str_at(*e, "audio_base_64"), raw)) + throw llm::ApiError("elevenlabs", 502, "BAD_RESPONSE", "audio event is not valid base64"); + if (turn.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { + turn.truncated = true; + } else { + turn.audio += raw; + ++turn.audio_chunks; + } + } + } else if (t == "interruption") { + turn.interrupted = true; + } else if (t == "ping") { + ++turn.pings; + Json pong = Json::object(); + pong.set("type", Json("pong")); + if (const Json* e = obj_at(j, "ping_event")) { + if (const Json* id = e->find("event_id")) pong.set("event_id", *id); + } + socket_->send_json(pong); + } else if (t == "client_tool_call") { + if (const Json* e = obj_at(j, "client_tool_call")) turn.tool_calls.push_back(*e); + } else if (t == "agent_response_complete") { + turn.complete = true; + } else if (t == "client_error" || t == "error") { + turn.error = str_at(j, "message"); + if (turn.error.empty()) turn.error = t; + } +} + +ConvaiTurn ConvaiSession::collect_turn(int max_wait_ms) { + ConvaiTurn turn; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return turn; + fold(*ev, turn); + if (turn.complete || turn.closed || !turn.error.empty() || turn.truncated) return turn; + } + return turn; +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/convai_session.h b/src/vendors/audio/elevenlabs/convai_session.h new file mode 100644 index 0000000..8daf95b --- /dev/null +++ b/src/vendors/audio/elevenlabs/convai_session.h @@ -0,0 +1,71 @@ +#pragma once + +// A conversation with an ElevenLabs conversational AI agent over WebSocket +// (wss://api.elevenlabs.io/v1/convai/conversation). Public agents connect by +// agent id, private agents with a signed URL (see Agents::signed_url); the +// signed URL is a credential and is never exposed by this class. + +#include +#include +#include +#include + +#include "vendors/audio/elevenlabs/service.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::elevenlabs { + +struct ConvaiOptions { + std::string agent_id; // public or key-authenticated agent + std::string signed_url; // wss URL from Agents::signed_url (takes the place of agent_id and key) + std::string url; // "" = wss://api.elevenlabs.io/v1/convai/conversation (tests) + Json initiation; // conversation_initiation_client_data fields: conversation_config_override, dynamic_variables, ... + SocketOptions socket; +}; + +struct ConvaiTurn { + std::string conversation_id; + std::string user_audio_format, agent_audio_format; + std::vector user_transcripts; + std::vector agent_responses; + std::string audio; // concatenated agent audio (decoded) + std::size_t audio_chunks = 0; + std::vector tool_calls; // client_tool_call objects: answer with send_tool_result + bool interrupted = false; + bool complete = false; // agent_response_complete seen + bool closed = false; + bool truncated = false; + std::string error; + std::size_t pings = 0; // answered automatically +}; + +class ConvaiSession { + public: + // `api_key` may be empty for public agents and signed URLs. + ConvaiSession(std::string api_key, ConvaiOptions options); + + std::string url() const; // without the signature/credential + void connect(); // sends conversation_initiation_client_data + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void send_audio(const std::string& pcm16_16k_mono); // {"user_audio_chunk": base64} + void send_user_message(const std::string& text); + void send_user_activity(); + void send_contextual_update(const std::string& text); + void send_tool_result(const std::string& tool_call_id, const std::string& result, bool is_error); + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + // Reads events (answering pings) until agent_response_complete, an error, + // close, or `max_wait_ms` without an event. + ConvaiTurn collect_turn(int max_wait_ms); + + static Vocabulary vocabulary(); + + private: + void fold(const SocketEvent& ev, ConvaiTurn& turn); + + ConvaiOptions options_; + std::unique_ptr socket_; +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/creative.cpp b/src/vendors/audio/elevenlabs/creative.cpp new file mode 100644 index 0000000..8f87300 --- /dev/null +++ b/src/vendors/audio/elevenlabs/creative.cpp @@ -0,0 +1,98 @@ +#include "vendors/audio/elevenlabs/creative.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +namespace { +Query format_query(const AudioService& svc, const std::string& format) { + Query q; + if (!format.empty()) { + if (!valid_output_format(format)) svc.bad("unknown output_format: " + format); + q.add("output_format", format); + } + return q; +} +} // namespace + +Json SoundEffects::build(const SfxRequest& r) const { + require_text("text", r.text, 4100); + Json o = Json::object(); + o.set("text", Json(r.text)); + if (r.loop >= 0) o.set("loop", Json(r.loop == 1)); + if (r.duration_seconds >= 0) { + require_range("duration_seconds", r.duration_seconds, ElevenCaps::kMinSfxSeconds, ElevenCaps::kMaxSfxSeconds); + o.set("duration_seconds", Json(r.duration_seconds)); + } + if (r.prompt_influence >= 0) { + require_range("prompt_influence", r.prompt_influence, 0, 1); + o.set("prompt_influence", Json(r.prompt_influence)); + } + if (!r.model_id.empty()) { + require_text("model_id", r.model_id, 128); + o.set("model_id", Json(r.model_id)); + } + return o; +} + +AudioBlob SoundEffects::generate(const SfxRequest& r) { + Json body = build(r); + llm::RawResponse raw = call_raw(C::kSfx, {}, &body, format_query(*this, r.output_format)); + return AudioBlob{raw.body, raw.content_type}; +} + +Json Music::build(const MusicRequest& r) const { + bool has_prompt = !r.prompt.empty(); + bool has_plan = r.composition_plan.is_object(); + if (has_prompt == has_plan) bad("give exactly one of prompt and composition_plan"); + Json o = Json::object(); + if (has_prompt) { + require_text("prompt", r.prompt, ElevenCaps::kMaxPromptChars); + o.set("prompt", Json(r.prompt)); + if (r.music_length_ms > 0) { + require_range("music_length_ms", static_cast(r.music_length_ms), ElevenCaps::kMinMusicMs, + ElevenCaps::kMaxMusicMs); + o.set("music_length_ms", Json(r.music_length_ms)); + } + if (r.force_instrumental >= 0) o.set("force_instrumental", Json(r.force_instrumental == 1)); + if (r.seed >= 0) bad("seed is only valid with composition_plan"); + } else { + o.set("composition_plan", r.composition_plan); + if (r.music_length_ms > 0) bad("music_length_ms is only valid with prompt"); + if (r.force_instrumental >= 0) bad("force_instrumental is only valid with prompt"); + if (r.seed >= 0) { + if (r.seed > 2147483647L) bad("seed must be 0..2147483647"); + o.set("seed", Json(r.seed)); + } + } + if (!r.model_id.empty()) { + require_text("model_id", r.model_id, 128); + o.set("model_id", Json(r.model_id)); + } + return o; +} + +AudioBlob Music::compose(const MusicRequest& r) { + Json body = build(r); + llm::RawResponse raw = call_raw(C::kMusic, {}, &body, format_query(*this, r.output_format)); + return AudioBlob{raw.body, raw.content_type}; +} + +ByteStreamer::Result Music::compose_stream(const MusicRequest& r, const ChunkSink& sink) { + Json body = build(r); + return call_stream(C::kMusicStream, {}, &body, format_query(*this, r.output_format), sink); +} + +Json Music::plan(const std::string& prompt, long music_length_ms) { + require_text("prompt", prompt, ElevenCaps::kMaxPromptChars); + Json o = Json::object(); + o.set("prompt", Json(prompt)); + if (music_length_ms > 0) { + require_range("music_length_ms", static_cast(music_length_ms), ElevenCaps::kMinMusicMs, + ElevenCaps::kMaxMusicMs); + o.set("music_length_ms", Json(music_length_ms)); + } + return call(C::kMusicPlan, {}, &o); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/creative.h b/src/vendors/audio/elevenlabs/creative.h new file mode 100644 index 0000000..a10269f --- /dev/null +++ b/src/vendors/audio/elevenlabs/creative.h @@ -0,0 +1,48 @@ +#pragma once + +// Sound effects and music generation. Both return audio bytes. + +#include + +#include "vendors/audio/elevenlabs/service.h" +#include "vendors/audio/elevenlabs/tts.h" + +namespace molo::audio::elevenlabs { + +struct SfxRequest { + std::string text; // required + int loop = -1; // -1 unset + double duration_seconds = -1; // 0.5..30, -1 = the model guesses + double prompt_influence = -1; // 0..1 + std::string model_id; // "" = eleven_text_to_sound_v2 + std::string output_format; // query +}; + +class SoundEffects : public ElevenService { + public: + using ElevenService::ElevenService; + Json build(const SfxRequest& request) const; + AudioBlob generate(const SfxRequest& request); +}; + +struct MusicRequest { + std::string prompt; // <= 4100 characters; exclusive with composition_plan + Json composition_plan; // object from compose_plan, or null + long music_length_ms = 0; // 3000..600000, only with prompt, 0 unset + std::string model_id; // "" = music_v1 + long seed = -1; // 0..2147483647, only with composition_plan + int force_instrumental = -1; // only with prompt + std::string output_format; // query +}; + +class Music : public ElevenService { + public: + using ElevenService::ElevenService; + Json build(const MusicRequest& request) const; + AudioBlob compose(const MusicRequest& request); + ByteStreamer::Result compose_stream(const MusicRequest& request, const ChunkSink& sink); + // Composition plan from a prompt (JSON, no audio). + Json plan(const std::string& prompt, long music_length_ms = 0); +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/dubbing.cpp b/src/vendors/audio/elevenlabs/dubbing.cpp new file mode 100644 index 0000000..756d732 --- /dev/null +++ b/src/vendors/audio/elevenlabs/dubbing.cpp @@ -0,0 +1,58 @@ +#include "vendors/audio/elevenlabs/dubbing.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +clients::body::MultipartBody Dubbing::build(const DubbingRequest& r) const { + if (r.media.empty() == r.source_url.empty()) bad("give exactly one of media and source_url"); + if (r.media.size() > ElevenCaps::kMaxFileBytes) bad("media is over the byte cap"); + require_text("source_url", r.source_url, AudioCaps::kMaxFieldChars, false); + require_text("target_language", r.target_language, 16); + require_text("source_language", r.source_language, 16, false); + require_text("model_id", r.model_id, 128, false); + if (r.keyterms.size() > AudioCaps::kMaxKeywords) bad("too many keyterms"); + clients::body::MultipartBody form; + if (!r.media.empty()) form.add_file("file", r.filename, r.content_type, r.media); + if (!r.source_url.empty()) form.add_text("source_url", r.source_url); + form.add_text("target_language", r.target_language); + if (!r.source_language.empty()) form.add_text("source_language", r.source_language); + if (!r.model_id.empty()) form.add_text("model_id", r.model_id); + for (const auto& k : r.keyterms) { + require_text("keyterm", k, 100); + form.add_text("keyterms", k); + } + return form; +} + +Json Dubbing::create(const DubbingRequest& r, bool confirm) { + auto form = build(r); + require_confirm("creating a dubbing project", confirm); + return call_form(C::kDubCreate, {}, form); +} + +Json Dubbing::list(const Json& q) { return call(C::kDubList, {}, nullptr, query_from(C::kDubList, q)); } + +Json Dubbing::get(const std::string& id) { + check_id("dubbing project", id); + return call(C::kDubGet, {id}); +} + +Json Dubbing::languages(const std::string& id, const Json& q) { + check_id("dubbing project", id); + return call("dubbing_language_list", {id}, nullptr, query_from("dubbing_language_list", q)); +} + +Json Dubbing::transcript(const std::string& id) { + check_id("dubbing project", id); + return call("dubbing_transcript_get", {id}); +} + +Json Dubbing::remove(const std::string& id, bool confirm) { + check_id("dubbing project", id); + return call_destructive(C::kDubDelete, {id}, nullptr, Query(), confirm); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/dubbing.h b/src/vendors/audio/elevenlabs/dubbing.h new file mode 100644 index 0000000..d7d75e2 --- /dev/null +++ b/src/vendors/audio/elevenlabs/dubbing.h @@ -0,0 +1,37 @@ +#pragma once + +// Dubbing projects: create (costs credits, confirm required), list, get, +// languages, transcripts, delete. + +#include +#include + +#include "vendors/audio/elevenlabs/service.h" + +namespace molo::audio::elevenlabs { + +struct DubbingRequest { + std::string media; // bytes of the audio or video, or use source_url + std::string filename = "media"; + std::string content_type = "application/octet-stream"; + std::string source_url; + std::string target_language; // required + std::string source_language; // "" = detect + std::string model_id; + std::vector keyterms; +}; + +class Dubbing : public ElevenService { + public: + using ElevenService::ElevenService; + clients::body::MultipartBody build(const DubbingRequest& request) const; + // Starts a dub. Costs credits: confirm must be true. + Json create(const DubbingRequest& request, bool confirm); + Json list(const Json& query_object = Json()); + Json get(const std::string& project_id); + Json languages(const std::string& project_id, const Json& query_object = Json()); + Json transcript(const std::string& project_id); + Json remove(const std::string& project_id, bool confirm); +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/scribe_stream.cpp b/src/vendors/audio/elevenlabs/scribe_stream.cpp new file mode 100644 index 0000000..5e16ded --- /dev/null +++ b/src/vendors/audio/elevenlabs/scribe_stream.cpp @@ -0,0 +1,128 @@ +#include "vendors/audio/elevenlabs/scribe_stream.h" + +#include "security/crypto/Base64.h" + +namespace molo::audio::elevenlabs { + +namespace { + +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("elevenlabs", why); } + +std::string build_query(const ScribeStreamOptions& o) { + Query q; + q.add_if("model_id", o.model_id); + q.add_if("audio_format", o.audio_format); + q.add_if("language_code", o.language_code); + if (!o.commit_strategy.empty()) { + if (o.commit_strategy != "manual" && o.commit_strategy != "vad") invalid("commit_strategy must be manual or vad"); + q.add("commit_strategy", o.commit_strategy); + } + if (o.include_timestamps >= 0) q.add_bool("include_timestamps", o.include_timestamps == 1); + if (o.include_language_detection >= 0) q.add_bool("include_language_detection", o.include_language_detection == 1); + if (o.keyterms.size() > AudioCaps::kMaxKeywords) invalid("too many keyterms"); + for (const auto& k : o.keyterms) { + if (k.empty() || k.size() > 100) invalid("keyterms are 1..100 characters"); + q.add("keyterms", k); + } + std::string s = q.str(); + return s.empty() ? s : s.substr(1); +} + +} // namespace + +Vocabulary ScribeStream::vocabulary() { + // AsyncAPI channel /v1/speech-to-text/realtime, discriminator message_type. + return Vocabulary( + "message_type", {"input_audio_chunk"}, + {"session_started", "partial_transcript", "committed_transcript", + "committed_transcript_with_timestamps", "committed_transcript_entities", "edited_transcript", + "warning", "error", "auth_error", "quota_exceeded", "transcriber_error", "input_error", + "invalid_request", "rate_limited", "resource_exhausted", "queue_overflow", + "commit_throttled", "chunk_size_exceeded", "insufficient_audio_activity", + "session_time_limit_exceeded", "unaccepted_terms"}); +} + +bool ScribeStream::is_error_type(const std::string& t) { + static const char* kErrors[] = {"error", "auth_error", "quota_exceeded", "transcriber_error", + "input_error", "invalid_request", "rate_limited", + "resource_exhausted", "queue_overflow", "commit_throttled", + "chunk_size_exceeded", "insufficient_audio_activity", + "session_time_limit_exceeded", "unaccepted_terms"}; + for (const char* e : kErrors) + if (t == e) return true; + return false; +} + +ScribeStream::ScribeStream(std::string api_key, ScribeStreamOptions options) { + std::string base = options.url.empty() ? ElevenEndpoints::ws_base() + "/v1/speech-to-text/realtime" : options.url; + std::string url = AudioSocket::with_query(base, build_query(options)); + socket_ = std::make_unique("elevenlabs", url, ElevenEndpoints::ws_headers(api_key), + vocabulary(), options.socket); +} + +void ScribeStream::send_audio(const std::string& audio, bool commit_now, int sample_rate, + const std::string& previous_text) { + if (audio.empty() || audio.size() > 3u * 1024u * 1024u) invalid("audio chunk must be 1 byte..3 MiB"); + Json m = Json::object(); + m.set("message_type", Json("input_audio_chunk")); + m.set("audio_base_64", Json(security::crypto::Base64::encode(audio))); + m.set("commit", Json(commit_now)); + if (sample_rate > 0) { + if (sample_rate < 8000 || sample_rate > 96000) invalid("sample_rate must be 8000..96000"); + m.set("sample_rate", Json(sample_rate)); + } + if (!previous_text.empty()) { + if (previous_text.size() > 50000) invalid("previous_text is over 50000 characters"); + m.set("previous_text", Json(previous_text)); + } + socket_->send_json(m); +} + +void ScribeStream::commit() { + Json m = Json::object(); + m.set("message_type", Json("input_audio_chunk")); + m.set("audio_base_64", Json("")); + m.set("commit", Json(true)); + socket_->send_json(m); +} + +ScribeResult ScribeStream::collect(int max_wait_ms, std::size_t want_committed) { + ScribeResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + ++out.events; + if (ev->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + if (!ev->json_ok || !ev->json.is_object()) continue; + const std::string& t = ev->type; + auto text_of = [&](const char* k) { + const Json* v = ev->json.find(k); + return v && v->is_string() ? v->as_string() : std::string(); + }; + if (t == "session_started") { + out.session_id = text_of("session_id"); + } else if (t == "partial_transcript") { + out.partial = text_of("text"); + } else if (t == "committed_transcript" || t == "committed_transcript_with_timestamps") { + // With timestamps on the server may send both forms for one commit: keep one. + if (t == "committed_transcript_with_timestamps" && !out.committed.empty() && + out.committed.back() == text_of("text")) + continue; + out.committed.push_back(text_of("text")); + out.committed_text += (out.committed_text.empty() ? "" : " ") + out.committed.back(); + if (out.committed.size() >= want_committed) return out; + } else if (t == "warning") { + out.warnings.push_back(text_of("warning")); + } else if (is_error_type(t)) { + out.error_type = t; + out.error_message = text_of("error"); + return out; + } + } + return out; +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/scribe_stream.h b/src/vendors/audio/elevenlabs/scribe_stream.h new file mode 100644 index 0000000..415fa4c --- /dev/null +++ b/src/vendors/audio/elevenlabs/scribe_stream.h @@ -0,0 +1,64 @@ +#pragma once + +// Realtime speech to text (Scribe) over WebSocket: +// wss://api.elevenlabs.io/v1/speech-to-text/realtime. Send audio as +// input_audio_chunk messages, receive partial and committed transcripts. + +#include +#include +#include +#include + +#include "vendors/audio/elevenlabs/service.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::elevenlabs { + +struct ScribeStreamOptions { + std::string model_id; // "" = server default + std::string audio_format; // "" = default, else pcm_16000 etc (as the AsyncAPI lists) + std::string language_code; + std::string commit_strategy; // "", manual, vad + int include_timestamps = -1; + int include_language_detection = -1; + std::vector keyterms; // <= 100 + std::string url; // "" = wss://api.elevenlabs.io/v1/speech-to-text/realtime + SocketOptions socket; +}; + +struct ScribeResult { + std::string committed_text; // committed transcripts joined by a space + std::vector committed; // each committed transcript + std::string partial; // last partial + std::string session_id; + std::string error_type; // message_type of an error event, "" when none + std::string error_message; + std::vector warnings; + bool closed = false; + std::size_t events = 0; +}; + +class ScribeStream { + public: + ScribeStream(std::string api_key, ScribeStreamOptions options); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + // Raw audio bytes (encoded as the audio_format says), 1..3 MiB per chunk. + void send_audio(const std::string& audio, bool commit = false, int sample_rate = 0, + const std::string& previous_text = ""); + void commit(); // empty chunk with commit=true + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + // Reads until a committed transcript, an error, close, or `max_wait_ms` of silence. + ScribeResult collect(int max_wait_ms, std::size_t want_committed = 1); + + static Vocabulary vocabulary(); + static bool is_error_type(const std::string& message_type); + + private: + std::unique_ptr socket_; +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/service.cpp b/src/vendors/audio/elevenlabs/service.cpp new file mode 100644 index 0000000..c61746d --- /dev/null +++ b/src/vendors/audio/elevenlabs/service.cpp @@ -0,0 +1,19 @@ +#include "vendors/audio/elevenlabs/service.h" + +#include + +namespace molo::audio::elevenlabs { + +bool valid_output_format(const std::string& format) { + // OpenAPI /v1/text-to-speech/{voice_id} parameter output_format, enum as read 2026-10-25. + static const std::set kFormats = { + "alaw_8000", "mp3_22050_32", "mp3_24000_48", "mp3_44100_128", "mp3_44100_192", + "mp3_44100_32", "mp3_44100_64", "mp3_44100_96", "opus_48000_128", "opus_48000_192", + "opus_48000_32", "opus_48000_64", "opus_48000_96", "pcm_16000", "pcm_22050", + "pcm_24000", "pcm_32000", "pcm_44100", "pcm_48000", "pcm_8000", + "ulaw_8000", "wav_16000", "wav_22050", "wav_24000", "wav_32000", + "wav_44100", "wav_48000", "wav_8000"}; + return kFormats.count(format) > 0; +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/service.h b/src/vendors/audio/elevenlabs/service.h new file mode 100644 index 0000000..9499843 --- /dev/null +++ b/src/vendors/audio/elevenlabs/service.h @@ -0,0 +1,36 @@ +#pragma once + +// Base of every ElevenLabs service: the ElevenLabs call table plus the +// ceilings that come from the ElevenLabs documentation. + +#include + +#include "vendors/audio/audio_service.h" +#include "vendors/audio/elevenlabs/calls.h" +#include "vendors/audio/elevenlabs/client.h" + +namespace molo::audio::elevenlabs { + +struct ElevenCaps { + static constexpr std::size_t kMaxTtsChars = 40000; // largest model limit in the docs (flash / turbo) + static constexpr std::size_t kMaxPromptChars = 4100; // music prompt (OpenAPI maxLength) + static constexpr double kMinSfxSeconds = 0.5; // OpenAPI: at least 0.5, at most 30 + static constexpr double kMaxSfxSeconds = 30.0; + static constexpr long kMinMusicMs = 3000; // OpenAPI: 3000..600000 + static constexpr long kMaxMusicMs = 600000; + static constexpr int kMaxPageSize = 100; + static constexpr std::size_t kMaxFileBytes = 128u * 1024u * 1024u; +}; + +class ElevenService : public AudioService { + public: + explicit ElevenService(std::shared_ptr hosts) + : AudioService(calls(), std::move(hosts)) {} + // From the environment (real HTTPS, key from ELEVENLABS_API_KEY). + ElevenService() : ElevenService(ElevenEndpoints::from_env()) {} +}; + +// Output formats the OpenAPI document lists for `output_format` (codec_rate[_bitrate]). +bool valid_output_format(const std::string& format); + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/speech.cpp b/src/vendors/audio/elevenlabs/speech.cpp new file mode 100644 index 0000000..9480ab8 --- /dev/null +++ b/src/vendors/audio/elevenlabs/speech.cpp @@ -0,0 +1,97 @@ +#include "vendors/audio/elevenlabs/speech.h" + +namespace molo::audio::elevenlabs { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("elevenlabs", why); } + +std::string format_name(const SpeechRequest& r) { + int rate = r.sample_rate; + if (r.format == "mp3") { + if (rate == 0 || rate == 44100) return "mp3_44100_128"; + if (rate == 22050) return "mp3_22050_32"; + if (rate == 24000) return "mp3_24000_48"; + invalid("mp3 sample_rate must be 22050, 24000 or 44100"); + } + if (rate == 0) rate = 24000; + for (int ok : {8000, 16000, 22050, 24000, 32000, 44100, 48000}) + if (rate == ok) return r.format + "_" + std::to_string(rate); + invalid(r.format + " sample_rate must be 8000, 16000, 22050, 24000, 32000, 44100 or 48000"); +} +} // namespace + +ElevenSynthesizer::ElevenSynthesizer(std::shared_ptr hosts) + : tts_(hosts), voices_(hosts) {} + +TtsRequest ElevenSynthesizer::map(const SpeechRequest& r) const { + validate(r); + if (r.voice.empty()) invalid("a voice id is required (list voices first)"); + TtsRequest t; + t.voice_id = r.voice; + t.text = r.text; + t.model_id = r.model; + t.language_code = r.language.substr(0, 2); + if (r.speed > 0) t.voice_settings.speed = r.speed; + t.output_format = format_name(r); + return t; +} + +SpeechResult ElevenSynthesizer::synthesize(const SpeechRequest& r) { + AudioBlob blob = tts_.synthesize(map(r)); + SpeechResult out; + out.audio = std::move(blob.bytes); + out.content_type = blob.content_type.empty() ? content_type_for(r.format) : blob.content_type; + out.format = r.format; + out.provider = name(); + return out; +} + +Json ElevenSynthesizer::voices() { return voices_.list_v2(VoiceListQuery()); } + +ElevenRecognizer::ElevenRecognizer(std::shared_ptr hosts) : stt_(std::move(hosts)) {} + +SttRequest ElevenRecognizer::map(const TranscribeRequest& r) const { + validate(r); + SttRequest s; + if (!r.model.empty()) s.model_id = r.model; + s.audio = r.audio; + s.filename = r.filename; + s.content_type = r.content_type; + s.source_url = r.url; + s.language_code = r.language; + s.diarize = r.diarize; + s.keyterms = r.keywords; + return s; +} + +Transcript ElevenRecognizer::fold(const Json& doc) { + Transcript t; + t.provider = "elevenlabs"; + t.raw = doc; + auto str = [&](const char* k) { + const Json* v = doc.is_object() ? doc.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); + }; + t.text = str("text"); + t.language = str("language_code"); + if (const Json* d = doc.is_object() ? doc.find("audio_duration_secs") : nullptr; d && d->is_number()) + t.duration = d->as_number(); + if (const Json* w = doc.is_object() ? doc.find("words") : nullptr; w && w->is_array()) { + for (const Json& e : w->as_array()) { + if (!e.is_object()) continue; + const Json* type = e.find("type"); + if (type && type->is_string() && type->as_string() != "word") continue; + TranscriptWord word; + if (const Json* x = e.find("text"); x && x->is_string()) word.text = x->as_string(); + if (const Json* x = e.find("start"); x && x->is_number()) word.start = x->as_number(); + if (const Json* x = e.find("end"); x && x->is_number()) word.end = x->as_number(); + if (const Json* x = e.find("speaker_id"); x && x->is_string()) word.speaker = x->as_string(); + t.words.push_back(word); + } + } + return t; +} + +Transcript ElevenRecognizer::transcribe(const TranscribeRequest& r) { return fold(stt_.transcribe(map(r))); } + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/speech.h b/src/vendors/audio/elevenlabs/speech.h new file mode 100644 index 0000000..21d16de --- /dev/null +++ b/src/vendors/audio/elevenlabs/speech.h @@ -0,0 +1,47 @@ +#pragma once + +// ElevenLabs behind the neutral speech seam. + +#include + +#include "vendors/audio/elevenlabs/account.h" +#include "vendors/audio/elevenlabs/stt.h" +#include "vendors/audio/elevenlabs/tts.h" +#include "vendors/audio/elevenlabs/voices.h" +#include "vendors/audio/speech.h" + +namespace molo::audio::elevenlabs { + +class ElevenSynthesizer : public SpeechSynthesizer { + public: + explicit ElevenSynthesizer(std::shared_ptr hosts = ElevenEndpoints::from_env()); + std::string name() const override { return "elevenlabs"; } + bool configured() const override { return tts_.configured(); } + std::size_t max_chars() const override { return ElevenCaps::kMaxTtsChars; } + std::vector formats() const override { return {"mp3", "wav", "pcm"}; } + SpeechResult synthesize(const SpeechRequest& request) override; + Json voices() override; + + // The provider request a neutral request maps to (no network). + TtsRequest map(const SpeechRequest& request) const; + + private: + TextToSpeech tts_; + Voices voices_; +}; + +class ElevenRecognizer : public SpeechRecognizer { + public: + explicit ElevenRecognizer(std::shared_ptr hosts = ElevenEndpoints::from_env()); + std::string name() const override { return "elevenlabs"; } + bool configured() const override { return stt_.configured(); } + Transcript transcribe(const TranscribeRequest& request) override; + SttRequest map(const TranscribeRequest& request) const; + // Folds a Scribe response document into the neutral transcript. + static Transcript fold(const Json& doc); + + private: + SpeechToText stt_; +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/stt.cpp b/src/vendors/audio/elevenlabs/stt.cpp new file mode 100644 index 0000000..36e067b --- /dev/null +++ b/src/vendors/audio/elevenlabs/stt.cpp @@ -0,0 +1,66 @@ +#include "vendors/audio/elevenlabs/stt.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +clients::body::MultipartBody SpeechToText::build(const SttRequest& r) const { + require_text("model_id", r.model_id, 128); + int sources = (!r.audio.empty()) + (!r.source_url.empty()) + (!r.cloud_storage_url.empty()); + if (sources != 1) bad("give exactly one of audio, source_url and cloud_storage_url"); + if (r.audio.size() > ElevenCaps::kMaxFileBytes) bad("audio is over the byte cap"); + require_text("source_url", r.source_url, AudioCaps::kMaxFieldChars, false); + require_text("cloud_storage_url", r.cloud_storage_url, AudioCaps::kMaxFieldChars, false); + require_text("language_code", r.language_code, 16, false); + if (r.num_speakers < 0 || r.num_speakers > 32) bad("num_speakers must be 0 or 1..32"); + const std::string& g = r.timestamps_granularity; + if (!g.empty() && g != "none" && g != "word" && g != "character") + bad("timestamps_granularity must be none, word or character"); + if (!r.file_format.empty() && r.file_format != "pcm_s16le_16" && r.file_format != "other") + bad("file_format must be pcm_s16le_16 or other"); + if (r.keyterms.size() > AudioCaps::kMaxKeywords) bad("too many keyterms"); + if (r.temperature > 2.0) bad("temperature must be 0..2"); + clients::body::MultipartBody form; + form.add_text("model_id", r.model_id); + if (!r.audio.empty()) form.add_file("file", r.filename, r.content_type, r.audio); + if (!r.source_url.empty()) form.add_text("source_url", r.source_url); + if (!r.cloud_storage_url.empty()) form.add_text("cloud_storage_url", r.cloud_storage_url); + if (!r.language_code.empty()) form.add_text("language_code", r.language_code); + if (r.diarize) form.add_text("diarize", "true"); + if (r.num_speakers > 0) form.add_text("num_speakers", std::to_string(r.num_speakers)); + if (!g.empty()) form.add_text("timestamps_granularity", g); + if (r.tag_audio_events >= 0) form.add_text("tag_audio_events", r.tag_audio_events ? "true" : "false"); + for (const auto& k : r.keyterms) { + require_text("keyterm", k, 100); + form.add_text("keyterms", k); + } + if (!r.file_format.empty()) form.add_text("file_format", r.file_format); + if (r.use_multi_channel >= 0) form.add_text("use_multi_channel", r.use_multi_channel ? "true" : "false"); + if (r.seed >= 0) form.add_text("seed", std::to_string(r.seed)); + if (r.temperature >= 0) form.add_text("temperature", std::to_string(r.temperature)); + return form; +} + +Query SpeechToText::query(const SttRequest& r) const { + Query q; + if (r.enable_logging >= 0) q.add_bool("enable_logging", r.enable_logging == 1); + return q; +} + +Json SpeechToText::transcribe(const SttRequest& r) { + return call_form(C::kStt, {}, build(r), query(r)); +} + +Json SpeechToText::get(const std::string& id) { + check_id("transcription", id); + return call(C::kSttGet, {id}); +} + +Json SpeechToText::remove(const std::string& id, bool confirm) { + check_id("transcription", id); + return call_destructive(C::kSttDelete, {id}, nullptr, Query(), confirm); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/stt.h b/src/vendors/audio/elevenlabs/stt.h new file mode 100644 index 0000000..549d06f --- /dev/null +++ b/src/vendors/audio/elevenlabs/stt.h @@ -0,0 +1,43 @@ +#pragma once + +// Speech to text (Scribe): POST /v1/speech-to-text as multipart/form-data, +// plus fetching and deleting stored transcripts. + +#include +#include + +#include "vendors/audio/elevenlabs/service.h" + +namespace molo::audio::elevenlabs { + +struct SttRequest { + std::string model_id = "scribe_v2"; // required by the API; scribe_v2 is the OpenAPI example + std::string audio; // bytes; or use source_url / cloud_storage_url + std::string filename = "audio"; + std::string content_type = "application/octet-stream"; + std::string source_url; + std::string cloud_storage_url; + std::string language_code; // ISO 639-1 / 639-3, "" = detect + bool diarize = false; + int num_speakers = 0; // 0 = unset, else 1..32 + std::string timestamps_granularity; // "", none, word, character + int tag_audio_events = -1; // -1 unset + std::vector keyterms; // <= AudioCaps::kMaxKeywords + std::string file_format; // "", pcm_s16le_16, other + int use_multi_channel = -1; + long seed = -1; + double temperature = -1; // 0..2 + int enable_logging = -1; // query: -1 unset, 0 false +}; + +class SpeechToText : public ElevenService { + public: + using ElevenService::ElevenService; + clients::body::MultipartBody build(const SttRequest& request) const; + Query query(const SttRequest& request) const; + Json transcribe(const SttRequest& request); + Json get(const std::string& transcription_id); + Json remove(const std::string& transcription_id, bool confirm); +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/tts.cpp b/src/vendors/audio/elevenlabs/tts.cpp new file mode 100644 index 0000000..2c909bd --- /dev/null +++ b/src/vendors/audio/elevenlabs/tts.cpp @@ -0,0 +1,110 @@ +#include "vendors/audio/elevenlabs/tts.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +bool VoiceSettings::empty() const { + return stability < 0 && similarity_boost < 0 && style < 0 && speed < 0 && use_speaker_boost < 0; +} + +Json VoiceSettings::to_json() const { + auto unit = [](double v, const char* what) { + if (v > 1.0) throw llm::ApiError::invalid("elevenlabs", std::string(what) + " must be 0..1"); + }; + Json o = Json::object(); + if (stability >= 0) { unit(stability, "stability"); o.set("stability", Json(stability)); } + if (similarity_boost >= 0) { unit(similarity_boost, "similarity_boost"); o.set("similarity_boost", Json(similarity_boost)); } + if (style >= 0) { unit(style, "style"); o.set("style", Json(style)); } + if (speed >= 0) { + if (speed == 0) + throw llm::ApiError::invalid("elevenlabs", "voice speed must be above 0"); + o.set("speed", Json(speed)); + } + if (use_speaker_boost >= 0) o.set("use_speaker_boost", Json(use_speaker_boost == 1)); + return o; +} + +Json TextToSpeech::build(const TtsRequest& r) const { + check_id("voice", r.voice_id); + require_text("text", r.text, ElevenCaps::kMaxTtsChars); + require_text("model_id", r.model_id, 128, false); + require_text("language_code", r.language_code, 16, false); + require_text("previous_text", r.previous_text, ElevenCaps::kMaxTtsChars, false); + require_text("next_text", r.next_text, ElevenCaps::kMaxTtsChars, false); + Json o = Json::object(); + o.set("text", Json(r.text)); + if (!r.model_id.empty()) o.set("model_id", Json(r.model_id)); + if (!r.language_code.empty()) o.set("language_code", Json(r.language_code)); + if (!r.voice_settings.empty()) o.set("voice_settings", r.voice_settings.to_json()); + if (r.seed >= 0) { + if (r.seed > 4294967295L) bad("seed must be 0..4294967295"); + o.set("seed", Json(r.seed)); + } + if (!r.previous_text.empty()) o.set("previous_text", Json(r.previous_text)); + if (!r.next_text.empty()) o.set("next_text", Json(r.next_text)); + if (!r.apply_text_normalization.empty()) { + const std::string& n = r.apply_text_normalization; + if (n != "auto" && n != "on" && n != "off") bad("apply_text_normalization must be auto, on or off"); + o.set("apply_text_normalization", Json(n)); + } + if (!r.pronunciation_dictionaries.empty()) { + if (r.pronunciation_dictionaries.size() > 3) bad("at most 3 pronunciation dictionaries"); + Json arr = Json::array(); + for (const auto& [id, version] : r.pronunciation_dictionaries) { + check_id("pronunciation dictionary", id); + Json loc = Json::object(); + loc.set("pronunciation_dictionary_id", Json(id)); + if (!version.empty()) { + check_id("pronunciation dictionary version", version); + loc.set("version_id", Json(version)); + } + arr.push(loc); + } + o.set("pronunciation_dictionary_locators", arr); + } + return o; +} + +Query TextToSpeech::query(const TtsRequest& r) const { + Query q; + if (!r.output_format.empty()) { + if (!valid_output_format(r.output_format)) bad("unknown output_format: " + r.output_format); + q.add("output_format", r.output_format); + } + if (r.optimize_streaming_latency >= 0) { + if (r.optimize_streaming_latency > 4) bad("optimize_streaming_latency must be 0..4"); + q.add_int("optimize_streaming_latency", r.optimize_streaming_latency); + } + if (r.enable_logging >= 0) q.add_bool("enable_logging", r.enable_logging == 1); + return q; +} + +std::string TextToSpeech::describe(const TtsRequest& r, bool streaming) const { + Json body = build(r); + return preview(streaming ? C::kTtsStream : C::kTts, {r.voice_id}, &body, query(r)); +} + +AudioBlob TextToSpeech::synthesize(const TtsRequest& r) { + Json body = build(r); + llm::RawResponse raw = call_raw(C::kTts, {r.voice_id}, &body, query(r)); + return AudioBlob{raw.body, raw.content_type}; +} + +ByteStreamer::Result TextToSpeech::stream(const TtsRequest& r, const ChunkSink& sink) { + Json body = build(r); + return call_stream(C::kTtsStream, {r.voice_id}, &body, query(r), sink); +} + +Json TextToSpeech::synthesize_with_timestamps(const TtsRequest& r) { + Json body = build(r); + return call(C::kTtsTimestamps, {r.voice_id}, &body, query(r)); +} + +ByteStreamer::Result TextToSpeech::stream_with_timestamps(const TtsRequest& r, const ChunkSink& sink) { + Json body = build(r); + // The streamed variant is declared "bytes" in the table (newline delimited JSON). + return call_stream(C::kTtsStreamTimestamps, {r.voice_id}, &body, query(r), sink); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/tts.h b/src/vendors/audio/elevenlabs/tts.h new file mode 100644 index 0000000..a80f884 --- /dev/null +++ b/src/vendors/audio/elevenlabs/tts.h @@ -0,0 +1,64 @@ +#pragma once + +// Text to speech: buffered, streamed (HTTP chunked bytes) and with +// character timestamps. WebSocket input streaming is TtsInputStream. + +#include +#include +#include +#include + +#include "vendors/audio/elevenlabs/service.h" + +namespace molo::audio::elevenlabs { + +struct VoiceSettings { + // Negative = not set (the voice's stored settings apply). + double stability = -1; + double similarity_boost = -1; + double style = -1; + double speed = -1; + int use_speaker_boost = -1; // -1 unset, 0 false, 1 true + bool empty() const; + Json to_json() const; // throws llm::ApiError::invalid when a value is out of range +}; + +struct TtsRequest { + std::string voice_id; // required, path segment + std::string text; // 1..ElevenCaps::kMaxTtsChars + std::string model_id; // "" = server default (eleven_multilingual_v2) + std::string language_code; // ISO 639-1 + VoiceSettings voice_settings; + long seed = -1; // 0..4294967295, -1 unset + std::string previous_text, next_text; + std::string apply_text_normalization; // "", auto, on, off + std::vector> pronunciation_dictionaries; // (id, version) <= 3 + // Query parameters. + std::string output_format; // "" = server default mp3_44100_128 + int optimize_streaming_latency = -1; // 0..4, -1 unset + int enable_logging = -1; // -1 unset, 0 false (zero retention), 1 true +}; + +struct AudioBlob { + std::string bytes; + std::string content_type; +}; + +class TextToSpeech : public ElevenService { + public: + using ElevenService::ElevenService; + + Json build(const TtsRequest& request) const; // body (validates) + Query query(const TtsRequest& request) const; // query (validates) + AudioBlob synthesize(const TtsRequest& request); + // Incremental audio over HTTP (/stream); `sink` returns false to stop. + ByteStreamer::Result stream(const TtsRequest& request, const ChunkSink& sink); + // JSON with audio_base64 and alignment / normalized_alignment. + Json synthesize_with_timestamps(const TtsRequest& request); + // Newline delimited JSON objects, each with audio_base64 and alignment. + ByteStreamer::Result stream_with_timestamps(const TtsRequest& request, const ChunkSink& sink); + // Request preview without sending (path with query, body). + std::string describe(const TtsRequest& request, bool streaming = false) const; +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/tts_stream.cpp b/src/vendors/audio/elevenlabs/tts_stream.cpp new file mode 100644 index 0000000..6f0d93f --- /dev/null +++ b/src/vendors/audio/elevenlabs/tts_stream.cpp @@ -0,0 +1,149 @@ +#include "vendors/audio/elevenlabs/tts_stream.h" + +#include "security/crypto/Base64.h" + +namespace molo::audio::elevenlabs { + +namespace { + +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("elevenlabs", why); } + +std::string build_query(const TtsStreamOptions& o) { + Query q; + q.add_if("model_id", o.model_id); + if (!o.output_format.empty()) { + if (!valid_output_format(o.output_format)) invalid("unknown output_format: " + o.output_format); + q.add("output_format", o.output_format); + } + q.add_if("language_code", o.language_code); + if (o.inactivity_timeout != 0) { + if (o.inactivity_timeout < 1 || o.inactivity_timeout > 180) invalid("inactivity_timeout must be 1..180"); + q.add_int("inactivity_timeout", o.inactivity_timeout); + } + if (o.sync_alignment >= 0) q.add_bool("sync_alignment", o.sync_alignment == 1); + if (o.auto_mode >= 0) q.add_bool("auto_mode", o.auto_mode == 1); + if (!o.apply_text_normalization.empty()) { + const std::string& n = o.apply_text_normalization; + if (n != "auto" && n != "on" && n != "off") invalid("apply_text_normalization must be auto, on or off"); + q.add("apply_text_normalization", n); + } + if (o.seed >= 0) { + if (o.seed > 4294967295L) invalid("seed must be 0..4294967295"); + q.add_int("seed", o.seed); + } + if (o.enable_ssml_parsing >= 0) q.add_bool("enable_ssml_parsing", o.enable_ssml_parsing == 1); + if (o.enable_logging >= 0) q.add_bool("enable_logging", o.enable_logging == 1); + std::string s = q.str(); + return s.empty() ? s : s.substr(1); +} + +} // namespace + +Vocabulary TtsInputStream::vocabulary() { + // The stream-input messages carry no discriminator field: they are plain + // {"text": ...} objects, so both sets are empty and sends are untyped. + return Vocabulary("type", {}, {}); +} + +TtsInputStream::TtsInputStream(std::string api_key, TtsStreamOptions options) + : options_(std::move(options)), api_key_(std::move(api_key)) { + if (!llm::Service::valid_id(options_.voice_id)) invalid("bad voice id"); + std::string base = options_.url.empty() ? ElevenEndpoints::ws_base() + "/v1/text-to-speech/" + + options_.voice_id + "/stream-input" + : options_.url; + std::string url = AudioSocket::with_query(base, build_query(options_)); + socket_ = std::make_unique("elevenlabs", url, ElevenEndpoints::ws_headers(api_key_), + vocabulary(), options_.socket); +} + +std::string TtsInputStream::url() const { return socket_->url(); } + +void TtsInputStream::connect() { + socket_->connect(); + Json init = Json::object(); + init.set("text", Json(" ")); + if (!options_.voice_settings.empty()) init.set("voice_settings", options_.voice_settings.to_json()); + if (!options_.chunk_length_schedule.empty()) { + if (options_.chunk_length_schedule.size() > 8) invalid("chunk_length_schedule has at most 8 entries"); + Json sched = Json::array(); + for (int v : options_.chunk_length_schedule) { + if (v < 50 || v > 500) invalid("chunk_length_schedule entries must be 50..500"); + sched.push(Json(v)); + } + Json gen = Json::object(); + gen.set("chunk_length_schedule", sched); + init.set("generation_config", gen); + } + socket_->send_json(init, true); +} + +void TtsInputStream::send_text(const std::string& text, bool try_trigger_generation) { + if (text.empty() || text.size() > ElevenCaps::kMaxTtsChars) invalid("text must be 1..40000 characters"); + if (text.find('\0') != std::string::npos) invalid("text has a NUL byte"); + Json m = Json::object(); + m.set("text", Json(text.back() == ' ' ? text : text + " ")); + if (try_trigger_generation) m.set("try_trigger_generation", Json(true)); + socket_->send_json(m, true); +} + +void TtsInputStream::flush() { + Json m = Json::object(); + m.set("text", Json("")); + m.set("flush", Json(true)); + socket_->send_json(m, true); +} + +void TtsInputStream::finish() { + Json m = Json::object(); + m.set("text", Json("")); + socket_->send_json(m, true); +} + +void TtsInputStream::fold(const SocketEvent& ev, TtsStreamResult& out) { + if (ev.kind == SocketEvent::Kind::kClose) { + out.closed = true; + return; + } + if (ev.kind != SocketEvent::Kind::kText || !ev.json_ok || !ev.json.is_object()) return; + const Json& j = ev.json; + if (const Json* audio = j.find("audio"); audio && audio->is_string()) { + std::string raw; + if (!security::crypto::Base64::decode(audio->as_string(), raw)) + throw llm::ApiError("elevenlabs", 502, "BAD_RESPONSE", "audio chunk is not valid base64"); + if (out.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { + out.truncated = true; + } else { + out.audio += raw; + ++out.chunks; + } + } + if (const Json* al = j.find("alignment"); al && al->is_object()) out.alignments.push_back(*al); + if (const Json* fin = j.find("isFinal"); fin && fin->is_bool() && fin->as_bool()) out.is_final = true; + if (!j.find("audio") && !j.find("isFinal")) { + for (const char* k : {"message", "error", "detail"}) { + if (const Json* e = j.find(k); e && e->is_string()) { + out.error = e->as_string(); + break; + } + } + } +} + +TtsStreamResult TtsInputStream::collect(int max_wait_ms) { + TtsStreamResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + fold(*ev, out); + if (out.is_final || out.closed || !out.error.empty() || out.truncated) return out; + } + return out; +} + +TtsStreamResult TtsInputStream::speak(const std::string& text, int max_wait_ms) { + send_text(text); + finish(); + return collect(max_wait_ms); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/tts_stream.h b/src/vendors/audio/elevenlabs/tts_stream.h new file mode 100644 index 0000000..781e957 --- /dev/null +++ b/src/vendors/audio/elevenlabs/tts_stream.h @@ -0,0 +1,77 @@ +#pragma once + +// ElevenLabs text to speech over WebSocket (stream-input): send text as it +// is produced (for example by an LLM), receive base64 audio chunks with +// optional character alignment. Channel and message shapes follow +// https://elevenlabs.io/docs/asyncapi/api-reference-2.json. + +#include +#include +#include +#include + +#include "vendors/audio/elevenlabs/tts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::elevenlabs { + +struct TtsStreamOptions { + std::string voice_id; // required + std::string model_id; // "" = server default + std::string output_format; // "" = mp3_44100_128 + std::string language_code; + int inactivity_timeout = 0; // seconds, 0 unset, else 1..180 + int sync_alignment = -1; + int auto_mode = -1; + std::string apply_text_normalization; // "", auto, on, off + long seed = -1; + int enable_ssml_parsing = -1; + int enable_logging = -1; + VoiceSettings voice_settings; // sent in the first message + std::vector chunk_length_schedule; // generation_config, each 50..500, at most 8 + std::string url; // "" = wss://api.elevenlabs.io/v1/text-to-speech/{voice}/stream-input + SocketOptions socket; +}; + +struct TtsStreamResult { + std::string audio; // concatenated decoded chunks + std::size_t chunks = 0; + std::vector alignments; // alignment objects when sync_alignment is on + bool is_final = false; // {"isFinal":true} seen + bool truncated = false; // audio cap hit + std::string error; // server error message, "" when none + bool closed = false; +}; + +class TtsInputStream { + public: + TtsInputStream(std::string api_key, TtsStreamOptions options); + + std::string url() const; // as opened, without any credential + void connect(); // sends the initial " " message with voice settings + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + // Text must end with a space per the docs; added when missing. 1..40000 chars. + void send_text(const std::string& text, bool try_trigger_generation = false); + void flush(); // {"text":"","flush":true} + void finish(); // {"text":""}: end of input, server closes after the last audio + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + // Reads until isFinal / error / close or `max_wait_ms` with no event. + TtsStreamResult collect(int max_wait_ms); + // send_text + finish + collect. + TtsStreamResult speak(const std::string& text, int max_wait_ms); + + const AudioSocket& socket() const { return *socket_; } + + static Vocabulary vocabulary(); + + private: + void fold(const SocketEvent& ev, TtsStreamResult& out); + + TtsStreamOptions options_; + std::string api_key_; + std::unique_ptr socket_; +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/voices.cpp b/src/vendors/audio/elevenlabs/voices.cpp new file mode 100644 index 0000000..f312b8e --- /dev/null +++ b/src/vendors/audio/elevenlabs/voices.cpp @@ -0,0 +1,145 @@ +#include "vendors/audio/elevenlabs/voices.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::elevenlabs { + +using C = ElevenCalls; + +namespace { +Query list_query(const AudioService& svc, const VoiceListQuery& q) { + Query out; + if (q.page_size < 0 || q.page_size > ElevenCaps::kMaxPageSize) svc.bad("page_size must be 1..100"); + if (q.page_size > 0) out.add_int("page_size", q.page_size); + out.add_if("next_page_token", q.next_page_token); + out.add_if("search", q.search); + if (!q.sort.empty() && q.sort != "created_at_unix" && q.sort != "name") + svc.bad("sort must be created_at_unix or name"); + out.add_if("sort", q.sort); + if (!q.sort_direction.empty() && q.sort_direction != "asc" && q.sort_direction != "desc") + svc.bad("sort_direction must be asc or desc"); + out.add_if("sort_direction", q.sort_direction); + out.add_if("voice_type", q.voice_type); + out.add_if("category", q.category); + if (q.include_total_count >= 0) out.add_bool("include_total_count", q.include_total_count == 1); + return out; +} +} // namespace + +Json Voices::list(bool show_legacy) { + Query q; + if (show_legacy) q.add_bool("show_legacy", true); + return call(C::kVoices, {}, nullptr, q); +} + +Json Voices::list_v2(const VoiceListQuery& q) { return call(C::kVoicesV2, {}, nullptr, list_query(*this, q)); } + +VoiceListing Voices::list_all_v2(VoiceListQuery q, std::size_t max_pages) { + if (max_pages == 0 || max_pages > AudioCaps::kMaxPages) bad("max_pages must be 1.." + std::to_string(AudioCaps::kMaxPages)); + VoiceListing out; + for (std::size_t page = 0; page < max_pages; ++page) { + Json doc = list_v2(q); + ++out.pages; + if (const Json* v = doc.find("voices"); v && v->is_array()) + for (const Json& e : v->as_array()) { + if (out.voices.as_array().size() >= AudioCaps::kMaxListItems) { + out.truncated = true; + return out; + } + out.voices.push(e); + } + if (const Json* t = doc.find("total_count"); t && t->is_number()) out.total_count = t->as_int(); + const Json* more = doc.find("has_more"); + const Json* token = doc.find("next_page_token"); + if (!more || !more->is_bool() || !more->as_bool() || !token || !token->is_string() || + token->as_string().empty()) + return out; + q.next_page_token = token->as_string(); + } + out.truncated = true; // loud: the page cap stopped the walk while more pages existed + return out; +} + +Json Voices::get(const std::string& id, bool with_settings) { + check_id("voice", id); + Query q; + if (with_settings) q.add_bool("with_settings", true); + return call(C::kVoiceGet, {id}, nullptr, q); +} + +Json Voices::settings(const std::string& id) { + check_id("voice", id); + return call(C::kVoiceSettings, {id}); +} + +Json Voices::default_settings() { return call(C::kVoiceSettingsDefault); } + +Json Voices::shared_library(const Json& query_object) { + return call(C::kSharedVoices, {}, nullptr, query_from(C::kSharedVoices, query_object)); +} + +Json Voices::design_build(const VoiceDesign& d) const { + if (d.voice_description.size() < 20 || d.voice_description.size() > 1000) + bad("voice_description must be 20..1000 characters"); + Json o = Json::object(); + o.set("voice_description", Json(d.voice_description)); + if (d.auto_generate_text) { + if (!d.text.empty()) bad("give text or auto_generate_text, not both"); + o.set("auto_generate_text", Json(true)); + } else if (!d.text.empty()) { + if (d.text.size() < 100 || d.text.size() > 1000) bad("text must be 100..1000 characters"); + o.set("text", Json(d.text)); + } + if (!d.model_id.empty()) { + if (d.model_id != "eleven_multilingual_ttv_v2" && d.model_id != "eleven_ttv_v3") + bad("model_id must be eleven_multilingual_ttv_v2 or eleven_ttv_v3"); + o.set("model_id", Json(d.model_id)); + } + if (d.loudness > -2) { require_range("loudness", d.loudness, -1, 1); o.set("loudness", Json(d.loudness)); } + if (d.seed >= 0) { + if (d.seed > 2147483647L) bad("seed must be 0..2147483647"); + o.set("seed", Json(d.seed)); + } + if (d.guidance_scale >= 0) { require_range("guidance_scale", d.guidance_scale, 0, 100); o.set("guidance_scale", Json(d.guidance_scale)); } + if (d.quality > -2) { require_range("quality", d.quality, -1, 1); o.set("quality", Json(d.quality)); } + if (d.stream_previews >= 0) o.set("stream_previews", Json(d.stream_previews == 1)); + if (d.should_enhance >= 0) o.set("should_enhance", Json(d.should_enhance == 1)); + return o; +} + +Json Voices::design(const VoiceDesign& d) { + Json body = design_build(d); + Query q; + if (!d.output_format.empty()) { + if (!valid_output_format(d.output_format)) bad("unknown output_format: " + d.output_format); + q.add("output_format", d.output_format); + } + return call(C::kVoiceDesign, {}, &body, q); +} + +AudioBlob Voices::design_preview_audio(const std::string& id) { + check_id("generated voice", id); + llm::RawResponse raw = call_raw(C::kVoiceDesignPreview, {id}); + return AudioBlob{raw.body, raw.content_type}; +} + +Json Voices::save_design(const std::string& name, const std::string& description, + const std::string& generated_voice_id, const Json& labels, bool confirm) { + require_text("voice_name", name, 200); + if (description.size() < 20 || description.size() > 1000) bad("voice_description must be 20..1000 characters"); + check_id("generated voice", generated_voice_id); + require_confirm("saving a voice", confirm); + Json o = Json::object(); + o.set("voice_name", Json(name)); + o.set("voice_description", Json(description)); + o.set("generated_voice_id", Json(generated_voice_id)); + if (labels.is_object()) o.set("labels", labels); + return call(C::kVoiceDesignSave, {}, &o); +} + +Json Voices::remove(const std::string& id, bool confirm) { + check_id("voice", id); + return call_destructive(C::kVoiceDelete, {id}, nullptr, Query(), confirm); +} + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/elevenlabs/voices.h b/src/vendors/audio/elevenlabs/voices.h new file mode 100644 index 0000000..4e95e4a --- /dev/null +++ b/src/vendors/audio/elevenlabs/voices.h @@ -0,0 +1,65 @@ +#pragma once + +// Voices: the read side (list, get, settings, shared library), voice design +// (text description to voice previews, then save) and the destructive +// delete. Cloning (add_voice, PVC) creates resources and is reached through +// the call table with confirm, not through a typed shortcut. + +#include + +#include "vendors/audio/elevenlabs/service.h" +#include "vendors/audio/elevenlabs/tts.h" + +namespace molo::audio::elevenlabs { + +struct VoiceListQuery { + int page_size = 0; // 0 unset, else 1..100 + std::string next_page_token; + std::string search, sort, sort_direction; // sort: created_at_unix | name; direction asc | desc + std::string voice_type; // personal community default workspace non-default non-community saved + std::string category; // premade cloned generated professional + int include_total_count = -1; +}; + +struct VoiceListing { + Json voices = Json::array(); + bool truncated = false; // page cap reached while a next page existed + std::size_t pages = 0; + long total_count = -1; +}; + +struct VoiceDesign { + std::string voice_description; // 20..1000 characters + std::string text; // "" with auto_generate_text, else 100..1000 + bool auto_generate_text = false; + std::string model_id; // "", eleven_multilingual_ttv_v2, eleven_ttv_v3 + double loudness = -2; // -1..1, -2 unset + long seed = -1; // 0..2147483647 + double guidance_scale = -1; // 0..100 + double quality = -2; // -1..1 + int stream_previews = -1; + int should_enhance = -1; + std::string output_format; // query +}; + +class Voices : public ElevenService { + public: + using ElevenService::ElevenService; + + Json list(bool show_legacy = false); // GET /v1/voices + Json list_v2(const VoiceListQuery& query); // GET /v2/voices, one page + VoiceListing list_all_v2(VoiceListQuery query, std::size_t max_pages); + Json get(const std::string& voice_id, bool with_settings = false); + Json settings(const std::string& voice_id); + Json default_settings(); + Json shared_library(const Json& query_object); // GET /v1/shared-voices + Json design_build(const VoiceDesign& design) const; // body + Json design(const VoiceDesign& design); // POST /v1/text-to-voice/design + AudioBlob design_preview_audio(const std::string& generated_voice_id); + // Creates a voice from a design preview: a write, needs confirm. + Json save_design(const std::string& name, const std::string& description, + const std::string& generated_voice_id, const Json& labels, bool confirm); + Json remove(const std::string& voice_id, bool confirm); +}; + +} // namespace molo::audio::elevenlabs diff --git a/src/vendors/audio/hosts.cpp b/src/vendors/audio/hosts.cpp new file mode 100644 index 0000000..0ea7e63 --- /dev/null +++ b/src/vendors/audio/hosts.cpp @@ -0,0 +1,153 @@ +#include "vendors/audio/hosts.h" + +#include "clients/transports/http_transport.h" +#include "llm/config.h" +#include "vendors/audio/caps.h" + +namespace molo::audio { + +namespace { +clients::ClientConfig stream_client_config() { + clients::ClientConfig c; + c.persist_cookies = false; + return c; +} +} // namespace + +ByteStreamer::ByteStreamer(std::unique_ptr transport, + llm::ApiClientConfig config, std::string key) + : config_(std::move(config)), + key_(std::move(key)), + client_(std::move(transport), stream_client_config()) {} + +ByteStreamer::Result ByteStreamer::stream(clients::Method method, const std::string& path, + const Json* body, const ChunkSink& sink, + const llm::Headers& extra) { + if (!configured()) throw llm::NotConfiguredError(config_.vendor, config_.key_env); + if (path.empty() || path.front() != '/' || path.find("..") != std::string::npos) + throw llm::ApiError::invalid(config_.vendor, "bad request path"); + Result result; + std::string probe; // leading bytes held back while they look like a JSON error + bool probing = true; + bool json_like = false; + bool stopped = false; + auto forward = [&](const std::string& chunk) -> bool { + if (chunk.empty()) return true; + if (result.bytes + chunk.size() > AudioCaps::kMaxStreamAudioBytes) { + result.truncated = true; + stopped = true; + return false; + } + result.bytes += chunk.size(); + if (!sink(chunk)) { + stopped = true; + return false; + } + return true; + }; + try { + clients::ClientRequest req(method, clients::Url::parse(config_.base_url + path)); + if (config_.auth == llm::AuthStyle::kBearer) req.bearer(key_); + else req.header(config_.auth_header, key_); + for (const auto& [k, v] : config_.default_headers) req.header(k, v); + for (const auto& [k, v] : extra) req.header(k, v); + if (body) req.json(*body); + client_.send_stream_blocking(std::move(req), [&](const std::string& chunk) -> bool { + if (!probing) return forward(chunk); + probe += chunk; + if (!json_like) { + std::size_t at = probe.find_first_not_of(" \t\r\n"); + if (at == std::string::npos) return true; + json_like = probe[at] == '{'; + if (!json_like) { + probing = false; + std::string held; + held.swap(probe); + return forward(held); + } + } + if (probe.size() > AudioCaps::kErrorProbeBytes) { // too big to be an error document + probing = false; + std::string held; + held.swap(probe); + return forward(held); + } + return true; + }); + if (probing && !probe.empty()) { + // The whole body fit the probe window. A JSON object with an error shape is the vendor's error. + try { + Json doc = Json::parse(probe); + if (doc.is_object() && (doc.find("detail") || doc.find("error") || doc.find("message") || + doc.find("err_code") || doc.find("err_msg"))) { + std::string msg = probe.substr(0, 300); + throw llm::ApiError(config_.vendor, 400, "stream_error", msg); + } + } catch (const clients::body::JsonError&) { + } + forward(probe); + } + } catch (const llm::ApiError&) { + throw; + } catch (const std::exception& e) { + if (!stopped) throw llm::ApiError(config_.vendor, 502, "UNAVAILABLE", e.what()); + } + return result; +} + +Hosts& Hosts::add(const std::string& name, AudioHost host) { + if (hosts_.empty()) first_ = name; + hosts_[name] = std::move(host); + return *this; +} + +const AudioHost* Hosts::find(const std::string& name) const { + auto it = hosts_.find(name); + return it == hosts_.end() ? nullptr : &it->second; +} + +const AudioHost& Hosts::get(const std::string& name, const std::string& vendor) const { + const AudioHost* h = find(name); + if (!h) throw llm::ApiError::invalid(vendor, "no such host: " + name); + return *h; +} + +bool Hosts::any_configured() const { + for (const auto& [name, h] : hosts_) + if (h.configured()) return true; + return false; +} + +std::string allowlisted_base(const char* env_name, const std::string& fallback, + std::initializer_list allowed) { + std::string v = llm::Config::get(env_name); + while (!v.empty() && v.back() == '/') v.pop_back(); + for (const char* a : allowed) + if (v == a) return v; + return fallback; +} + +std::string to_ws_url(const std::string& http_url) { + if (http_url.rfind("https://", 0) == 0) return "wss://" + http_url.substr(8); + if (http_url.rfind("http://", 0) == 0) return "ws://" + http_url.substr(7); + return http_url; +} + +AudioHost HostFactory::from_env(llm::ApiClientConfig config, const std::string& key_prefix) { + std::string key = llm::Config::get(config.key_env.c_str()); + if (!key.empty()) key = key_prefix + key; + return with_transports(std::make_unique(), + std::make_unique(), std::move(config), + key); +} + +AudioHost HostFactory::with_transports(std::unique_ptr api_transport, + std::unique_ptr stream_transport, + llm::ApiClientConfig config, const std::string& key) { + AudioHost host; + host.api = std::make_shared(std::move(api_transport), config, key); + host.streamer = std::make_shared(std::move(stream_transport), config, key); + return host; +} + +} // namespace molo::audio diff --git a/src/vendors/audio/hosts.h b/src/vendors/audio/hosts.h new file mode 100644 index 0000000..5192655 --- /dev/null +++ b/src/vendors/audio/hosts.h @@ -0,0 +1,91 @@ +#pragma once + +// One provider talks to one or more hosts (api, gateway, streaming, ...). +// Each host has an ApiClient for buffered JSON / bytes calls and a +// ByteStreamer for incremental bodies (streamed audio). Both authenticate the +// same way and refuse to send without a key. + +#include +#include +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/client.h" +#include "llm/api_client.h" + +namespace molo::audio { + +using Json = clients::body::Json; +using ChunkSink = std::function; + +// Incremental POST/GET bodies. The shared client reports no status for a +// stream, so a small JSON document where audio was expected is treated as the +// vendor's error (same rule llm::SseDecoder applies to SSE streams). +class ByteStreamer { + public: + ByteStreamer(std::unique_ptr transport, llm::ApiClientConfig config, + std::string key); + bool configured() const { return !key_.empty(); } + const llm::ApiClientConfig& config() const { return config_; } + + // Streams the response body to `sink` (return false to stop). Total bytes + // are capped by AudioCaps::kMaxStreamAudioBytes: past it the stream stops + // and `truncated` is set. Throws llm::ApiError for a JSON error body. + struct Result { + std::size_t bytes = 0; + bool truncated = false; + }; + Result stream(clients::Method method, const std::string& path, const Json* body, + const ChunkSink& sink, const llm::Headers& extra = {}); + + private: + llm::ApiClientConfig config_; + std::string key_; + clients::Client client_; +}; + +struct AudioHost { + std::shared_ptr api; + std::shared_ptr streamer; + bool configured() const { return api && api->configured(); } +}; + +class Hosts { + public: + Hosts() = default; + Hosts& add(const std::string& name, AudioHost host); + const AudioHost& get(const std::string& name, const std::string& vendor) const; + const AudioHost* find(const std::string& name) const; + // The first host added: what llm::Service uses as its default client. + const AudioHost& first() const { return hosts_.at(first_); } + bool empty() const { return hosts_.empty(); } + bool any_configured() const; + const std::map& all() const { return hosts_; } + + private: + std::map hosts_; + std::string first_; +}; + +// Base URL override from the environment/config, accepted only when it is +// exactly one of `allowed` (regional endpoints); anything else falls back to +// `fallback` so a stray value can never redirect a credential to a new host. +std::string allowlisted_base(const char* env_name, const std::string& fallback, + std::initializer_list allowed); +// "https://h/x" -> "wss://h/x", "http://" -> "ws://". +std::string to_ws_url(const std::string& http_url); + +class HostFactory { + public: + // Real HTTPS transports. The key comes from llm::Config::get(config.key_env); + // `key_prefix` is prepended when the key is non-empty (Deepgram "Token "). + static AudioHost from_env(llm::ApiClientConfig config, const std::string& key_prefix = ""); + // Injected transports (tests): one transport per client. + static AudioHost with_transports(std::unique_ptr api_transport, + std::unique_ptr stream_transport, + llm::ApiClientConfig config, const std::string& key); +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/hume/calls.cpp b/src/vendors/audio/hume/calls.cpp new file mode 100644 index 0000000..20eac56 --- /dev/null +++ b/src/vendors/audio/hume/calls.cpp @@ -0,0 +1,68 @@ +#include "vendors/audio/hume/calls.h" + +namespace molo::audio::hume { + +// One row per REST call. tools/check_spec.py parses the block between the +// markers: {id, method, path, host, body, fields, required, query, response, source}. +// CALLS-BEGIN +static const CallSpec kRows[] = { + {"voices_list", "GET", "/v0/tts/voices", "api", "-", "", "", "provider,page_number,page_size,ascending_order,filter_tag", "json", "openapi-tts:voices_list"}, + {"voices_create", "POST", "/v0/tts/voices", "api", "json", "generation_id,name", "generation_id,name", "", "json", "openapi-tts:voices_create"}, + {"voices_delete", "DELETE", "/v0/tts/voices", "api", "-", "", "", "name", "json", "openapi-tts:voices_delete"}, + {"controlPlane_send", "POST", "/v0/evi/chat/{chat_id}/send", "api", "json", "audio,builtin_tools,context,custom_session_id,language_model_api_key,metadata,system_prompt,tools,type,variables,voice_id,text,content,tool_call_id,tool_name,tool_type,code,error,level", "", "", "json", "openapi-evi:controlPlane_send"}, + {"chatGroups_list-chat-groups", "GET", "/v0/evi/chat_groups", "api", "-", "", "", "page_number,page_size,ascending_order,config_id", "json", "openapi-evi:chatGroups_list-chat-groups"}, + {"chatGroups_get-chat-group", "GET", "/v0/evi/chat_groups/{id}", "api", "-", "", "", "status,page_size,page_number,ascending_order", "json", "openapi-evi:chatGroups_get-chat-group"}, + {"chatGroups_get-audio", "GET", "/v0/evi/chat_groups/{id}/audio", "api", "-", "", "", "page_number,page_size,ascending_order", "json", "openapi-evi:chatGroups_get-audio"}, + {"chatGroups_list-chat-group-events", "GET", "/v0/evi/chat_groups/{id}/events", "api", "-", "", "", "page_size,page_number,ascending_order", "json", "openapi-evi:chatGroups_list-chat-group-events"}, + {"chats_list-chats", "GET", "/v0/evi/chats", "api", "-", "", "", "page_number,page_size,ascending_order,config_id,status", "json", "openapi-evi:chats_list-chats"}, + {"chats_list-chat-events", "GET", "/v0/evi/chats/{id}", "api", "-", "", "", "page_size,page_number,ascending_order", "json", "openapi-evi:chats_list-chat-events"}, + {"chats_get-audio", "GET", "/v0/evi/chats/{id}/audio", "api", "-", "", "", "", "json", "openapi-evi:chats_get-audio"}, + {"configs_list-configs", "GET", "/v0/evi/configs", "api", "-", "", "", "page_number,page_size,restrict_to_most_recent,name", "json", "openapi-evi:configs_list-configs"}, + {"configs_create-config", "POST", "/v0/evi/configs", "api", "json", "builtin_tools,ellm_model,event_messages,evi_version,interruption,language_model,name,nudges,prompt,timeouts,tools,turn_detection,version_description,voice,webhooks", "evi_version,name", "", "json", "openapi-evi:configs_create-config"}, + {"configs_list-config-versions", "GET", "/v0/evi/configs/{id}", "api", "-", "", "", "page_number,page_size,restrict_to_most_recent", "json", "openapi-evi:configs_list-config-versions"}, + {"configs_create-config-version", "POST", "/v0/evi/configs/{id}", "api", "json", "builtin_tools,ellm_model,event_messages,evi_version,interruption,language_model,nudges,prompt,timeouts,tools,turn_detection,version_description,voice,webhooks", "evi_version", "", "json", "openapi-evi:configs_create-config-version"}, + {"configs_update-config-name", "PATCH", "/v0/evi/configs/{id}", "api", "json", "name", "name", "", "json", "openapi-evi:configs_update-config-name"}, + {"configs_delete-config", "DELETE", "/v0/evi/configs/{id}", "api", "-", "", "", "", "json", "openapi-evi:configs_delete-config"}, + {"configs_get-config-version", "GET", "/v0/evi/configs/{id}/version/{version}", "api", "-", "", "", "", "json", "openapi-evi:configs_get-config-version"}, + {"configs_update-config-description", "PATCH", "/v0/evi/configs/{id}/version/{version}", "api", "json", "version_description", "", "", "json", "openapi-evi:configs_update-config-description"}, + {"configs_delete-config-version", "DELETE", "/v0/evi/configs/{id}/version/{version}", "api", "-", "", "", "", "json", "openapi-evi:configs_delete-config-version"}, + {"prompts_list-prompts", "GET", "/v0/evi/prompts", "api", "-", "", "", "page_number,page_size,restrict_to_most_recent,name", "json", "openapi-evi:prompts_list-prompts"}, + {"prompts_create-prompt", "POST", "/v0/evi/prompts", "api", "json", "name,prompt_expansion,text,version_description", "name,text", "", "json", "openapi-evi:prompts_create-prompt"}, + {"prompts_list-prompt-versions", "GET", "/v0/evi/prompts/{id}", "api", "-", "", "", "page_number,page_size,restrict_to_most_recent", "json", "openapi-evi:prompts_list-prompt-versions"}, + {"prompts_create-prompt-version", "POST", "/v0/evi/prompts/{id}", "api", "json", "prompt_expansion,text,version_description", "text", "", "json", "openapi-evi:prompts_create-prompt-version"}, + {"prompts_update-prompt-name", "PATCH", "/v0/evi/prompts/{id}", "api", "json", "name", "name", "", "json", "openapi-evi:prompts_update-prompt-name"}, + {"prompts_delete-prompt", "DELETE", "/v0/evi/prompts/{id}", "api", "-", "", "", "", "json", "openapi-evi:prompts_delete-prompt"}, + {"prompts_get-prompt-version", "GET", "/v0/evi/prompts/{id}/version/{version}", "api", "-", "", "", "", "json", "openapi-evi:prompts_get-prompt-version"}, + {"prompts_update-prompt-description", "PATCH", "/v0/evi/prompts/{id}/version/{version}", "api", "json", "version_description", "", "", "json", "openapi-evi:prompts_update-prompt-description"}, + {"prompts_delete-prompt-version", "DELETE", "/v0/evi/prompts/{id}/version/{version}", "api", "-", "", "", "", "json", "openapi-evi:prompts_delete-prompt-version"}, + {"tools_list-tools", "GET", "/v0/evi/tools", "api", "-", "", "", "page_number,page_size,restrict_to_most_recent,name", "json", "openapi-evi:tools_list-tools"}, + {"tools_create-tool", "POST", "/v0/evi/tools", "api", "json", "description,fallback_content,name,parameters,version_description", "name,parameters", "", "json", "openapi-evi:tools_create-tool"}, + {"tools_list-tool-versions", "GET", "/v0/evi/tools/{id}", "api", "-", "", "", "page_number,page_size,restrict_to_most_recent", "json", "openapi-evi:tools_list-tool-versions"}, + {"tools_create-tool-version", "POST", "/v0/evi/tools/{id}", "api", "json", "description,fallback_content,parameters,version_description", "parameters", "", "json", "openapi-evi:tools_create-tool-version"}, + {"tools_update-tool-name", "PATCH", "/v0/evi/tools/{id}", "api", "json", "name", "name", "", "json", "openapi-evi:tools_update-tool-name"}, + {"tools_delete-tool", "DELETE", "/v0/evi/tools/{id}", "api", "-", "", "", "", "json", "openapi-evi:tools_delete-tool"}, + {"tools_get-tool-version", "GET", "/v0/evi/tools/{id}/version/{version}", "api", "-", "", "", "", "json", "openapi-evi:tools_get-tool-version"}, + {"tools_update-tool-description", "PATCH", "/v0/evi/tools/{id}/version/{version}", "api", "json", "version_description", "", "", "json", "openapi-evi:tools_update-tool-description"}, + {"tools_delete-tool-version", "DELETE", "/v0/evi/tools/{id}/version/{version}", "api", "-", "", "", "", "json", "openapi-evi:tools_delete-tool-version"}, + {"audio_measure", "POST", "/v1/expression/audio/file", "expression", "form", "config,file", "file", "", "json", "openapi-expression:audio_measure"}, + {"runs_list", "GET", "/v1/expression/run", "expression", "-", "", "", "page_size,cursor,user_id,status,mode,endpoint,started_before,started_after", "json", "openapi-expression:runs_list"}, + {"runs_get", "GET", "/v1/expression/run/{run_id}", "expression", "-", "", "", "", "json", "openapi-expression:runs_get"}, + {"runs_list-events", "GET", "/v1/expression/run/{run_id}/events", "expression", "-", "", "", "page_size,cursor", "json", "openapi-expression:runs_list-events"}, + {"video_measure", "POST", "/v1/expression/video/file", "expression", "form", "config,file", "file", "", "json", "openapi-expression:video_measure"}, + {"tts_synthesize_json", "POST", "/v0/tts", "api", "json", "utterances,context,format,include_timestamp_types,num_generations,split_utterances,strip_headers,temperature,version,instant_mode", "utterances", "", "json", "sdk:hume-python-sdk src/hume/tts/raw_client.py synthesize_json"}, + {"tts_synthesize_file", "POST", "/v0/tts/file", "api", "json", "utterances,context,format,include_timestamp_types,num_generations,split_utterances,strip_headers,temperature,version,instant_mode", "utterances", "", "bytes", "sdk:hume-python-sdk src/hume/tts/raw_client.py synthesize_file"}, + {"tts_synthesize_json_streaming", "POST", "/v0/tts/stream/json", "api", "json", "utterances,context,format,include_timestamp_types,num_generations,split_utterances,strip_headers,temperature,version,instant_mode", "utterances", "", "bytes", "sdk:hume-python-sdk src/hume/tts/raw_client.py synthesize_json_streaming"}, + {"tts_synthesize_file_streaming", "POST", "/v0/tts/stream/file", "api", "json", "utterances,context,format,include_timestamp_types,num_generations,split_utterances,strip_headers,temperature,version,instant_mode", "utterances", "", "bytes", "sdk:hume-python-sdk src/hume/tts/raw_client.py synthesize_file_streaming"}, + {"tts_convert_voice_json", "POST", "/v0/tts/voice_conversion/json", "api", "form", "strip_headers,audio,context,voice,format,include_timestamp_types", "", "", "bytes", "sdk:hume-python-sdk src/hume/tts/raw_client.py convert_voice_json"}, + {"tts_convert_voice_file", "POST", "/v0/tts/voice_conversion/file", "api", "form", "audio,strip_headers,context,voice,format,include_timestamp_types", "audio", "", "bytes", "sdk:hume-python-sdk src/hume/tts/raw_client.py convert_voice_file"}, +}; +// CALLS-END + +static_assert(sizeof(kRows) / sizeof(kRows[0]) == 49, "hume call table size"); + +const CallTable& calls() { + static const CallTable table("hume", kRows, sizeof(kRows) / sizeof(kRows[0])); + return table; +} + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/calls.h b/src/vendors/audio/hume/calls.h new file mode 100644 index 0000000..500244c --- /dev/null +++ b/src/vendors/audio/hume/calls.h @@ -0,0 +1,45 @@ +#pragma once + +// The Hume call table: the three OpenAPI documents on dev.hume.ai +// (text-to-speech-tts.json: voices; speech-to-speech-evi.json: EVI +// configs, prompts, tools, chats, chat groups; expression-measurement-api.json: +// batch runs) plus the TTS synthesis calls, which the published OpenAPI +// document leaves out and which are taken from the official Python SDK +// (src/hume/tts/raw_client.py). Hosts: api (api.hume.ai) and expression +// (api.cloud.hume.ai). + +#include "vendors/audio/call_spec.h" + +namespace molo::audio::hume { + +const CallTable& calls(); + +struct HumeCalls { + static constexpr const char* kTts = "tts_synthesize_json"; + static constexpr const char* kTtsFile = "tts_synthesize_file"; + static constexpr const char* kTtsJsonStream = "tts_synthesize_json_streaming"; + static constexpr const char* kTtsFileStream = "tts_synthesize_file_streaming"; + static constexpr const char* kVoiceConvertJson = "tts_convert_voice_json"; + static constexpr const char* kVoiceConvertFile = "tts_convert_voice_file"; + static constexpr const char* kVoices = "voices_list"; + static constexpr const char* kVoiceCreate = "voices_create"; + static constexpr const char* kVoiceDelete = "voices_delete"; + static constexpr const char* kChats = "chats_list-chats"; + static constexpr const char* kChatEvents = "chats_list-chat-events"; + static constexpr const char* kChatAudio = "chats_get-audio"; + static constexpr const char* kChatGroups = "chatGroups_list-chat-groups"; + static constexpr const char* kChatGroup = "chatGroups_get-chat-group"; + static constexpr const char* kConfigs = "configs_list-configs"; + static constexpr const char* kConfigVersions = "configs_list-config-versions"; + static constexpr const char* kConfigVersion = "configs_get-config-version"; + static constexpr const char* kPrompts = "prompts_list-prompts"; + static constexpr const char* kTools = "tools_list-tools"; + static constexpr const char* kControlSend = "controlPlane_send"; + static constexpr const char* kRuns = "runs_list"; + static constexpr const char* kRun = "runs_get"; + static constexpr const char* kRunEvents = "runs_list-events"; + static constexpr const char* kAudioMeasure = "audio_measure"; + static constexpr const char* kVideoMeasure = "video_measure"; +}; + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/client.cpp b/src/vendors/audio/hume/client.cpp new file mode 100644 index 0000000..09e2e98 --- /dev/null +++ b/src/vendors/audio/hume/client.cpp @@ -0,0 +1,35 @@ +#include "vendors/audio/hume/client.h" + +#include "llm/config.h" + +namespace molo::audio::hume { + +llm::ApiClientConfig HumeEndpoints::config(const std::string& base) { + llm::ApiClientConfig c; + c.vendor = kVendor; + c.base_url = base; + c.key_env = kKeyEnv; + c.auth = llm::AuthStyle::kHeader; + c.auth_header = kKeyHeader; + return c; +} + +std::shared_ptr HumeEndpoints::from_env() { + auto hosts = std::make_shared(); + hosts->add("api", HostFactory::from_env(config(kApiBase))); + hosts->add("expression", HostFactory::from_env(config(kExpressionBase))); + return hosts; +} + +std::shared_ptr HumeEndpoints::with(AudioHost api, AudioHost expression) { + auto hosts = std::make_shared(); + hosts->add("api", std::move(api)); + hosts->add("expression", std::move(expression)); + return hosts; +} + +std::string HumeEndpoints::key() { return llm::Config::get(kKeyEnv); } + +AudioSocket::Headers HumeEndpoints::ws_headers(const std::string& key) { return {{kKeyHeader, key}}; } + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/client.h b/src/vendors/audio/hume/client.h new file mode 100644 index 0000000..08bbadc --- /dev/null +++ b/src/vendors/audio/hume/client.h @@ -0,0 +1,33 @@ +#pragma once + +// Hume endpoints and credentials. +// api https://api.hume.ai header X-Hume-Api-Key: $HUME_API_KEY +// expression https://api.cloud.hume.ai same key +// WebSocket wss://api.hume.ai/v0/tts, wss://api.hume.ai/v0/evi same header + +#include +#include + +#include "vendors/audio/hosts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::hume { + +struct HumeEndpoints { + static constexpr const char* kVendor = "hume"; + static constexpr const char* kKeyEnv = "HUME_API_KEY"; + static constexpr const char* kKeyHeader = "X-Hume-Api-Key"; + static constexpr const char* kApiBase = "https://api.hume.ai"; + static constexpr const char* kExpressionBase = "https://api.cloud.hume.ai"; + static constexpr const char* kTtsWs = "wss://api.hume.ai/v0/tts"; + static constexpr const char* kEviWs = "wss://api.hume.ai/v0/evi"; + + static llm::ApiClientConfig config(const std::string& base); + static std::shared_ptr from_env(); + // Injected hosts (tests): "api" and "expression". + static std::shared_ptr with(AudioHost api, AudioHost expression); + static std::string key(); + static AudioSocket::Headers ws_headers(const std::string& key); +}; + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/evi.cpp b/src/vendors/audio/hume/evi.cpp new file mode 100644 index 0000000..d55f88c --- /dev/null +++ b/src/vendors/audio/hume/evi.cpp @@ -0,0 +1,110 @@ +#include "vendors/audio/hume/evi.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::hume { + +using C = HumeCalls; + +namespace { +void page_checks(const AudioService& svc, const Json& q) { + if (!q.is_object()) return; + const Json* s = q.find("page_size"); + if (s && s->is_number() && (s->as_int() < 1 || s->as_int() > HumeCaps::kMaxPageSize)) svc.bad("page_size must be 1..100"); + const Json* p = q.find("page_number"); + if (p && p->is_number() && p->as_int() < 0) svc.bad("page_number must not be negative"); +} +} // namespace + +Json HumeEvi::chats(const Json& q) { + page_checks(*this, q); + return call(C::kChats, {}, nullptr, query_from(C::kChats, q)); +} + +Json HumeEvi::chat_events(const std::string& id, const Json& q) { + check_id("chat", id); + page_checks(*this, q); + return call(C::kChatEvents, {id}, nullptr, query_from(C::kChatEvents, q)); +} + +Json HumeEvi::chat_audio(const std::string& id) { + check_id("chat", id); + return call(C::kChatAudio, {id}); +} + +Json HumeEvi::chat_groups(const Json& q) { + page_checks(*this, q); + return call(C::kChatGroups, {}, nullptr, query_from(C::kChatGroups, q)); +} + +Json HumeEvi::chat_group(const std::string& id, const Json& q) { + check_id("chat group", id); + page_checks(*this, q); + return call(C::kChatGroup, {id}, nullptr, query_from(C::kChatGroup, q)); +} + +Json HumeEvi::configs(const Json& q) { + page_checks(*this, q); + return call(C::kConfigs, {}, nullptr, query_from(C::kConfigs, q)); +} + +Json HumeEvi::config_versions(const std::string& id, const Json& q) { + check_id("config", id); + page_checks(*this, q); + return call(C::kConfigVersions, {id}, nullptr, query_from(C::kConfigVersions, q)); +} + +Json HumeEvi::config_version(const std::string& id, int version) { + check_id("config", id); + if (version < 0 || version > 100000) bad("version must be 0..100000"); + return call(C::kConfigVersion, {id, std::to_string(version)}); +} + +Json HumeEvi::prompts(const Json& q) { + page_checks(*this, q); + return call(C::kPrompts, {}, nullptr, query_from(C::kPrompts, q)); +} + +Json HumeEvi::tools(const Json& q) { + page_checks(*this, q); + return call(C::kTools, {}, nullptr, query_from(C::kTools, q)); +} + +Json HumeEvi::control_send(const std::string& id, const Json& event, bool confirm) { + check_id("chat", id); + check_body(table_.get(C::kControlSend), &event); + require_confirm("sending a control message to a live chat", confirm); + return call(C::kControlSend, {id}, &event); +} + +Json HumeExpression::runs(const Json& q) { return call(C::kRuns, {}, nullptr, query_from(C::kRuns, q)); } + +Json HumeExpression::run(const std::string& id) { + check_id("run", id); + return call(C::kRun, {id}); +} + +Json HumeExpression::run_events(const std::string& id, const Json& q) { + check_id("run", id); + return call(C::kRunEvents, {id}, nullptr, query_from(C::kRunEvents, q)); +} + +Json HumeExpression::measure_audio(const std::string& audio, const std::string& filename, + const std::string& content_type, const Json& config, bool confirm) { + if (audio.empty() || audio.size() > AudioCaps::kMaxAudioInBytes) bad("audio must be 1 byte..128 MiB"); + if (!config.is_object()) bad("config must be a JSON object"); + if (const Json* ms = config.find("measurement_timer_ms")) { + if (!ms->is_number() || ms->as_int() < 3000 || ms->as_int() > 10000) bad("measurement_timer_ms must be 3000..10000"); + } + for (const auto& [k, v] : config.as_object()) { + (void)v; + if (k != "measurement_timer_ms") bad("unknown config field '" + k + "' (AudioFileConfig has measurement_timer_ms)"); + } + require_confirm("starting a measurement run", confirm); + clients::body::MultipartBody form; + form.add_text("config", config.dump()); + form.add_file("file", filename.empty() ? "audio" : filename, content_type, audio); + return call_form(C::kAudioMeasure, {}, form); +} + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/evi.h b/src/vendors/audio/hume/evi.h new file mode 100644 index 0000000..5282816 --- /dev/null +++ b/src/vendors/audio/hume/evi.h @@ -0,0 +1,44 @@ +#pragma once + +// Empathic Voice Interface (EVI) management reads (chats, chat groups, +// configs, prompts, tools) and Expression Measurement batch runs. Creating, +// updating and deleting configs, prompts and tools is in the call table and +// reached through call() / call_destructive(). + +#include + +#include "vendors/audio/hume/tts.h" + +namespace molo::audio::hume { + +class HumeEvi : public HumeService { + public: + using HumeService::HumeService; + Json chats(const Json& query_object = Json()); + Json chat_events(const std::string& chat_id, const Json& query_object = Json()); + Json chat_audio(const std::string& chat_id); + Json chat_groups(const Json& query_object = Json()); + Json chat_group(const std::string& group_id, const Json& query_object = Json()); + Json configs(const Json& query_object = Json()); + Json config_versions(const std::string& config_id, const Json& query_object = Json()); + Json config_version(const std::string& config_id, int version); + Json prompts(const Json& query_object = Json()); + Json tools(const Json& query_object = Json()); + // Sends a control message (pause / resume assistant, session settings) to a live chat: confirm required. + Json control_send(const std::string& chat_id, const Json& event, bool confirm); +}; + +class HumeExpression : public HumeService { + public: + using HumeService::HumeService; + Json runs(const Json& query_object = Json()); + Json run(const std::string& run_id); + Json run_events(const std::string& run_id, const Json& query_object = Json()); + // Starts an audio measurement run (billed). The audio is 16-bit PCM, 16 kHz, mono: a WAV file or + // headerless little-endian samples (the OpenAPI text). `config` is the AudioFileConfig object, + // for example {"measurement_timer_ms":3000}, sent as the JSON `config` part. + Json measure_audio(const std::string& audio, const std::string& filename, const std::string& content_type, + const Json& config, bool confirm); +}; + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/sockets.cpp b/src/vendors/audio/hume/sockets.cpp new file mode 100644 index 0000000..73203d5 --- /dev/null +++ b/src/vendors/audio/hume/sockets.cpp @@ -0,0 +1,252 @@ +#include "vendors/audio/hume/sockets.h" + +#include "security/crypto/Base64.h" +#include "vendors/audio/caps.h" + +namespace molo::audio::hume { + +namespace { + +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("hume", why); } + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} + +Json typed(const char* type) { + Json m = Json::object(); + m.set("type", Json(type)); + return m; +} + +} // namespace + +Vocabulary HumeTtsStream::vocabulary() { + return Vocabulary("type", {}, {"audio", "timestamp", "error"}); +} + +HumeTtsStream::HumeTtsStream(std::string api_key, HumeStreamOptions o) { + Query q; + q.add_if("format_type", o.format_type); + if (!o.format_type.empty() && o.format_type != "mp3" && o.format_type != "wav" && o.format_type != "pcm") + invalid("format_type must be mp3, wav or pcm"); + if (!o.context_generation_id.empty()) { + if (!llm::Service::valid_id(o.context_generation_id)) invalid("bad generation id"); + q.add("context_generation_id", o.context_generation_id); + } + for (const auto& t : o.include_timestamp_types) { + if (t != "word" && t != "phoneme") invalid("timestamp type must be word or phoneme"); + q.add("include_timestamp_types", t); + } + if (o.instant_mode >= 0) q.add_bool("instant_mode", o.instant_mode == 1); + if (o.no_binary >= 0) q.add_bool("no_binary", o.no_binary == 1); + if (o.strip_headers >= 0) q.add_bool("strip_headers", o.strip_headers == 1); + if (!o.version.empty()) { + if (o.version != "1" && o.version != "2") invalid("version must be 1 or 2"); + q.add("version", o.version); + } + std::string base = o.url.empty() ? std::string(HumeEndpoints::kTtsWs) + "/stream/input" : o.url; + socket_ = std::make_unique("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(), + std::move(o.socket)); +} + +void HumeTtsStream::send_text(const std::string& text, const std::string& voice_name, const std::string& description, + double speed, double trailing_silence) { + if (text.empty() || text.size() > HumeCaps::kMaxUtteranceChars) invalid("text must be 1..5000 characters"); + if (text.find('\0') != std::string::npos) invalid("text has a NUL byte"); + Json m = Json::object(); + m.set("text", Json(text)); + if (!voice_name.empty()) { + if (voice_name.size() > 128) invalid("voice name is too long"); + Json v = Json::object(); + v.set("name", Json(voice_name)); + m.set("voice", v); + } + if (!description.empty()) { + if (description.size() > HumeCaps::kMaxDescriptionChars) invalid("description is too long"); + m.set("description", Json(description)); + } + if (speed != 0) { + if (speed < 0) invalid("speed must be positive"); + m.set("speed", Json(speed)); + } + if (trailing_silence >= 0) m.set("trailing_silence", Json(trailing_silence)); + socket_->send_json(m, true); +} + +void HumeTtsStream::flush() { + Json m = Json::object(); + m.set("flush", Json(true)); + socket_->send_json(m, true); +} + +void HumeTtsStream::finish() { + Json m = Json::object(); + m.set("close", Json(true)); + socket_->send_json(m, true); +} + +HumeStreamResult HumeTtsStream::collect(int max_wait_ms) { + HumeStreamResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + if (ev->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + if (ev->kind == SocketEvent::Kind::kBinary) { + if (out.audio.size() + ev->bytes.size() > AudioCaps::kMaxStreamAudioBytes) { + out.truncated = true; + return out; + } + out.audio += ev->bytes; + ++out.chunks; + continue; + } + if (!ev->json_ok || !ev->json.is_object()) continue; + if (ev->type == "audio") { + std::string raw; + if (!security::crypto::Base64::decode(str_at(ev->json, "audio"), raw)) + throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio chunk is not valid base64"); + if (out.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { + out.truncated = true; + return out; + } + out.audio += raw; + ++out.chunks; + out.generation_id = str_at(ev->json, "generation_id"); + if (const Json* l = ev->json.find("is_last_chunk"); l && l->is_bool() && l->as_bool()) { + out.last_chunk = true; + return out; + } + } else if (ev->type == "timestamp") { + out.timestamps.push_back(ev->json); + } else if (ev->type == "error" || ev->json.find("error")) { + out.error = str_at(ev->json, "message"); + if (out.error.empty()) out.error = str_at(ev->json, "error"); + if (out.error.empty()) out.error = "stream error"; + return out; + } + } + return out; +} + +Vocabulary EviChat::vocabulary() { + return Vocabulary("type", + {"audio_input", "session_settings", "user_input", "assistant_input", "tool_response", + "tool_error", "pause_assistant_message", "resume_assistant_message"}, + {"assistant_end", "assistant_message", "assistant_prosody", "audio_output", "chat_metadata", + "error", "user_interruption", "user_message", "tool_call", "tool_response", "tool_error", + "session_settings"}); +} + +EviChat::EviChat(std::string api_key, EviOptions o) { + Query q; + if (!o.config_id.empty()) { + if (!llm::Service::valid_id(o.config_id)) invalid("bad config id"); + q.add("config_id", o.config_id); + } + if (o.config_version >= 0) q.add_int("config_version", o.config_version); + if (!o.resumed_chat_group_id.empty()) { + if (!llm::Service::valid_id(o.resumed_chat_group_id)) invalid("bad chat group id"); + q.add("resumed_chat_group_id", o.resumed_chat_group_id); + } + if (o.verbose_transcription >= 0) q.add_bool("verbose_transcription", o.verbose_transcription == 1); + if (o.event_limit < 0 || o.event_limit > 10000) invalid("event_limit must be 0..10000"); + if (o.event_limit > 0) q.add_int("event_limit", o.event_limit); + std::string base = o.url.empty() ? std::string(HumeEndpoints::kEviWs) + "/chat" : o.url; + socket_ = std::make_unique("hume", base + q.str(), HumeEndpoints::ws_headers(api_key), vocabulary(), + std::move(o.socket)); +} + +void EviChat::send_session_settings(const Json& settings) { + if (!settings.is_object()) invalid("settings must be a JSON object"); + Json m = settings; + m.set("type", Json("session_settings")); + socket_->send_json(m); +} + +void EviChat::send_audio(const std::string& audio) { + if (audio.empty() || audio.size() > 1024u * 1024u) invalid("audio chunk must be 1 byte..1 MiB"); + Json m = typed("audio_input"); + m.set("data", Json(security::crypto::Base64::encode(audio))); + socket_->send_json(m); +} + +void EviChat::send_user_text(const std::string& text) { + if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters"); + Json m = typed("user_input"); + m.set("text", Json(text)); + socket_->send_json(m); +} + +void EviChat::send_assistant_text(const std::string& text) { + if (text.empty() || text.size() > 20000) invalid("text must be 1..20000 characters"); + Json m = typed("assistant_input"); + m.set("text", Json(text)); + socket_->send_json(m); +} + +void EviChat::send_tool_response(const std::string& id, const std::string& content) { + if (!llm::Service::valid_id(id)) invalid("bad tool_call_id"); + if (content.size() > 200000) invalid("content is over 200000 characters"); + Json m = typed("tool_response"); + m.set("tool_call_id", Json(id)); + m.set("content", Json(content)); + socket_->send_json(m); +} + +void EviChat::pause_assistant() { socket_->send_json(typed("pause_assistant_message")); } +void EviChat::resume_assistant() { socket_->send_json(typed("resume_assistant_message")); } + +EviTurn EviChat::collect_turn(int max_wait_ms) { + EviTurn turn; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return turn; + if (ev->kind == SocketEvent::Kind::kClose) { + turn.closed = true; + return turn; + } + if (!ev->json_ok || !ev->json.is_object()) continue; + const std::string& t = ev->type; + auto content = [&](const char* key) { + const Json* m = ev->json.find("message"); + return m ? str_at(*m, key) : std::string(); + }; + if (t == "chat_metadata") { + turn.chat_id = str_at(ev->json, "chat_id"); + turn.chat_group_id = str_at(ev->json, "chat_group_id"); + } else if (t == "user_message") { + turn.user_messages.push_back(content("content")); + } else if (t == "assistant_message") { + turn.assistant_messages.push_back(content("content")); + } else if (t == "audio_output") { + std::string raw; + if (!security::crypto::Base64::decode(str_at(ev->json, "data"), raw)) + throw llm::ApiError("hume", 502, "BAD_RESPONSE", "audio_output is not valid base64"); + if (turn.audio.size() + raw.size() > AudioCaps::kMaxStreamAudioBytes) { + turn.truncated = true; + return turn; + } + turn.audio += raw; + ++turn.audio_chunks; + } else if (t == "user_interruption") { + turn.interrupted = true; + } else if (t == "tool_call") { + turn.tool_calls.push_back(ev->json); + } else if (t == "assistant_end") { + turn.assistant_end = true; + return turn; + } else if (t == "error") { + turn.error = str_at(ev->json, "message"); + if (turn.error.empty()) turn.error = "chat error"; + return turn; + } + } + return turn; +} + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/sockets.h b/src/vendors/audio/hume/sockets.h new file mode 100644 index 0000000..7425ae2 --- /dev/null +++ b/src/vendors/audio/hume/sockets.h @@ -0,0 +1,111 @@ +#pragma once + +// Hume WebSockets: TTS stream input (wss://api.hume.ai/v0/tts/stream/input) +// and the EVI chat (wss://api.hume.ai/v0/evi/chat). Message shapes follow the +// Python SDK types (tts/types/publish_tts.py, snippet_audio_chunk.py, +// empathic_voice/types/*.py). + +#include +#include +#include +#include + +#include "vendors/audio/hume/tts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::hume { + +struct HumeStreamOptions { + std::string format_type; // "", mp3, wav, pcm (query parameter format_type) + std::string context_generation_id; + std::vector include_timestamp_types; // word | phoneme + int instant_mode = -1; + int no_binary = -1; // ask for base64 JSON chunks only + int strip_headers = -1; + std::string version; // "", "1", "2" + std::string url; // "" = wss://api.hume.ai/v0/tts/stream/input + SocketOptions socket; +}; + +struct HumeStreamResult { + std::string audio; // decoded chunks + std::size_t chunks = 0; + std::vector timestamps; + std::string generation_id; + bool last_chunk = false; // is_last_chunk seen + bool truncated = false; + std::string error; + bool closed = false; +}; + +class HumeTtsStream { + public: + HumeTtsStream(std::string api_key, HumeStreamOptions options); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + // PublishTts: text (and optionally voice name / description / speed / trailing_silence). + void send_text(const std::string& text, const std::string& voice_name = "", const std::string& description = "", + double speed = 0, double trailing_silence = -1); + void flush(); // {"flush": true} + void finish(); // {"close": true}: force generation and close + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + HumeStreamResult collect(int max_wait_ms); + + static Vocabulary vocabulary(); + + private: + std::unique_ptr socket_; +}; + +struct EviOptions { + std::string config_id; + int config_version = -1; + std::string resumed_chat_group_id; + int verbose_transcription = -1; + int event_limit = 0; + std::string url; // "" = wss://api.hume.ai/v0/evi/chat + SocketOptions socket; +}; + +struct EviTurn { + std::string chat_id, chat_group_id; + std::vector user_messages; // user_message.message.content + std::vector assistant_messages; // assistant_message.message.content + std::string audio; // decoded audio_output data + std::size_t audio_chunks = 0; + std::vector tool_calls; + bool interrupted = false; + bool assistant_end = false; + bool truncated = false; + std::string error; + bool closed = false; +}; + +class EviChat { + public: + EviChat(std::string api_key, EviOptions options); + std::string url() const { return socket_->url(); } + void connect() { socket_->connect(); } + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + void send_session_settings(const Json& settings); // {"type":"session_settings",...} + void send_audio(const std::string& audio); // {"type":"audio_input","data": base64} + void send_user_text(const std::string& text); // user_input + void send_assistant_text(const std::string& text); // assistant_input (assistant speaks this) + void send_tool_response(const std::string& tool_call_id, const std::string& content); + void pause_assistant(); + void resume_assistant(); + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + EviTurn collect_turn(int max_wait_ms); + + static Vocabulary vocabulary(); + + private: + std::unique_ptr socket_; +}; + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/speech.cpp b/src/vendors/audio/hume/speech.cpp new file mode 100644 index 0000000..9640f08 --- /dev/null +++ b/src/vendors/audio/hume/speech.cpp @@ -0,0 +1,55 @@ +#include "vendors/audio/hume/speech.h" + +#include + +namespace molo::audio::hume { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("hume", why); } + +bool looks_like_id(const std::string& s) { + if (s.size() != 36) return false; + for (std::size_t i = 0; i < s.size(); ++i) { + bool dash = i == 8 || i == 13 || i == 18 || i == 23; + if (dash ? s[i] != '-' : !std::isxdigit(static_cast(s[i]))) return false; + } + return true; +} +} // namespace + +HumeTtsRequest HumeSynthesizer::map(const SpeechRequest& r) const { + validate(r); + if (r.sample_rate != 0) invalid("Hume chooses the sample rate; leave sample_rate at 0"); + HumeTtsRequest h; + Utterance u; + u.text = r.text; + if (!r.voice.empty()) { + if (looks_like_id(r.voice)) u.voice_id = r.voice; + else u.voice_name = r.voice; + u.voice_provider = "HUME_AI"; // library voices; custom voices go through the typed API directly + } + if (r.speed > 0) u.speed = r.speed; + h.utterances.push_back(u); + h.format = r.format; + if (r.model == "1" || r.model == "2") h.version = r.model; + return h; +} + +SpeechResult HumeSynthesizer::synthesize(const SpeechRequest& r) { + HumeAudio a = tts_.synthesize_file(map(r)); + SpeechResult out; + out.audio = std::move(a.bytes); + out.content_type = a.content_type.empty() ? content_type_for(r.format) : a.content_type; + out.format = r.format; + out.provider = name(); + return out; +} + +Json HumeSynthesizer::voices() { + Json q = Json::object(); + q.set("provider", Json("HUME_AI")); + q.set("page_size", Json(100)); + return tts_.voices(q); +} + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/speech.h b/src/vendors/audio/hume/speech.h new file mode 100644 index 0000000..3f3b4f8 --- /dev/null +++ b/src/vendors/audio/hume/speech.h @@ -0,0 +1,31 @@ +#pragma once + +// Hume Octave behind the neutral synthesizer seam. Hume has no +// speech-to-text product (EVI is a voice agent, Expression Measurement +// analyses prosody), so there is no recognizer. + +#include + +#include "vendors/audio/hume/tts.h" +#include "vendors/audio/speech.h" + +namespace molo::audio::hume { + +class HumeSynthesizer : public SpeechSynthesizer { + public: + explicit HumeSynthesizer(std::shared_ptr hosts = HumeEndpoints::from_env()) : tts_(std::move(hosts)) {} + std::string name() const override { return "hume"; } + bool configured() const override { return tts_.configured(); } + std::size_t max_chars() const override { return HumeCaps::kMaxUtteranceChars; } + std::vector formats() const override { return {"mp3", "wav", "pcm"}; } + SpeechResult synthesize(const SpeechRequest& request) override; + Json voices() override; + // The Hume request a neutral request maps to. `voice` is a voice name from the + // library unless it looks like an id (36 characters with dashes). + HumeTtsRequest map(const SpeechRequest& request) const; + + private: + HumeTts tts_; +}; + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/tts.cpp b/src/vendors/audio/hume/tts.cpp new file mode 100644 index 0000000..a7a30b7 --- /dev/null +++ b/src/vendors/audio/hume/tts.cpp @@ -0,0 +1,134 @@ +#include "vendors/audio/hume/tts.h" + +#include "security/crypto/Base64.h" + +namespace molo::audio::hume { + +using C = HumeCalls; + +Json HumeTts::build(const HumeTtsRequest& r) const { + if (r.utterances.empty() || r.utterances.size() > HumeCaps::kMaxUtterances) bad("utterances must hold 1..100 entries"); + Json arr = Json::array(); + for (const Utterance& u : r.utterances) { + require_text("text", u.text, HumeCaps::kMaxUtteranceChars); + require_text("description", u.description, HumeCaps::kMaxDescriptionChars, false); + Json j = Json::object(); + j.set("text", Json(u.text)); + if (!u.description.empty()) j.set("description", Json(u.description)); + if (!u.voice_id.empty() && !u.voice_name.empty()) bad("give a voice id or a voice name, not both"); + if (!u.voice_id.empty() || !u.voice_name.empty()) { + Json v = Json::object(); + if (!u.voice_id.empty()) { + check_id("voice", u.voice_id); + v.set("id", Json(u.voice_id)); + } else { + require_text("voice name", u.voice_name, 128); + v.set("name", Json(u.voice_name)); + } + if (!u.voice_provider.empty()) { + if (u.voice_provider != "HUME_AI" && u.voice_provider != "CUSTOM_VOICE") + bad("voice provider must be HUME_AI or CUSTOM_VOICE"); + v.set("provider", Json(u.voice_provider)); + } + j.set("voice", v); + } + if (u.speed != 0) { + if (u.speed < 0) bad("speed must be positive"); + j.set("speed", Json(u.speed)); + } + if (u.trailing_silence >= 0) { + require_range("trailing_silence", u.trailing_silence, 0, 60); + j.set("trailing_silence", Json(u.trailing_silence)); + } + arr.push(j); + } + Json o = Json::object(); + o.set("utterances", arr); + if (r.format != "mp3" && r.format != "wav" && r.format != "pcm") bad("format must be mp3, wav or pcm"); + Json fmt = Json::object(); + fmt.set("type", Json(r.format)); + o.set("format", fmt); + if (r.num_generations < 0 || r.num_generations > HumeCaps::kMaxGenerations) bad("num_generations must be 0 or 1..5"); + if (r.num_generations > 0) o.set("num_generations", Json(r.num_generations)); + if (r.split_utterances >= 0) o.set("split_utterances", Json(r.split_utterances == 1)); + if (r.strip_headers >= 0) o.set("strip_headers", Json(r.strip_headers == 1)); + if (r.instant_mode >= 0) o.set("instant_mode", Json(r.instant_mode == 1)); + if (!r.version.empty()) { + if (r.version != "1" && r.version != "2") bad("version must be 1 or 2"); + o.set("version", Json(r.version)); + } + if (!r.context_generation_id.empty()) { + check_id("generation", r.context_generation_id); + Json ctx = Json::object(); + ctx.set("generation_id", Json(r.context_generation_id)); + o.set("context", ctx); + } + if (!r.include_timestamp_types.empty()) { + Json ts = Json::array(); + for (const auto& t : r.include_timestamp_types) { + if (t != "word" && t != "phoneme") bad("timestamp type must be word or phoneme"); + ts.push(Json(t)); + } + o.set("include_timestamp_types", ts); + } + check_body(table_.get(C::kTts), &o); + return o; +} + +Json HumeTts::synthesize(const HumeTtsRequest& r) { + Json body = build(r); + return call(C::kTts, {}, &body); +} + +std::string HumeTts::decode_first_audio(const Json& result) { + const Json* gens = result.is_object() ? result.find("generations") : nullptr; + if (!gens || !gens->is_array() || gens->as_array().empty()) + throw llm::ApiError("hume", 502, "BAD_RESPONSE", "no generations in the result"); + const Json* audio = gens->as_array()[0].find("audio"); + std::string raw; + if (!audio || !audio->is_string() || !security::crypto::Base64::decode(audio->as_string(), raw)) + throw llm::ApiError("hume", 502, "BAD_RESPONSE", "generation audio is missing or not valid base64"); + return raw; +} + +HumeAudio HumeTts::synthesize_file(const HumeTtsRequest& r) { + Json body = build(r); + llm::RawResponse raw = call_raw(C::kTtsFile, {}, &body); + return HumeAudio{raw.body, raw.content_type}; +} + +ByteStreamer::Result HumeTts::stream_json(const HumeTtsRequest& r, const ChunkSink& sink) { + Json body = build(r); + return call_stream(C::kTtsJsonStream, {}, &body, Query(), sink); +} + +ByteStreamer::Result HumeTts::stream_file(const HumeTtsRequest& r, const ChunkSink& sink) { + Json body = build(r); + return call_stream(C::kTtsFileStream, {}, &body, Query(), sink); +} + +Json HumeTts::voices(const Json& q) { + if (q.is_object()) { + const Json* s = q.find("page_size"); + if (s && s->is_number() && (s->as_int() < 1 || s->as_int() > HumeCaps::kMaxPageSize)) bad("page_size must be 1..100"); + const Json* p = q.find("provider"); + if (p && p->is_string() && p->as_string() != "HUME_AI" && p->as_string() != "CUSTOM_VOICE") + bad("provider must be HUME_AI or CUSTOM_VOICE"); + } + return call(C::kVoices, {}, nullptr, query_from(C::kVoices, q)); +} + +Json HumeTts::create_voice(const Json& body, bool confirm) { + check_body(table_.get(C::kVoiceCreate), &body); + require_confirm("saving a voice", confirm); + return call(C::kVoiceCreate, {}, &body); +} + +Json HumeTts::delete_voice(const std::string& name, bool confirm) { + require_text("voice name", name, 128); + Query q; + q.add("name", name); + return call_destructive(C::kVoiceDelete, {}, nullptr, q, confirm); +} + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/hume/tts.h b/src/vendors/audio/hume/tts.h new file mode 100644 index 0000000..86a2361 --- /dev/null +++ b/src/vendors/audio/hume/tts.h @@ -0,0 +1,76 @@ +#pragma once + +// Hume Octave text to speech and voices: synthesize (JSON with base64 audio, +// or a file), stream, list voices. Utterances can carry acting directions +// ("description"), a voice by id or name, speed and trailing silence. + +#include +#include +#include + +#include "vendors/audio/audio_service.h" +#include "vendors/audio/hume/calls.h" +#include "vendors/audio/hume/client.h" + +namespace molo::audio::hume { + +struct HumeCaps { + static constexpr std::size_t kMaxUtterances = 100; // our ceiling + static constexpr std::size_t kMaxUtteranceChars = 5000; // our ceiling per utterance + static constexpr std::size_t kMaxDescriptionChars = 1000; + static constexpr int kMaxPageSize = 100; + static constexpr int kMaxGenerations = 5; // our ceiling for num_generations +}; + +class HumeService : public AudioService { + public: + explicit HumeService(std::shared_ptr hosts) : AudioService(calls(), std::move(hosts)) {} + HumeService() : HumeService(HumeEndpoints::from_env()) {} +}; + +struct Utterance { + std::string text; // required + std::string description; // acting directions, or a voice prompt when no voice is given + std::string voice_id; // exactly one of voice_id and voice_name, or neither + std::string voice_name; + std::string voice_provider; // "", HUME_AI, CUSTOM_VOICE (default CUSTOM_VOICE per the SDK docs) + double speed = 0; // 0 unset; 0.75..1.5 is stable per the SDK docs + double trailing_silence = -1; // seconds, -1 unset +}; + +struct HumeTtsRequest { + std::vector utterances; + std::string format = "mp3"; // mp3 | wav | pcm + int num_generations = 0; // 0 unset, else 1..5 + int split_utterances = -1; + int strip_headers = -1; + int instant_mode = -1; + std::string version; // "", "1", "2" + std::string context_generation_id; // continue the prosody of an earlier generation + std::vector include_timestamp_types; // word | phoneme +}; + +struct HumeAudio { + std::string bytes; + std::string content_type; +}; + +class HumeTts : public HumeService { + public: + using HumeService::HumeService; + Json build(const HumeTtsRequest& request) const; + // JSON result: generations[].audio is base64 (see decode_first_audio). + Json synthesize(const HumeTtsRequest& request); + static std::string decode_first_audio(const Json& result); + HumeAudio synthesize_file(const HumeTtsRequest& request); + // Newline delimited JSON chunks ({"type":"audio","audio":base64,...}); raw bytes to `sink`. + ByteStreamer::Result stream_json(const HumeTtsRequest& request, const ChunkSink& sink); + ByteStreamer::Result stream_file(const HumeTtsRequest& request, const ChunkSink& sink); + + // Voices. + Json voices(const Json& query_object = Json()); // provider, page_number, page_size, ascending_order, filter_tag + Json create_voice(const Json& body, bool confirm); // saves a generated voice: a write + Json delete_voice(const std::string& name, bool confirm); +}; + +} // namespace molo::audio::hume diff --git a/src/vendors/audio/providers.cpp b/src/vendors/audio/providers.cpp new file mode 100644 index 0000000..b3d5985 --- /dev/null +++ b/src/vendors/audio/providers.cpp @@ -0,0 +1,50 @@ +#include "vendors/audio/providers.h" + +#include "vendors/audio/assemblyai/calls.h" +#include "vendors/audio/cartesia/calls.h" +#include "vendors/audio/deepgram/calls.h" +#include "vendors/audio/elevenlabs/calls.h" +#include "vendors/audio/hume/calls.h" +#include "vendors/audio/speechmatics/calls.h" + +namespace molo::audio { + +const std::vector& Providers::all() { + static const std::vector v = { + {"elevenlabs", "ElevenLabs", "ELEVENLABS_API_KEY", "", "https://elevenlabs.io/app/settings/api-keys", true, true, + "tts-stream-input,stt-realtime,convai-conversation", &elevenlabs::calls}, + {"assemblyai", "AssemblyAI", "ASSEMBLYAI_API_KEY", "", "https://www.assemblyai.com/dashboard/api-keys", false, true, + "streaming-v3", &assemblyai::calls}, + {"deepgram", "Deepgram", "DEEPGRAM_API_KEY", "", "https://console.deepgram.com", true, true, + "listen-v1,listen-v2,speak-v1,speak-v2,agent-converse", &deepgram::calls}, + {"cartesia", "Cartesia", "CARTESIA_API_KEY", "", "https://play.cartesia.ai/keys", true, true, + "tts-websocket,stt-turns-websocket,stt-websocket", &cartesia::calls}, + {"speechmatics", "Speechmatics", "SPEECHMATICS_API_KEY", "SPEECHMATICS_MANAGEMENT_TOKEN", + "https://portal.speechmatics.com/settings/api-keys/", true, true, "realtime-v2", &speechmatics::calls}, + {"hume", "Hume", "HUME_API_KEY", "", "https://app.hume.ai/keys", true, false, "tts-stream-input,evi-chat", + &hume::calls}, + }; + return v; +} + +const ProviderInfo* Providers::find(const std::string& name) { + for (const ProviderInfo& p : all()) + if (name == p.name) return &p; + return nullptr; +} + +std::size_t Providers::call_count() { + std::size_t n = 0; + for (const ProviderInfo& p : all()) n += p.table().size(); + return n; +} + +const std::vector& Providers::absent() { + static const std::vector v = { + {"playht", + "PlayHT's API hosts do not resolve (api.play.ht and api.play.ai, 2026-10-25): no public API to wrap, none invented"}, + }; + return v; +} + +} // namespace molo::audio diff --git a/src/vendors/audio/providers.h b/src/vendors/audio/providers.h new file mode 100644 index 0000000..31ee393 --- /dev/null +++ b/src/vendors/audio/providers.h @@ -0,0 +1,41 @@ +#pragma once + +// The provider table of this package: one descriptor per vendor (name, +// credential variable(s), what it offers, the size of its call table). The +// demo status route and the registry read from here, so a provider is +// described in exactly one place. + +#include +#include + +#include "vendors/audio/call_spec.h" + +namespace molo::audio { + +struct ProviderInfo { + const char* name; // "elevenlabs" + const char* title; // "ElevenLabs" + const char* key_env; // "ELEVENLABS_API_KEY" + const char* extra_env; // second credential ("" none) + const char* key_page; // where to create the key + bool synthesizer; // SpeechSynthesizer available + bool recognizer; // SpeechRecognizer available + const char* websockets; // comma list of WebSocket channels this package speaks + const CallTable& (*table)(); // the call table +}; + +class Providers { + public: + static const std::vector& all(); + static const ProviderInfo* find(const std::string& name); + // Total number of REST calls over all tables. + static std::size_t call_count(); + // Providers asked about but without a public API (PlayHT), with the reason. + struct Absent { + const char* name; + const char* reason; + }; + static const std::vector& absent(); +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/query.cpp b/src/vendors/audio/query.cpp new file mode 100644 index 0000000..4136269 --- /dev/null +++ b/src/vendors/audio/query.cpp @@ -0,0 +1,75 @@ +#include "vendors/audio/query.h" + +#include +#include +#include + +#include "vendors/audio/caps.h" + +namespace molo::audio { + +std::string Query::encode(const std::string& s) { + static const char* hex = "0123456789ABCDEF"; + std::string out; + for (unsigned char c : s) { + if (std::isalnum(c) || c == '-' || c == '_' || c == '~' || c == '.') { + out.push_back(static_cast(c)); + } else { + out.push_back('%'); + out.push_back(hex[c >> 4]); + out.push_back(hex[c & 15]); + } + } + return out; +} + +Query& Query::add(const std::string& name, const std::string& value) { + pairs_.emplace_back(name, value); + return *this; +} +Query& Query::add_int(const std::string& name, long value) { return add(name, std::to_string(value)); } +Query& Query::add_bool(const std::string& name, bool value) { return add(name, value ? "true" : "false"); } +Query& Query::add_if(const std::string& name, const std::string& value) { + if (!value.empty()) add(name, value); + return *this; +} + +namespace { +bool scalar(const std::string& name, const clients::body::Json& v, Query& q) { + if (v.is_string()) { + q.add(name, v.as_string()); + } else if (v.is_bool()) { + q.add_bool(name, v.as_bool()); + } else if (v.is_number()) { + double d = v.as_number(); + if (d == std::floor(d) && std::fabs(d) < 9e15) q.add_int(name, static_cast(d)); + else q.add(name, std::to_string(d)); + } else { + return false; + } + return true; +} +} // namespace + +bool Query::add_json(const std::string& name, const clients::body::Json& value) { + if (value.is_null()) return true; + if (value.is_array()) { + if (value.as_array().size() > AudioCaps::kMaxListItems) return false; + for (const auto& e : value.as_array()) + if (!scalar(name, e, *this)) return false; + return true; + } + return scalar(name, value, *this); +} + +std::string Query::str() const { + std::string out; + for (const auto& [k, v] : pairs_) { + out += out.empty() ? '?' : '&'; + out += encode(k) + "=" + encode(v); + if (out.size() > AudioCaps::kMaxQueryChars) throw std::length_error("query string over the cap"); + } + return out; +} + +} // namespace molo::audio diff --git a/src/vendors/audio/query.h b/src/vendors/audio/query.h new file mode 100644 index 0000000..32378e6 --- /dev/null +++ b/src/vendors/audio/query.h @@ -0,0 +1,33 @@ +#pragma once + +// Percent-encoded query string builder with a hard size cap. + +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::audio { + +class Query { + public: + Query& add(const std::string& name, const std::string& value); + Query& add_int(const std::string& name, long value); + Query& add_bool(const std::string& name, bool value); + Query& add_if(const std::string& name, const std::string& value); // skips "" + // Scalars become one pair, arrays one pair per element, null is skipped. + // Objects are rejected (llm::ApiError::invalid is thrown by the caller). + bool add_json(const std::string& name, const clients::body::Json& value); + bool empty() const { return pairs_.empty(); } + // "" or "?a=b&c=d". Throws std::length_error past AudioCaps::kMaxQueryChars. + std::string str() const; + const std::vector>& pairs() const { return pairs_; } + + static std::string encode(const std::string& s); + + private: + std::vector> pairs_; +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/registry.cpp b/src/vendors/audio/registry.cpp new file mode 100644 index 0000000..e4eff17 --- /dev/null +++ b/src/vendors/audio/registry.cpp @@ -0,0 +1,45 @@ +#include "llm/error.h" +#include "vendors/audio/assemblyai/speech.h" +#include "vendors/audio/cartesia/speech.h" +#include "vendors/audio/deepgram/speech.h" +#include "vendors/audio/elevenlabs/speech.h" +#include "vendors/audio/hume/speech.h" +#include "vendors/audio/providers.h" +#include "vendors/audio/speech.h" +#include "vendors/audio/speechmatics/speech.h" + +namespace molo::audio { + +std::vector SpeechRegistry::synthesizer_names() { + std::vector out; + for (const ProviderInfo& p : Providers::all()) + if (p.synthesizer) out.push_back(p.name); + return out; +} + +std::vector SpeechRegistry::recognizer_names() { + std::vector out; + for (const ProviderInfo& p : Providers::all()) + if (p.recognizer) out.push_back(p.name); + return out; +} + +std::shared_ptr SpeechRegistry::synthesizer(const std::string& name) { + if (name == "elevenlabs") return std::make_shared(); + if (name == "deepgram") return std::make_shared(); + if (name == "cartesia") return std::make_shared(); + if (name == "speechmatics") return std::make_shared(); + if (name == "hume") return std::make_shared(); + throw llm::ApiError::invalid("audio", "no speech synthesizer named '" + name + "'"); +} + +std::shared_ptr SpeechRegistry::recognizer(const std::string& name) { + if (name == "elevenlabs") return std::make_shared(); + if (name == "assemblyai") return std::make_shared(); + if (name == "deepgram") return std::make_shared(); + if (name == "cartesia") return std::make_shared(); + if (name == "speechmatics") return std::make_shared(); + throw llm::ApiError::invalid("audio", "no speech recognizer named '" + name + "'"); +} + +} // namespace molo::audio diff --git a/src/vendors/audio/socket.cpp b/src/vendors/audio/socket.cpp new file mode 100644 index 0000000..e36b7e5 --- /dev/null +++ b/src/vendors/audio/socket.cpp @@ -0,0 +1,180 @@ +#include "vendors/audio/socket.h" + +#include + +#include "llm/error.h" + +namespace molo::audio { + +namespace gl = google::live; +using clients::body::Json; + +namespace { + +long now_ms() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +llm::ApiError map_error(const std::string& vendor, const gl::WsError& e) { + using K = gl::WsError::Kind; + int status = e.http_status(); + if (status == 401 || status == 403) + return llm::ApiError(vendor, status, "authentication_error", e.what()); + if (e.kind() == K::Invalid) return llm::ApiError(vendor, 400, "invalid_request_error", e.what()); + if (e.kind() == K::Timeout) return llm::ApiError(vendor, 504, "timeout", e.what()); + return llm::ApiError(vendor, status ? status : 502, "UNAVAILABLE", e.what()); +} + +} // namespace + +AudioSocket::AudioSocket(std::string vendor, std::string url, Headers headers, Vocabulary vocabulary, + SocketOptions options) + : vendor_(std::move(vendor)), + url_(std::move(url)), + headers_(std::move(headers)), + vocabulary_(std::move(vocabulary)), + options_(std::move(options)) {} + +AudioSocket::~AudioSocket() { + try { + close(); + } catch (...) { + } +} + +std::string AudioSocket::with_query(const std::string& url, const std::string& query) { + if (query.empty()) return url; + return url + (url.find('?') == std::string::npos ? "?" : "&") + query; +} + +void AudioSocket::connect() { + if (open_) throw llm::ApiError::invalid(vendor_, "socket already open"); + for (const auto& [name, value] : headers_) { + if (value.empty() || value.find_first_of("\r\n") != std::string::npos || + name.find_first_of("\r\n:") != std::string::npos) + throw llm::ApiError::invalid(vendor_, "missing or malformed credential header"); + } + try { + gl::WsUrl url = gl::WsUrl::parse(url_); + ws_ = std::make_unique( + gl::open_stream(url, options_.connect_timeout_ms, AudioCaps::kIoTimeoutMs, options_.tls)); + gl::WsHeaders headers; + for (const auto& h : headers_) headers.emplace_back(h.first, h.second); + ws_->handshake(url, headers, options_.handshake_timeout_ms); + ws_->set_read_timeout_ms(AudioCaps::kPollSliceMs); + } catch (const gl::WsError& e) { + if (ws_) ws_->shutdown(); + ws_.reset(); + throw map_error(vendor_, e); + } + open_ = true; + started_ms_ = now_ms(); +} + +void AudioSocket::send_json(const Json& message, bool allow_untyped) { + if (!open_) throw llm::ApiError(vendor_, 409, "not_open", "socket is not open"); + const Json* type = message.is_object() ? message.find(vocabulary_.discriminator()) : nullptr; + if (type) { + if (!type->is_string() || !vocabulary_.known_client(type->as_string())) + throw llm::ApiError::invalid(vendor_, "unknown client message type"); + } else if (!allow_untyped) { + throw llm::ApiError::invalid(vendor_, "client message has no '" + vocabulary_.discriminator() + "'"); + } + std::string text = message.dump(); + send_text(text); +} + +void AudioSocket::send_text(const std::string& text) { + if (!open_) throw llm::ApiError(vendor_, 409, "not_open", "socket is not open"); + if (text.size() > AudioCaps::kMaxSocketFrameBytes) + throw llm::ApiError::invalid(vendor_, "text frame over the byte cap"); + try { + ws_->send_text(text); + } catch (const gl::WsError& e) { + throw map_error(vendor_, e); + } + ++counters_.sent; +} + +void AudioSocket::send_binary(const std::string& bytes) { + if (!open_) throw llm::ApiError(vendor_, 409, "not_open", "socket is not open"); + if (bytes.empty() || bytes.size() > AudioCaps::kMaxSocketFrameBytes) + throw llm::ApiError::invalid(vendor_, "binary frame must be 1.." + + std::to_string(AudioCaps::kMaxSocketFrameBytes) + " bytes"); + try { + ws_->send_binary(bytes); + } catch (const gl::WsError& e) { + throw map_error(vendor_, e); + } + ++counters_.sent; +} + +void AudioSocket::close() { + if (!ws_) return; + try { + if (open_) ws_->close(1000, ""); + } catch (...) { + } + ws_->shutdown(); + open_ = false; +} + +SocketEvent AudioSocket::decode(const gl::WsMessage& m) { + SocketEvent ev; + if (m.kind == gl::WsMessage::Kind::Close) { + ev.kind = SocketEvent::Kind::kClose; + ev.text = m.data; + ev.close_code = m.code; + ev.type = "websocket.close"; + open_ = false; + return ev; + } + ++counters_.received; + if (m.kind == gl::WsMessage::Kind::Binary) { + ev.kind = SocketEvent::Kind::kBinary; + ev.bytes = m.data; + ev.type = "binary"; + ++counters_.binary_frames; + return ev; + } + ev.kind = SocketEvent::Kind::kText; + ev.text = m.data; + try { + ev.json = Json::parse(m.data); + ev.json_ok = true; + } catch (const std::exception&) { + ++counters_.decode_errors; + ev.known = false; + return ev; + } + if (ev.json.is_object()) { + if (const Json* t = ev.json.find(vocabulary_.discriminator()); t && t->is_string()) + ev.type = t->as_string(); + } + if (!ev.type.empty()) ev.known = vocabulary_.known_server(ev.type); + if (!ev.known) ++counters_.unknown_events; + return ev; +} + +std::optional AudioSocket::next(int wait_ms) { + if (!ws_) throw llm::ApiError(vendor_, 409, "not_open", "socket is not open"); + if (wait_ms < 0) wait_ms = 0; + if (wait_ms > AudioCaps::kMaxWaitMs) wait_ms = AudioCaps::kMaxWaitMs; + long deadline = now_ms() + wait_ms; + for (;;) { + if (now_ms() - started_ms_ > static_cast(options_.max_session_seconds) * 1000) + throw llm::ApiError(vendor_, 504, "max_duration", "session ran past the local time cap"); + try { + auto m = ws_->receive(); + if (m) return decode(*m); + } catch (const gl::WsError& e) { + open_ = false; + throw map_error(vendor_, e); + } + if (now_ms() >= deadline) return std::nullopt; + } +} + +} // namespace molo::audio diff --git a/src/vendors/audio/socket.h b/src/vendors/audio/socket.h new file mode 100644 index 0000000..63acbb5 --- /dev/null +++ b/src/vendors/audio/socket.h @@ -0,0 +1,112 @@ +#pragma once + +// One provider WebSocket: connect with explicit headers (the credential +// travels in a header or a provider-defined sub-protocol, never logged and +// never part of an error), send JSON text or binary frames, read decoded +// events with a deadline. Built on the RFC 6455 client in src/google/live. +// Protocol knowledge (message names, field names) lives in the per-provider +// session classes and their Vocabulary, not here. + +#include +#include +#include +#include +#include +#include + +#include "clients/body/json.h" +#include "google/live/ws_client.h" +#include "google/live/ws_stream.h" +#include "vendors/audio/caps.h" + +namespace molo::audio { + +struct SocketOptions { + int connect_timeout_ms = AudioCaps::kConnectTimeoutMs; + int handshake_timeout_ms = AudioCaps::kHandshakeTimeoutMs; + int max_session_seconds = AudioCaps::kMaxSessionSeconds; + google::live::TlsFactory tls; // null: default wolfSSL session +}; + +// A decoded server frame. +struct SocketEvent { + enum class Kind { kText, kBinary, kClose }; + Kind kind = Kind::kText; + std::string text; // text frames and the close reason + std::string bytes; // binary frames + clients::body::Json json; // parsed text frame (null when it is not JSON) + bool json_ok = false; + std::string type; // value of the provider's discriminator field, "" if absent + bool known = true; // false: type not in the provider's server vocabulary + int close_code = 0; +}; + +// Message names a provider speaks. Unknown server events are kept and counted, +// unknown client messages are refused before sending. +class Vocabulary { + public: + Vocabulary(std::string discriminator, std::set client, std::set server) + : discriminator_(std::move(discriminator)), client_(std::move(client)), server_(std::move(server)) {} + const std::string& discriminator() const { return discriminator_; } + bool known_client(const std::string& t) const { return client_.count(t) > 0; } + bool known_server(const std::string& t) const { return server_.count(t) > 0; } + const std::set& client() const { return client_; } + const std::set& server() const { return server_; } + + private: + std::string discriminator_; // "type", "message_type", "event", ... + std::set client_; + std::set server_; +}; + +class AudioSocket { + public: + using Headers = std::vector>; + + AudioSocket(std::string vendor, std::string url, Headers headers, Vocabulary vocabulary, + SocketOptions options = {}); + ~AudioSocket(); + AudioSocket(const AudioSocket&) = delete; + AudioSocket& operator=(const AudioSocket&) = delete; + + // Throws llm::ApiError (401 on a refused upgrade with 401/403, else 502/504). + void connect(); + bool is_open() const { return open_; } + // URL as given, without any header value: safe to log. + const std::string& url() const { return url_; } + + // `type` must be in the client vocabulary when the message has the + // discriminator field; messages without it (provider specific raw JSON such + // as {"text":"..."}) pass when allow_untyped is true. + void send_json(const clients::body::Json& message, bool allow_untyped = false); + void send_text(const std::string& text); + void send_binary(const std::string& bytes); + void close(); + + // Next server message, or nullopt when `wait_ms` elapsed. A close frame is + // returned as kClose. Throws when the session ran past max_session_seconds. + std::optional next(int wait_ms); + + struct Counters { + std::size_t sent = 0, received = 0, unknown_events = 0, binary_frames = 0, decode_errors = 0; + }; + const Counters& counters() const { return counters_; } + const Vocabulary& vocabulary() const { return vocabulary_; } + + static std::string with_query(const std::string& url, const std::string& query); + + private: + SocketEvent decode(const google::live::WsMessage& m); + + std::string vendor_; + std::string url_; + Headers headers_; + Vocabulary vocabulary_; + SocketOptions options_; + std::unique_ptr ws_; + bool open_ = false; + long started_ms_ = 0; + Counters counters_; +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/speech.cpp b/src/vendors/audio/speech.cpp new file mode 100644 index 0000000..e1c21e3 --- /dev/null +++ b/src/vendors/audio/speech.cpp @@ -0,0 +1,162 @@ +#include "vendors/audio/speech.h" + +#include +#include +#include +#include +#include + +#include "llm/error.h" +#include "vendors/audio/caps.h" + +namespace molo::audio { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("audio", why); } + +bool has_nul(const std::string& s) { return s.find('\0') != std::string::npos; } + +bool is_utf8_continuation(unsigned char c) { return (c & 0xC0) == 0x80; } +} // namespace + +std::string content_type_for(const std::string& format) { + if (format == "mp3") return "audio/mpeg"; + if (format == "wav") return "audio/wav"; + if (format == "pcm") return "audio/pcm"; + return ""; +} + +void validate(const SpeechRequest& r) { + if (r.text.empty()) invalid("text is required"); + if (r.text.size() > AudioCaps::kMaxTextChars) + invalid("text is over " + std::to_string(AudioCaps::kMaxTextChars) + " characters"); + if (has_nul(r.text)) invalid("text has a NUL byte"); + for (const std::string* s : {&r.voice, &r.model, &r.language}) { + if (s->size() > 256) invalid("voice, model and language are at most 256 characters"); + if (has_nul(*s)) invalid("voice, model and language must not hold NUL bytes"); + } + if (content_type_for(r.format).empty()) invalid("format must be mp3, wav or pcm"); + if (r.sample_rate != 0 && (r.sample_rate < 8000 || r.sample_rate > 96000)) + invalid("sample_rate must be 0 or 8000..96000"); + if (r.speed != 0 && !(r.speed >= 0.25 && r.speed <= 4.0)) invalid("speed must be 0 or 0.25..4.0"); +} + +void validate(const TranscribeRequest& r) { + if (r.audio.empty() == r.url.empty()) invalid("give exactly one of audio and url"); + if (r.audio.size() > AudioCaps::kMaxAudioInBytes) invalid("audio is over the byte cap"); + if (!r.url.empty()) { + if (r.url.size() > AudioCaps::kMaxFieldChars) invalid("url is too long"); + if (r.url.rfind("https://", 0) != 0 && r.url.rfind("http://", 0) != 0) invalid("url must be http or https"); + if (r.url.find_first_of(" \r\n") != std::string::npos) invalid("url has whitespace"); + } + if (r.filename.empty() || r.filename.size() > 255 || r.filename.find_first_of("\r\n\"/\\") != std::string::npos) + invalid("filename is 1..255 characters without quotes, slashes or line breaks"); + if (r.content_type.empty() || r.content_type.size() > 128 || r.content_type.find_first_of("\r\n") != std::string::npos) + invalid("content_type is required"); + if (r.language.size() > 32 || r.model.size() > 128) invalid("language or model is too long"); + if (r.keywords.size() > AudioCaps::kMaxKeywords) + invalid("at most " + std::to_string(AudioCaps::kMaxKeywords) + " keywords"); + for (const auto& k : r.keywords) + if (k.empty() || k.size() > 100 || has_nul(k)) invalid("keywords are 1..100 characters"); + if (r.max_wait_seconds < 1 || r.max_wait_seconds > AudioCaps::kMaxPollSeconds) + invalid("max_wait_seconds must be 1.." + std::to_string(AudioCaps::kMaxPollSeconds)); +} + +std::vector split_text(const std::string& text, std::size_t max_chars) { + if (max_chars == 0) invalid("max_chars must be positive"); + std::vector out; + std::size_t pos = 0; + while (pos < text.size()) { + while (pos < text.size() && std::isspace(static_cast(text[pos]))) ++pos; + if (pos >= text.size()) break; + std::size_t remaining = text.size() - pos; + std::size_t len = remaining; + if (remaining > max_chars) { + std::size_t window = max_chars; + std::size_t cut = std::string::npos; + // Prefer the last sentence end in the second half of the window, then any whitespace. + for (std::size_t i = window; i > window / 2; --i) { + char c = text[pos + i - 1]; + char next = pos + i < text.size() ? text[pos + i] : ' '; + if ((c == '.' || c == '!' || c == '?' || c == '\n') && std::isspace(static_cast(next))) { + cut = i; + break; + } + } + if (cut == std::string::npos) { + for (std::size_t i = window; i > window / 2; --i) { + if (std::isspace(static_cast(text[pos + i - 1]))) { + cut = i; + break; + } + } + } + if (cut == std::string::npos) { + cut = window; + while (cut > 1 && is_utf8_continuation(static_cast(text[pos + cut]))) --cut; + } + len = cut; + } + std::string chunk = text.substr(pos, len); + while (!chunk.empty() && std::isspace(static_cast(chunk.back()))) chunk.pop_back(); + if (!chunk.empty()) out.push_back(chunk); + if (out.size() > AudioCaps::kMaxChunks) + invalid("text needs more than " + std::to_string(AudioCaps::kMaxChunks) + " chunks"); + pos += len; + } + return out; +} + +std::size_t ChunkedSynthesizer::max_chars() const { return AudioCaps::kMaxTextChars; } + +SpeechResult ChunkedSynthesizer::synthesize(const SpeechRequest& request) { + validate(request); + std::size_t per_call = std::min(inner_->max_chars(), AudioCaps::kMaxChunkChars * 8); + if (request.text.size() <= per_call) return inner_->synthesize(request); + std::vector chunks = split_text(request.text, per_call); + if (chunks.size() > 1 && request.format == "wav") + invalid("wav cannot be joined across chunks: use mp3 or pcm for long text"); + SpeechResult joined; + for (const std::string& c : chunks) { + SpeechRequest part = request; + part.text = c; + SpeechResult r = inner_->synthesize(part); + if (joined.audio.size() + r.audio.size() > AudioCaps::kMaxAudioOutBytes) + invalid("joined audio is over the byte cap"); + joined.audio += r.audio; + joined.content_type = r.content_type; + joined.format = r.format; + joined.provider = r.provider; + } + return joined; +} + +PollOutcome poll(const PollPolicy& requested, const std::function& step, + const std::function& sleep_ms) { + PollPolicy policy = PollPolicy::clamped(requested.interval_ms, requested.max_seconds); + PollOutcome out; + const long cap_ms = static_cast(policy.max_seconds) * 1000; + for (;;) { + ++out.polls; + if (step(out.polls)) { + out.done = true; + return out; + } + if (out.waited_ms + policy.interval_ms > cap_ms) { + out.timed_out = true; + return out; + } + if (sleep_ms) sleep_ms(policy.interval_ms); + else std::this_thread::sleep_for(std::chrono::milliseconds(policy.interval_ms)); + out.waited_ms += policy.interval_ms; + } +} + +PollPolicy PollPolicy::clamped(int interval_ms, int max_seconds) { + PollPolicy p; + p.interval_ms = std::clamp(interval_ms, AudioCaps::kMinPollIntervalMs, AudioCaps::kMaxPollIntervalMs); + p.max_seconds = std::clamp(max_seconds, 1, AudioCaps::kMaxPollSeconds); + return p; +} + +} // namespace molo::audio diff --git a/src/vendors/audio/speech.h b/src/vendors/audio/speech.h new file mode 100644 index 0000000..5b1c9b9 --- /dev/null +++ b/src/vendors/audio/speech.h @@ -0,0 +1,152 @@ +#pragma once + +// The neutral speech seam of this package. A SpeechSynthesizer turns text +// into audio, a SpeechRecognizer turns audio (bytes or a URL) into a +// transcript. Every provider with such a product implements them; callers +// (the demo, an agent runtime, tests with a fake) never see a provider type. +// This seam is local to the audio package and does not touch src/llm. + +#include +#include +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::audio { + +// ---- synthesis ------------------------------------------------------------ + +struct SpeechRequest { + std::string text; // 1..AudioCaps::kMaxTextChars + std::string voice; // provider voice id or name; "" = the provider default when it has one + std::string model; // "" = provider default + std::string language; // BCP-47 code, "" = provider default + std::string format = "mp3"; // "mp3" | "wav" | "pcm" + int sample_rate = 0; // 0 = provider default + double speed = 0; // 0 = provider default +}; + +struct SpeechResult { + std::string audio; // encoded bytes + std::string content_type; // audio/mpeg, audio/wav, audio/pcm + std::string format; // as requested + std::string provider; +}; + +// Throws llm::ApiError::invalid (vendor "audio") for an unusable request. +void validate(const SpeechRequest& request); +// Content type for a neutral format name ("mp3" -> "audio/mpeg"); "" if unknown. +std::string content_type_for(const std::string& format); + +class SpeechSynthesizer { + public: + virtual ~SpeechSynthesizer() = default; + virtual std::string name() const = 0; + virtual bool configured() const = 0; + // Longest text one call accepts; ChunkedSynthesizer splits above it. + virtual std::size_t max_chars() const = 0; + // Neutral formats this provider can produce. + virtual std::vector formats() const = 0; + virtual SpeechResult synthesize(const SpeechRequest& request) = 0; + // Provider voice catalogue as JSON (read only). + virtual clients::body::Json voices() = 0; +}; + +// ---- recognition ------------------------------------------------------------ + +struct TranscribeRequest { + std::string audio; // bytes; or empty with `url` + std::string url; // publicly reachable media URL + std::string filename = "audio"; + std::string content_type = "application/octet-stream"; + std::string language; // "" = provider default or detection + std::string model; // "" = provider default + bool diarize = false; + std::vector keywords; // vocabulary boost, <= AudioCaps::kMaxKeywords + int max_wait_seconds = 300; // asynchronous providers poll up to this, <= kMaxPollSeconds +}; + +struct TranscriptWord { + std::string text; + double start = 0; // seconds + double end = 0; + std::string speaker; + double confidence = 0; +}; + +struct Transcript { + std::string text; + std::string language; + double duration = 0; // seconds, 0 when the provider does not say + std::vector words; + clients::body::Json raw; // the provider's own document + std::string provider; +}; + +void validate(const TranscribeRequest& request); + +class SpeechRecognizer { + public: + virtual ~SpeechRecognizer() = default; + virtual std::string name() const = 0; + virtual bool configured() const = 0; + virtual Transcript transcribe(const TranscribeRequest& request) = 0; +}; + +// ---- helpers ------------------------------------------------------------------ + +// Sentence aware splitter: chunks of at most `max_chars` bytes that end at a +// sentence or word boundary when one exists. Throws llm::ApiError::invalid +// past AudioCaps::kMaxChunks chunks or for max_chars == 0. +std::vector split_text(const std::string& text, std::size_t max_chars); + +// Decorator: splits long text over the inner synthesizer's limit and joins +// the audio. Joining is only valid for formats that concatenate (mp3, pcm); +// "wav" over more than one chunk is refused loudly. +class ChunkedSynthesizer : public SpeechSynthesizer { + public: + explicit ChunkedSynthesizer(std::shared_ptr inner) : inner_(std::move(inner)) {} + std::string name() const override { return inner_->name(); } + bool configured() const override { return inner_->configured(); } + std::size_t max_chars() const override; + std::vector formats() const override { return inner_->formats(); } + SpeechResult synthesize(const SpeechRequest& request) override; + clients::body::Json voices() override { return inner_->voices(); } + + private: + std::shared_ptr inner_; +}; + +// Polling pace for asynchronous recognizers. Floors and ceilings come from +// AudioCaps; a policy outside them is clamped, not trusted. +struct PollPolicy { + int interval_ms = 2000; + int max_seconds = 300; + static PollPolicy clamped(int interval_ms, int max_seconds); +}; + +// Bounded polling loop. `step(attempt)` returns true when the work is done. +// Time is accounted in slept milliseconds (the sum of what `sleep_ms` was +// asked for), so the cap is deterministic and a fake sleeper keeps tests fast. +struct PollOutcome { + bool done = false; + bool timed_out = false; + int polls = 0; + long waited_ms = 0; +}; +PollOutcome poll(const PollPolicy& policy, const std::function& step, + const std::function& sleep_ms = nullptr); + +// Provider lookup by name, built from the environment (llm::Config). +class SpeechRegistry { + public: + static std::vector synthesizer_names(); + static std::vector recognizer_names(); + // Throws llm::ApiError::invalid for an unknown name. + static std::shared_ptr synthesizer(const std::string& name); + static std::shared_ptr recognizer(const std::string& name); +}; + +} // namespace molo::audio diff --git a/src/vendors/audio/speechmatics/batch.cpp b/src/vendors/audio/speechmatics/batch.cpp new file mode 100644 index 0000000..95712bb --- /dev/null +++ b/src/vendors/audio/speechmatics/batch.cpp @@ -0,0 +1,160 @@ +#include "vendors/audio/speechmatics/batch.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::speechmatics { + +using C = SpeechmaticsCalls; + +Json Batch::config_json(const JobConfig& c) const { + require_text("language", c.language, 16); + Json tc = Json::object(); + tc.set("language", Json(c.language)); + if (!c.operating_point.empty()) { + if (c.operating_point != "standard" && c.operating_point != "enhanced") bad("operating_point must be standard or enhanced"); + tc.set("operating_point", Json(c.operating_point)); + } + if (!c.diarization.empty()) { + const std::string& d = c.diarization; + if (d != "none" && d != "speaker" && d != "channel" && d != "speaker_change") + bad("diarization must be none, speaker, channel or speaker_change"); + tc.set("diarization", Json(d)); + } + if (c.enable_entities >= 0) tc.set("enable_entities", Json(c.enable_entities == 1)); + if (!c.domain.empty()) { + require_text("domain", c.domain, 64); + tc.set("domain", Json(c.domain)); + } + if (!c.vocab.empty()) { + if (c.vocab.size() > SpeechmaticsCaps::kMaxVocab) bad("too many vocabulary entries"); + Json arr = Json::array(); + for (const auto& v : c.vocab) { + require_text("vocabulary entry", v, 100); + Json e = Json::object(); + e.set("content", Json(v)); + arr.push(e); + } + tc.set("additional_vocab", arr); + } + Json o = Json::object(); + o.set("type", Json("transcription")); + o.set("transcription_config", tc); + if (!c.fetch_url.empty()) { + if (c.fetch_url.rfind("https://", 0) != 0 && c.fetch_url.rfind("http://", 0) != 0) bad("fetch_url must be http or https"); + require_text("fetch_url", c.fetch_url, AudioCaps::kMaxFieldChars); + Json f = Json::object(); + f.set("url", Json(c.fetch_url)); + o.set("fetch_data", f); + } + if (c.extra.is_object()) + for (const auto& [k, v] : c.extra.as_object()) { + if (o.find(k)) bad("extra repeats the section '" + k + "'"); + o.set(k, v); + } + return o; +} + +clients::body::MultipartBody Batch::build(const JobConfig& c, const std::string& audio, const std::string& filename, + const std::string& content_type) const { + if (audio.empty() == c.fetch_url.empty()) bad("give exactly one of audio and fetch_url"); + if (audio.size() > SpeechmaticsCaps::kMaxUploadBytes) bad("audio is over the byte cap"); + clients::body::MultipartBody form; + form.add_text("config", config_json(c).dump()); + if (!audio.empty()) { + if (filename.empty() || filename.size() > 255 || filename.find_first_of("\r\n\"/\\") != std::string::npos) + bad("filename is 1..255 characters without quotes, slashes or line breaks"); + form.add_file("data_file", filename, content_type, audio); + } + check_form(table_.get(C::kJobCreate), form); + return form; +} + +Json Batch::submit(const JobConfig& c, const std::string& audio, const std::string& filename, + const std::string& content_type) { + return call_form(C::kJobCreate, {}, build(c, audio, filename, content_type)); +} + +Json Batch::list(const Json& q) { + if (q.is_object()) { + const Json* l = q.find("limit"); + if (l && l->is_number() && (l->as_int() < 1 || l->as_int() > SpeechmaticsCaps::kMaxListLimit)) + bad("limit must be 1..100"); + } + return call(C::kJobList, {}, nullptr, query_from(C::kJobList, q)); +} + +Json Batch::get(const std::string& id) { + check_id("job", id); + return call(C::kJobGet, {id}); +} + +Json Batch::remove(const std::string& id, bool force, bool confirm) { + check_id("job", id); + Query q; + if (force) q.add_bool("force", true); + return call_destructive(C::kJobDelete, {id}, nullptr, q, confirm); +} + +Json Batch::transcript(const std::string& id, const std::string& format) { + check_id("job", id); + Query q; + if (!format.empty()) { + if (format != "json-v2" && format != "txt" && format != "srt") bad("format must be json-v2, txt or srt"); + q.add("format", format); + } + llm::RawResponse raw = call_raw(C::kTranscript, {id}, nullptr, q); + if (format.empty() || format == "json-v2") { + try { + return Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError("speechmatics", 502, "BAD_RESPONSE", e.what()); + } + } + Json o = Json::object(); + o.set("format", Json(format)); + o.set("content", Json(raw.body)); + return o; +} + +std::string Batch::log(const std::string& id) { + check_id("job", id); + return call_raw(C::kJobLog, {id}).body; +} + +Json Batch::usage(const Json& q) { return call(C::kUsage, {}, nullptr, query_from(C::kUsage, q)); } + +Json Batch::wait(const std::string& id, const PollPolicy& policy, const std::function& sleep_ms) { + check_id("job", id); + Json last; + std::string status; + PollOutcome out = poll( + policy, + [&](int) { + last = get(id); + const Json* job = last.find("job"); + status = job ? string_at(*job, "status") : ""; + return status == "done" || status == "rejected" || status == "deleted" || status == "expired"; + }, + sleep_ms); + if (!out.done) + throw llm::ApiError("speechmatics", 408, "poll_timeout", + "job " + id + " is still " + (status.empty() ? "pending" : status) + " after " + + std::to_string(out.waited_ms / 1000) + " s"); + if (status != "done") + throw llm::ApiError("speechmatics", 422, "job_" + status, "job " + id + " ended as " + status); + return last; +} + +Json Batch::temporary_key(const std::string& type, int ttl, const std::string& client_ref) { + if (type != "batch" && type != "rt") bad("type must be batch or rt"); + if (ttl < 60 || ttl > 86400) bad("ttl must be 60..86400 seconds"); + require_text("client_ref", client_ref, 256, false); + Json body = Json::object(); + body.set("ttl", Json(ttl)); + if (!client_ref.empty()) body.set("client_ref", Json(client_ref)); + Query q; + q.add("type", type); + return call(C::kTempKey, {}, &body, q); +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/batch.h b/src/vendors/audio/speechmatics/batch.h new file mode 100644 index 0000000..891306b --- /dev/null +++ b/src/vendors/audio/speechmatics/batch.h @@ -0,0 +1,65 @@ +#pragma once + +// Speechmatics batch transcription: jobs (create, list, get, delete), +// transcripts (json-v2, txt, srt), logs, usage, temporary keys. + +#include +#include +#include +#include + +#include "vendors/audio/audio_service.h" +#include "vendors/audio/speech.h" +#include "vendors/audio/speechmatics/calls.h" +#include "vendors/audio/speechmatics/client.h" + +namespace molo::audio::speechmatics { + +struct SpeechmaticsCaps { + static constexpr std::size_t kMaxUploadBytes = 128u * 1024u * 1024u; // our ceiling (provider allows 2 GB) + static constexpr std::size_t kMaxVocab = 1000; // additional_vocab entries + static constexpr int kMaxListLimit = 100; + static constexpr int kMaxWaitSeconds = 300; // transcript ?wait= + static constexpr std::size_t kMaxTtsChars = 5000; // our ceiling for one generate call +}; + +class SpeechmaticsService : public AudioService { + public: + explicit SpeechmaticsService(std::shared_ptr hosts) : AudioService(calls(), std::move(hosts)) {} + SpeechmaticsService() : SpeechmaticsService(SpeechmaticsEndpoints::from_env()) {} +}; + +struct JobConfig { + std::string language = "en"; // ISO 639-1, or "auto" for language identification + std::string operating_point; // "", standard, enhanced + std::string diarization; // "", none, speaker, channel, speaker_change + int enable_entities = -1; + std::string domain; + std::vector vocab; // additional_vocab content entries + std::string fetch_url; // fetch_data.url: Speechmatics downloads the media + Json extra; // other top level config sections (summarization_config, translation_config, ...) +}; + +class Batch : public SpeechmaticsService { + public: + using SpeechmaticsService::SpeechmaticsService; + Json config_json(const JobConfig& config) const; + clients::body::MultipartBody build(const JobConfig& config, const std::string& audio, + const std::string& filename, const std::string& content_type) const; + // Submits a job; exactly one of audio and config.fetch_url. + Json submit(const JobConfig& config, const std::string& audio, const std::string& filename = "audio", + const std::string& content_type = "application/octet-stream"); + Json list(const Json& query_object = Json()); + Json get(const std::string& job_id); + Json remove(const std::string& job_id, bool force, bool confirm); + // format: "" or json-v2 (JSON document), txt, srt (plain text in "content"). + Json transcript(const std::string& job_id, const std::string& format = ""); + std::string log(const std::string& job_id); + Json usage(const Json& query_object = Json()); + // Polls get() until done or rejected/deleted/expired, within the policy. + Json wait(const std::string& job_id, const PollPolicy& policy, const std::function& sleep_ms = nullptr); + // Temporary key for browsers: type batch or rt, ttl 60..86400 seconds. + Json temporary_key(const std::string& type, int ttl_seconds, const std::string& client_ref = ""); +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/calls.cpp b/src/vendors/audio/speechmatics/calls.cpp new file mode 100644 index 0000000..121fabc --- /dev/null +++ b/src/vendors/audio/speechmatics/calls.cpp @@ -0,0 +1,36 @@ +#include "vendors/audio/speechmatics/calls.h" + +namespace molo::audio::speechmatics { + +// One row per REST call. tools/check_spec.py parses the block between the +// markers: {id, method, path, host, body, fields, required, query, response, source}. +// CALLS-BEGIN +static const CallSpec kRows[] = { + {"jobs_create", "POST", "/jobs", "batch", "form", "config,data_file,text_file", "config", "wait,format", "json", "sdk:speechmatics-python-sdk sdk/batch _async_client.py POST /jobs"}, + {"jobs_list", "GET", "/jobs", "batch", "-", "", "", "limit,created_before,created_after", "json", "sdk:speechmatics-python-sdk sdk/batch _async_client.py GET /jobs; docs:api-ref/batch/list-all-jobs"}, + {"jobs_get", "GET", "/jobs/{jobid}", "batch", "-", "", "", "", "json", "sdk:speechmatics-python-sdk sdk/batch GET /jobs/{id}; docs:api-ref/batch/get-job-details"}, + {"jobs_delete", "DELETE", "/jobs/{jobid}", "batch", "-", "", "", "force", "json", "sdk:speechmatics-python-sdk sdk/batch DELETE /jobs/{id}; docs:api-ref/batch/delete-a-job"}, + {"jobs_transcript", "GET", "/jobs/{jobid}/transcript", "batch", "-", "", "", "format,wait", "json", "sdk:speechmatics-python-sdk sdk/batch GET /jobs/{id}/transcript; docs:api-ref/batch/get-the-transcript-for-a-transcription-job"}, + {"jobs_log", "GET", "/jobs/{jobid}/log", "batch", "-", "", "", "", "text", "docs:api-ref/batch/get-the-log-file-for-a-job"}, + {"usage", "GET", "/usage", "batch", "-", "", "", "since,until", "json", "docs:api-ref/batch/get-usage-statistics"}, + {"tts_generate", "POST", "/generate/{voice}", "tts", "json", "text", "text", "output_format", "bytes", "sdk:speechmatics-python-sdk sdk/tts _async_client.py POST /generate/{voice}"}, + {"temporary_key_create", "POST", "/api_keys", "mp", "json", "ttl,client_ref", "ttl", "type", "json", "docs:get-started/authentication (temporary keys)"}, + {"mgmt_projects_list", "GET", "/projects", "mgmt", "-", "", "", "", "json", "docs:api-ref/management/get-all-projects"}, + {"mgmt_projects_create", "POST", "/projects", "mgmt", "json", "*", "", "", "json", "docs:api-ref/management/post-a-new-project"}, + {"mgmt_projects_get", "GET", "/projects/{project_id}", "mgmt", "-", "", "", "", "json", "docs:api-ref/management/get-a-project-by-id"}, + {"mgmt_projects_update", "PUT", "/projects/{project_id}", "mgmt", "json", "*", "", "", "json", "docs:api-ref/management/update-a-project"}, + {"mgmt_projects_delete", "DELETE", "/projects/{project_id}", "mgmt", "-", "", "", "", "json", "docs:api-ref/management/delete-a-project"}, + {"mgmt_api_keys_list", "GET", "/api-keys", "mgmt", "-", "", "", "", "json", "docs:api-ref/management/get-all-api-keys"}, + {"mgmt_api_keys_create", "POST", "/api-keys", "mgmt", "json", "*", "", "", "json", "docs:api-ref/management/create-an-api-key"}, + {"mgmt_api_keys_delete", "DELETE", "/api-keys/{apikey_id}", "mgmt", "-", "", "", "", "json", "docs:api-ref/management/delete-an-api-key"}, +}; +// CALLS-END + +static_assert(sizeof(kRows) / sizeof(kRows[0]) == 17, "speechmatics call table size"); + +const CallTable& calls() { + static const CallTable table("speechmatics", kRows, sizeof(kRows) / sizeof(kRows[0])); + return table; +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/calls.h b/src/vendors/audio/speechmatics/calls.h new file mode 100644 index 0000000..d299c40 --- /dev/null +++ b/src/vendors/audio/speechmatics/calls.h @@ -0,0 +1,29 @@ +#pragma once + +// The Speechmatics call table. Speechmatics publishes its batch OpenAPI only +// inside the rendered docs, so the rows come from the official Python SDK +// (speechmatics-python-sdk: batch, tts) and the rendered API reference pages +// (api-ref/batch, api-ref/management). Hosts: batch, tts, mp (temporary keys, +// API key), mgmt (management API, management token). + +#include "vendors/audio/call_spec.h" + +namespace molo::audio::speechmatics { + +const CallTable& calls(); + +struct SpeechmaticsCalls { + static constexpr const char* kJobCreate = "jobs_create"; + static constexpr const char* kJobList = "jobs_list"; + static constexpr const char* kJobGet = "jobs_get"; + static constexpr const char* kJobDelete = "jobs_delete"; + static constexpr const char* kTranscript = "jobs_transcript"; + static constexpr const char* kJobLog = "jobs_log"; + static constexpr const char* kUsage = "usage"; + static constexpr const char* kTts = "tts_generate"; + static constexpr const char* kTempKey = "temporary_key_create"; + static constexpr const char* kProjects = "mgmt_projects_list"; + static constexpr const char* kApiKeys = "mgmt_api_keys_list"; +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/client.cpp b/src/vendors/audio/speechmatics/client.cpp new file mode 100644 index 0000000..81bf4f1 --- /dev/null +++ b/src/vendors/audio/speechmatics/client.cpp @@ -0,0 +1,54 @@ +#include "vendors/audio/speechmatics/client.h" + +#include "llm/config.h" + +namespace molo::audio::speechmatics { + +std::string SpeechmaticsEndpoints::batch_url() { + return allowlisted_base(kBatchEnv, kBatchDefault, + {"https://asr.api.speechmatics.com/v2", "https://eu1.asr.api.speechmatics.com/v2", + "https://eu2.asr.api.speechmatics.com/v2", "https://us1.asr.api.speechmatics.com/v2", + "https://us2.asr.api.speechmatics.com/v2", "https://au1.asr.api.speechmatics.com/v2"}); +} + +std::string SpeechmaticsEndpoints::rt_url() { + return allowlisted_base(kRtEnv, kRtDefault, + {"wss://global.rt.speechmatics.com/v2", "wss://eu.rt.speechmatics.com/v2", + "wss://us.rt.speechmatics.com/v2", "wss://au.rt.speechmatics.com/v2", + "wss://eu2.rt.speechmatics.com/v2"}); +} + +llm::ApiClientConfig SpeechmaticsEndpoints::config(const std::string& base, const char* key_env) { + llm::ApiClientConfig c; + c.vendor = kVendor; + c.base_url = base; + c.key_env = key_env; + c.auth = llm::AuthStyle::kBearer; + return c; +} + +std::shared_ptr SpeechmaticsEndpoints::from_env() { + auto hosts = std::make_shared(); + hosts->add("batch", HostFactory::from_env(config(batch_url(), kKeyEnv))); + hosts->add("tts", HostFactory::from_env(config(kTtsBase, kKeyEnv))); + hosts->add("mp", HostFactory::from_env(config(kMpBase, kKeyEnv))); + hosts->add("mgmt", HostFactory::from_env(config(kMpBase, kManagementEnv))); + return hosts; +} + +std::shared_ptr SpeechmaticsEndpoints::with(AudioHost batch, AudioHost tts, AudioHost mp, AudioHost mgmt) { + auto hosts = std::make_shared(); + hosts->add("batch", std::move(batch)); + hosts->add("tts", std::move(tts)); + hosts->add("mp", std::move(mp)); + hosts->add("mgmt", std::move(mgmt)); + return hosts; +} + +std::string SpeechmaticsEndpoints::key() { return llm::Config::get(kKeyEnv); } + +AudioSocket::Headers SpeechmaticsEndpoints::ws_headers(const std::string& key) { + return {{"Authorization", "Bearer " + key}}; +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/client.h b/src/vendors/audio/speechmatics/client.h new file mode 100644 index 0000000..355d020 --- /dev/null +++ b/src/vendors/audio/speechmatics/client.h @@ -0,0 +1,40 @@ +#pragma once + +// Speechmatics endpoints and credentials (all Bearer): +// batch https://asr.api.speechmatics.com/v2 $SPEECHMATICS_API_KEY (regions: eu1 eu2 us1 us2 au1 .asr.api.speechmatics.com) +// tts https://preview.tts.speechmatics.com $SPEECHMATICS_API_KEY +// mp https://mp.speechmatics.com/v1 $SPEECHMATICS_API_KEY (temporary keys) +// mgmt https://mp.speechmatics.com/v1 $SPEECHMATICS_MANAGEMENT_TOKEN (projects, API keys) +// realtime wss://eu.rt.speechmatics.com/v2 same API key +// SPEECHMATICS_BATCH_URL and SPEECHMATICS_RT_URL select a documented region. + +#include +#include + +#include "vendors/audio/hosts.h" +#include "vendors/audio/socket.h" + +namespace molo::audio::speechmatics { + +struct SpeechmaticsEndpoints { + static constexpr const char* kVendor = "speechmatics"; + static constexpr const char* kKeyEnv = "SPEECHMATICS_API_KEY"; + static constexpr const char* kManagementEnv = "SPEECHMATICS_MANAGEMENT_TOKEN"; + static constexpr const char* kBatchEnv = "SPEECHMATICS_BATCH_URL"; + static constexpr const char* kRtEnv = "SPEECHMATICS_RT_URL"; + static constexpr const char* kBatchDefault = "https://asr.api.speechmatics.com/v2"; + static constexpr const char* kTtsBase = "https://preview.tts.speechmatics.com"; + static constexpr const char* kMpBase = "https://mp.speechmatics.com/v1"; + static constexpr const char* kRtDefault = "wss://eu.rt.speechmatics.com/v2"; + + static std::string batch_url(); + static std::string rt_url(); + static llm::ApiClientConfig config(const std::string& base, const char* key_env); + static std::shared_ptr from_env(); + // Injected hosts (tests): "batch", "tts", "mp", "mgmt". + static std::shared_ptr with(AudioHost batch, AudioHost tts, AudioHost mp, AudioHost mgmt); + static std::string key(); + static AudioSocket::Headers ws_headers(const std::string& key); +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/manage.cpp b/src/vendors/audio/speechmatics/manage.cpp new file mode 100644 index 0000000..2353408 --- /dev/null +++ b/src/vendors/audio/speechmatics/manage.cpp @@ -0,0 +1,38 @@ +#include "vendors/audio/speechmatics/manage.h" + +namespace molo::audio::speechmatics { + +using C = SpeechmaticsCalls; + +Json SmManage::projects() { return call(C::kProjects); } + +Json SmManage::project(const std::string& id) { + check_id("project", id); + return call("mgmt_projects_get", {id}); +} + +Json SmManage::api_keys() { return call(C::kApiKeys); } + +Json SmManage::create_api_key(const Json& body, bool confirm) { + check_body(table_.get("mgmt_api_keys_create"), &body); + require_confirm("creating an API key", confirm); + return call("mgmt_api_keys_create", {}, &body); +} + +Json SmManage::create_project(const Json& body, bool confirm) { + check_body(table_.get("mgmt_projects_create"), &body); + require_confirm("creating a project", confirm); + return call("mgmt_projects_create", {}, &body); +} + +Json SmManage::delete_project(const std::string& id, bool confirm) { + check_id("project", id); + return call_destructive("mgmt_projects_delete", {id}, nullptr, Query(), confirm); +} + +Json SmManage::delete_api_key(const std::string& id, bool confirm) { + check_id("api key", id); + return call_destructive("mgmt_api_keys_delete", {id}, nullptr, Query(), confirm); +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/manage.h b/src/vendors/audio/speechmatics/manage.h new file mode 100644 index 0000000..c7784bf --- /dev/null +++ b/src/vendors/audio/speechmatics/manage.h @@ -0,0 +1,25 @@ +#pragma once + +// Management API (https://mp.speechmatics.com/v1, management token): +// projects and API keys. Creating keys and every delete need confirm. + +#include + +#include "vendors/audio/speechmatics/batch.h" + +namespace molo::audio::speechmatics { + +class SmManage : public SpeechmaticsService { + public: + using SpeechmaticsService::SpeechmaticsService; + Json projects(); + Json project(const std::string& project_id); + Json api_keys(); + // Creates a key (the secret is shown once): a write, confirm required. + Json create_api_key(const Json& body, bool confirm); + Json create_project(const Json& body, bool confirm); + Json delete_project(const std::string& project_id, bool confirm); + Json delete_api_key(const std::string& api_key_id, bool confirm); +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/realtime.cpp b/src/vendors/audio/speechmatics/realtime.cpp new file mode 100644 index 0000000..deaf0a0 --- /dev/null +++ b/src/vendors/audio/speechmatics/realtime.cpp @@ -0,0 +1,138 @@ +#include "vendors/audio/speechmatics/realtime.h" + +#include "vendors/audio/caps.h" + +namespace molo::audio::speechmatics { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("speechmatics", why); } + +std::string str_at(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} +} // namespace + +Vocabulary RealtimeSession::vocabulary() { + return Vocabulary("message", + {"StartRecognition", "EndOfStream", "SetRecognitionConfig", "GetSpeakers", "ForceEndOfUtterance"}, + {"RecognitionStarted", "AudioAdded", "AddPartialTranscript", "AddTranscript", "EndOfTranscript", + "EndOfUtterance", "AudioEventStarted", "AudioEventEnded", "AddTranslation", + "AddPartialTranslation", "SpeakersResult", "LanguageInfo", "Info", "Warning", "Error"}); +} + +RealtimeSession::RealtimeSession(std::string api_key, RealtimeConfig config) : config_(std::move(config)) { + std::string url = config_.url.empty() ? SpeechmaticsEndpoints::rt_url() : config_.url; + socket_ = std::make_unique("speechmatics", url, SpeechmaticsEndpoints::ws_headers(api_key), + vocabulary(), config_.socket); +} + +Json RealtimeSession::start_message() const { + const RealtimeConfig& c = config_; + if (c.language.empty() || c.language.size() > 16) invalid("language is required"); + if (c.encoding != "pcm_f32le" && c.encoding != "pcm_s16le" && c.encoding != "mulaw") + invalid("encoding must be pcm_f32le, pcm_s16le or mulaw"); + if (c.sample_rate < 8000 || c.sample_rate > 96000) invalid("sample_rate must be 8000..96000"); + Json audio = Json::object(); + audio.set("type", Json("raw")); + audio.set("encoding", Json(c.encoding)); + audio.set("sample_rate", Json(c.sample_rate)); + Json tc = Json::object(); + tc.set("language", Json(c.language)); + tc.set("enable_partials", Json(c.enable_partials != 0)); + if (c.max_delay >= 0) tc.set("max_delay", Json(c.max_delay)); + if (!c.diarization.empty()) { + if (c.diarization != "none" && c.diarization != "speaker") invalid("diarization must be none or speaker"); + tc.set("diarization", Json(c.diarization)); + } + if (!c.operating_point.empty()) { + if (c.operating_point != "standard" && c.operating_point != "enhanced") invalid("operating_point must be standard or enhanced"); + tc.set("operating_point", Json(c.operating_point)); + } + if (!c.vocab.empty()) { + if (c.vocab.size() > SpeechmaticsCaps::kMaxVocab) invalid("too many vocabulary entries"); + Json arr = Json::array(); + for (const auto& v : c.vocab) { + if (v.empty() || v.size() > 100) invalid("vocabulary entries are 1..100 characters"); + Json e = Json::object(); + e.set("content", Json(v)); + arr.push(e); + } + tc.set("additional_vocab", arr); + } + if (c.extra.is_object()) + for (const auto& [k, v] : c.extra.as_object()) { + if (tc.find(k)) invalid("extra repeats the field '" + k + "'"); + tc.set(k, v); + } + Json m = Json::object(); + m.set("message", Json("StartRecognition")); + m.set("audio_format", audio); + m.set("transcription_config", tc); + return m; +} + +void RealtimeSession::connect() { + Json start = start_message(); + socket_->connect(); + socket_->send_json(start); +} + +void RealtimeSession::send_audio(const std::string& audio) { + if (audio.empty() || audio.size() > 1024u * 1024u) invalid("audio chunk must be 1 byte..1 MiB"); + socket_->send_binary(audio); + ++frames_; +} + +void RealtimeSession::end_of_stream() { + Json m = Json::object(); + m.set("message", Json("EndOfStream")); + m.set("last_seq_no", Json(frames_)); + socket_->send_json(m); +} + +RealtimeResult RealtimeSession::collect(int max_wait_ms) { + RealtimeResult out; + for (std::size_t n = 0; n < AudioCaps::kMaxEventsPerCollect; ++n) { + auto ev = socket_->next(max_wait_ms); + if (!ev) return out; + ++out.events; + if (ev->kind == SocketEvent::Kind::kClose) { + out.closed = true; + return out; + } + if (!ev->json_ok) continue; + const std::string& t = ev->type; + if (t == "RecognitionStarted") { + out.session_id = str_at(ev->json, "id"); + } else if (t == "AudioAdded") { + if (const Json* s = ev->json.find("seq_no"); s && s->is_number()) out.last_seq_no = s->as_int(); + } else if (t == "AddPartialTranscript" || t == "AddTranscript") { + std::string text; + if (const Json* md = ev->json.find("metadata"); md && md->is_object()) text = str_at(*md, "transcript"); + if (t == "AddTranscript") { + out.finals.push_back(text); + if (!text.empty()) { + bool glue = out.transcript.empty() || text.front() == ' ' || out.transcript.back() == ' '; + out.transcript += (glue ? "" : " ") + text; + } + out.partial.clear(); + } else { + out.partial = text; + } + } else if (t == "EndOfTranscript") { + out.ended = true; + return out; + } else if (t == "Warning") { + out.warnings.push_back(str_at(ev->json, "reason")); + } else if (t == "Error") { + out.error_type = str_at(ev->json, "type"); + out.error_reason = str_at(ev->json, "reason"); + if (out.error_type.empty()) out.error_type = "error"; + return out; + } + } + return out; +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/realtime.h b/src/vendors/audio/speechmatics/realtime.h new file mode 100644 index 0000000..eca27fd --- /dev/null +++ b/src/vendors/audio/speechmatics/realtime.h @@ -0,0 +1,70 @@ +#pragma once + +// Speechmatics realtime transcription over WebSocket (wss://.rt.speechmatics.com/v2). +// Protocol (rt_models.py ClientMessageType / ServerMessageType in the Python SDK): +// the client sends StartRecognition, then binary AddAudio frames, then +// EndOfStream; the server answers RecognitionStarted, AudioAdded, +// AddPartialTranscript, AddTranscript, EndOfTranscript, Info, Warning, Error. + +#include +#include +#include +#include + +#include "vendors/audio/socket.h" +#include "vendors/audio/speechmatics/batch.h" + +namespace molo::audio::speechmatics { + +struct RealtimeConfig { + std::string language = "en"; + std::string encoding = "pcm_s16le"; // pcm_f32le pcm_s16le mulaw (raw audio) + int sample_rate = 16000; + int enable_partials = 1; + double max_delay = -1; // seconds, 0.7..4 per the docs; -1 unset + std::string diarization; // "", none, speaker + std::string operating_point; // "", standard, enhanced + std::vector vocab; + Json extra; // other transcription_config fields + std::string url; // "" = SpeechmaticsEndpoints::rt_url() + SocketOptions socket; +}; + +struct RealtimeResult { + std::string session_id; + std::vector finals; // AddTranscript metadata.transcript, in order + std::string transcript; // finals joined by a space + std::string partial; + long last_seq_no = 0; // from AudioAdded + std::string error_type, error_reason; + std::vector warnings; + bool ended = false; // EndOfTranscript + bool closed = false; + std::size_t events = 0; +}; + +class RealtimeSession { + public: + RealtimeSession(std::string api_key, RealtimeConfig config); + std::string url() const { return socket_->url(); } + void connect(); // opens the socket and sends StartRecognition + void close() { socket_->close(); } + bool is_open() const { return socket_->is_open(); } + + Json start_message() const; // the StartRecognition message (validates) + void send_audio(const std::string& audio); // binary AddAudio, 1..1 MiB; counted for EndOfStream + void end_of_stream(); // {"message":"EndOfStream","last_seq_no":N} + std::optional next(int wait_ms) { return socket_->next(wait_ms); } + // Reads until EndOfTranscript, error, close, or silence of `max_wait_ms`. + RealtimeResult collect(int max_wait_ms); + long audio_frames_sent() const { return frames_; } + + static Vocabulary vocabulary(); + + private: + RealtimeConfig config_; + std::unique_ptr socket_; + long frames_ = 0; +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/speech.cpp b/src/vendors/audio/speechmatics/speech.cpp new file mode 100644 index 0000000..cda11f1 --- /dev/null +++ b/src/vendors/audio/speechmatics/speech.cpp @@ -0,0 +1,94 @@ +#include "vendors/audio/speechmatics/speech.h" + +namespace molo::audio::speechmatics { + +namespace { +[[noreturn]] void invalid(const std::string& why) { throw llm::ApiError::invalid("speechmatics", why); } +} // namespace + +SpeechResult SpeechmaticsSynthesizer::synthesize(const SpeechRequest& r) { + validate(r); + std::string voice = r.voice.empty() ? "sarah" : r.voice; + if (r.format == "mp3") invalid("the preview TTS API produces wav or pcm only"); + if (r.sample_rate != 0 && r.sample_rate != 16000) invalid("the preview TTS API produces 16000 Hz only"); + SmTtsAudio a = tts_.generate(voice, r.text, r.format == "wav" ? "wav_16000" : "pcm_16000"); + SpeechResult out; + out.audio = std::move(a.bytes); + out.content_type = a.content_type.empty() ? content_type_for(r.format) : a.content_type; + out.format = r.format; + out.provider = name(); + return out; +} + +Json SpeechmaticsSynthesizer::voices() { + Json list = Json::array(); + for (const char* v : {"sarah", "theo", "megan", "jack"}) { + Json e = Json::object(); + e.set("id", Json(v)); + list.push(e); + } + Json o = Json::object(); + o.set("voices", list); + o.set("source", Json("python SDK Voice enum; the API has no list endpoint")); + return o; +} + +JobConfig SpeechmaticsRecognizer::map(const TranscribeRequest& r) const { + validate(r); + JobConfig c; + c.language = r.language.empty() ? "auto" : r.language; + c.operating_point = r.model; // "standard" / "enhanced" or empty + if (r.diarize) c.diarization = "speaker"; + c.vocab = r.keywords; + c.fetch_url = r.url; + return c; +} + +Transcript SpeechmaticsRecognizer::fold(const Json& doc) { + Transcript t; + t.provider = "speechmatics"; + t.raw = doc; + if (!doc.is_object()) return t; + if (const Json* md = doc.find("metadata"); md && md->is_object()) { + if (const Json* tc = md->find("transcription_config"); tc && tc->is_object()) + t.language = llm::Service::string_at(*tc, "language"); + } + if (const Json* job = doc.find("job"); job && job->is_object()) { + if (const Json* d = job->find("duration"); d && d->is_number()) t.duration = d->as_number(); + } + const Json* results = doc.find("results"); + if (!results || !results->is_array()) return t; + for (const Json& r : results->as_array()) { + if (!r.is_object()) continue; + const Json* alts = r.find("alternatives"); + if (!alts || !alts->is_array() || alts->as_array().empty()) continue; + const Json& alt = alts->as_array()[0]; + std::string content = llm::Service::string_at(alt, "content"); + std::string type = llm::Service::string_at(r, "type"); + if (type == "punctuation") { + t.text += content; // punctuation attaches to the previous word + continue; + } + if (!t.text.empty()) t.text += " "; + t.text += content; + TranscriptWord w; + w.text = content; + if (const Json* x = r.find("start_time"); x && x->is_number()) w.start = x->as_number(); + if (const Json* x = r.find("end_time"); x && x->is_number()) w.end = x->as_number(); + if (const Json* x = alt.find("confidence"); x && x->is_number()) w.confidence = x->as_number(); + w.speaker = llm::Service::string_at(alt, "speaker"); + t.words.push_back(w); + } + return t; +} + +Transcript SpeechmaticsRecognizer::transcribe(const TranscribeRequest& r) { + JobConfig c = map(r); + Json created = batch_.submit(c, r.audio, r.filename, r.content_type); + std::string id = llm::Service::string_at(created, "id"); + if (id.empty()) throw llm::ApiError("speechmatics", 502, "BAD_RESPONSE", "job creation returned no id"); + batch_.wait(id, PollPolicy::clamped(3000, r.max_wait_seconds), sleep_ms_); + return fold(batch_.transcript(id)); +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/speech.h b/src/vendors/audio/speechmatics/speech.h new file mode 100644 index 0000000..19a36c1 --- /dev/null +++ b/src/vendors/audio/speechmatics/speech.h @@ -0,0 +1,47 @@ +#pragma once + +// Speechmatics behind the neutral speech seam. + +#include +#include + +#include "vendors/audio/speech.h" +#include "vendors/audio/speechmatics/batch.h" +#include "vendors/audio/speechmatics/tts.h" + +namespace molo::audio::speechmatics { + +class SpeechmaticsSynthesizer : public SpeechSynthesizer { + public: + explicit SpeechmaticsSynthesizer(std::shared_ptr hosts = SpeechmaticsEndpoints::from_env()) + : tts_(std::move(hosts)) {} + std::string name() const override { return "speechmatics"; } + bool configured() const override { return tts_.configured(); } + std::size_t max_chars() const override { return SpeechmaticsCaps::kMaxTtsChars; } + // The preview API produces 16 kHz wav or raw pcm only. + std::vector formats() const override { return {"wav", "pcm"}; } + SpeechResult synthesize(const SpeechRequest& request) override; + Json voices() override; // the fixed English voice list from the SDK (no list endpoint exists) + + private: + SmTts tts_; +}; + +class SpeechmaticsRecognizer : public SpeechRecognizer { + public: + explicit SpeechmaticsRecognizer(std::shared_ptr hosts = SpeechmaticsEndpoints::from_env(), + std::function sleep_ms = nullptr) + : batch_(std::move(hosts)), sleep_ms_(std::move(sleep_ms)) {} + std::string name() const override { return "speechmatics"; } + bool configured() const override { return batch_.configured(); } + Transcript transcribe(const TranscribeRequest& request) override; + JobConfig map(const TranscribeRequest& request) const; + // Folds a json-v2 transcript document. + static Transcript fold(const Json& doc); + + private: + Batch batch_; + std::function sleep_ms_; +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/tts.cpp b/src/vendors/audio/speechmatics/tts.cpp new file mode 100644 index 0000000..c50c455 --- /dev/null +++ b/src/vendors/audio/speechmatics/tts.cpp @@ -0,0 +1,38 @@ +#include "vendors/audio/speechmatics/tts.h" + +namespace molo::audio::speechmatics { + +using C = SpeechmaticsCalls; + +bool SmTts::valid_voice(const std::string& v) { return v == "sarah" || v == "theo" || v == "megan" || v == "jack"; } +bool SmTts::valid_format(const std::string& f) { return f == "wav_16000" || f == "pcm_16000"; } + +Json SmTts::build(const std::string& text) const { + require_text("text", text, SpeechmaticsCaps::kMaxTtsChars); + Json o = Json::object(); + o.set("text", Json(text)); + return o; +} + +Query SmTts::query(const std::string& format) const { + if (!valid_format(format)) bad("output_format must be wav_16000 or pcm_16000"); + Query q; + q.add("output_format", format); + return q; +} + +SmTtsAudio SmTts::generate(const std::string& voice, const std::string& text, const std::string& format) { + if (!valid_voice(voice)) bad("voice must be sarah, theo, megan or jack"); + Json body = build(text); + llm::RawResponse raw = call_raw(C::kTts, {voice}, &body, query(format)); + return SmTtsAudio{raw.body, raw.content_type}; +} + +ByteStreamer::Result SmTts::stream(const std::string& voice, const std::string& text, const std::string& format, + const ChunkSink& sink) { + if (!valid_voice(voice)) bad("voice must be sarah, theo, megan or jack"); + Json body = build(text); + return call_stream(C::kTts, {voice}, &body, query(format), sink); +} + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/speechmatics/tts.h b/src/vendors/audio/speechmatics/tts.h new file mode 100644 index 0000000..6a0e558 --- /dev/null +++ b/src/vendors/audio/speechmatics/tts.h @@ -0,0 +1,30 @@ +#pragma once + +// Speechmatics text to speech (preview): POST /generate/{voice}?output_format=... +// with {"text": ...}. Voices and formats are the ones the SDK enumerates. + +#include + +#include "vendors/audio/speechmatics/batch.h" + +namespace molo::audio::speechmatics { + +struct SmTtsAudio { + std::string bytes; + std::string content_type; +}; + +class SmTts : public SpeechmaticsService { + public: + using SpeechmaticsService::SpeechmaticsService; + // SDK enum: sarah, theo, megan, jack (English). Formats: wav_16000, pcm_16000. + static bool valid_voice(const std::string& voice); + static bool valid_format(const std::string& format); + Json build(const std::string& text) const; + Query query(const std::string& output_format) const; + SmTtsAudio generate(const std::string& voice, const std::string& text, const std::string& output_format = "wav_16000"); + ByteStreamer::Result stream(const std::string& voice, const std::string& text, const std::string& output_format, + const ChunkSink& sink); +}; + +} // namespace molo::audio::speechmatics diff --git a/src/vendors/audio/tools/check_spec.py b/src/vendors/audio/tools/check_spec.py new file mode 100644 index 0000000..904bd99 --- /dev/null +++ b/src/vendors/audio/tools/check_spec.py @@ -0,0 +1,731 @@ +#!/usr/bin/env python3 +"""Check the audio call tables and WebSocket vocabularies against the vendors' published specs. + +Fails (exit 1) when a declared call is absent from its source, a body field, +required field or query parameter we use is not in the source, a WebSocket +query parameter or message name in the C++ differs from the vendor's +AsyncAPI document or SDK types, or a source operation in a covered product +area is neither in the table nor in an explicit exclusion list. + +Python 3 stdlib only, no credentials, read-only network access. + + python3 -I src/vendors/audio/tools/check_spec.py [provider ...] + +Sources (all public, fetched at run time): + elevenlabs https://api.elevenlabs.io/openapi.json, https://elevenlabs.io/docs/asyncapi/*.json + assemblyai https://www.assemblyai.com/docs/openapi.yaml, docs/api-reference/specs/{streaming-token,llm-gateway,streaming}.yaml + deepgram deepgram-api-specs openapi.yml and asyncapi.yml (raw.githubusercontent.com) + cartesia cartesia-python api.md, .stats.yml and *_params.py (the docs host answers bots with a challenge page) + speechmatics speechmatics-python-sdk (batch, rt, tts) and docs.speechmatics.com/api-ref/*.md + hume dev.hume.ai/openapi/*.json and hume-python-sdk (tts/raw_client.py, event types) +YAML documents are scanned by indentation (no YAML parser in the stdlib): operations, +operationIds, query parameters, request schema properties and channel parameters. +""" +import json +import os +import re +import sys +import time +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +SRC = os.path.join(HERE, "..") + +ROW = re.compile(r'^\s*\{"([^"]*)", "([A-Z]+)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)"\},?\s*$') + +GH = "https://raw.githubusercontent.com/" +_cache = {} +BAD = [] +STATS = {} + + +def fetch(url, binary=False): + if url not in _cache: + last = None + for attempt in range(3): + try: + req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"}) + with urllib.request.urlopen(req, timeout=60) as r: + _cache[url] = r.read().decode("utf-8", "replace") + break + except Exception as e: # noqa: BLE001 + last = e + time.sleep(1 + attempt) + else: + raise SystemExit("cannot fetch %s: %s" % (url, last)) + return _cache[url] + + +def fail(provider, msg): + BAD.append("%s: %s" % (provider, msg)) + + +def count(provider, key, n=1): + STATS.setdefault(provider, {}) + STATS[provider][key] = STATS[provider].get(key, 0) + n + + +def read_rows(provider): + text = open(os.path.join(SRC, provider, "calls.cpp")).read() + a = text.index("// CALLS-BEGIN") + b = text.index("// CALLS-END") + out = [] + for line in text[a:b].splitlines(): + m = ROW.match(line) + if m: + out.append(m.groups()) + if not out: + fail(provider, "no rows parsed from calls.cpp") + return out + + +def norm_path(p): + return re.sub(r"\{[^}]*\}|:[A-Za-z_]+", "{}", p) + + +def csv(s): + return [x for x in s.split(",") if x] + + +def cpp_words(provider, rel, func): + """Quoted strings inside the function whose signature contains `func` in /.""" + text = open(os.path.join(SRC, provider, rel)).read() + i = text.index(func) + j = text.index("\n}\n", i) + body = text[i:j] + return re.findall(r'"([A-Za-z0-9_.:\-]+)"', body) + + +def cpp_quoted_after(provider, rel, start, end): + text = open(os.path.join(SRC, provider, rel)).read() + i = text.index(start) + j = text.index(end, i) + return re.findall(r'"([A-Za-z0-9_.:\-]+)"', text[i:j]) + + +# ---------------------------------------------------------------- OpenAPI (JSON) +class OpenApi: + def __init__(self, doc): + self.d = doc + + def res(self, x, depth=0): + while isinstance(x, dict) and "$ref" in x and depth < 30: + cur = self.d + for part in x["$ref"].lstrip("#/").split("/"): + cur = cur[part.replace("~1", "/")] + x = cur + depth += 1 + return x + + def props(self, s, depth=0): + s = self.res(s) + names, req, open_ = [], [], False + if not isinstance(s, dict) or depth > 8: + return names, req, True + for sub in s.get("allOf", []): + n, r, o = self.props(sub, depth + 1) + names += n + req += r + open_ = open_ or o + for key in ("oneOf", "anyOf"): + for sub in s.get(key, []): + n, r, o = self.props(sub, depth + 1) + names += n + open_ = open_ or o + names += list((s.get("properties") or {}).keys()) + req += s.get("required", []) + if not names and not s.get("allOf") and not s.get("oneOf") and not s.get("anyOf"): + open_ = True + return names, req, open_ + + def op(self, method, path): + item = self.d["paths"].get(path) + if not item: + return None + return item.get(method.lower()) + + def query(self, item_path, op): + item = self.d["paths"][item_path] + params = [self.res(x) for x in item.get("parameters", []) + op.get("parameters", [])] + return [x["name"] for x in params if x.get("in") == "query"] + + def body_schema(self, op): + rb = self.res(op.get("requestBody", {})) if op.get("requestBody") else None + if not rb: + return None + for ct, v in rb.get("content", {}).items(): + return v.get("schema", {}) + return None + + +def check_json_rows(provider, rows, api, prefix_ok): + seen = set() + for (rid, method, path, host, body, fields, required, query, response, source) in rows: + if not source.startswith(prefix_ok): + continue + op = api.op(method, path) + if op is None: + fail(provider, "%s %s (%s) is not in the spec" % (method, path, rid)) + continue + seen.add((method, path)) + count(provider, "rows checked") + opid = source.split(":", 1)[1] + if op.get("operationId") and op.get("operationId") != opid: + fail(provider, "%s: operationId is %s, table says %s" % (rid, op.get("operationId"), opid)) + qs = api.query(path, op) + for q in csv(query): + if q not in qs: + fail(provider, "%s: query parameter %s not in spec (%s)" % (rid, q, ",".join(qs))) + schema = api.body_schema(op) + if body == "-": + if schema is not None and csv(required): + fail(provider, "%s: table has no body but the spec does" % rid) + elif schema is None: + fail(provider, "%s: table has a %s body but the spec has none" % (rid, body)) + else: + names, req, open_ = api.props(schema) + if body != "bytes": + if fields == "*": + if names and not open_: + fail(provider, "%s: fields is * but the spec lists properties" % rid) + else: + for f in csv(fields): + if names and f not in names: + fail(provider, "%s: body field %s not in spec" % (rid, f)) + for f in csv(required): + if f not in req: + fail(provider, "%s: required field %s is not required in spec" % (rid, f)) + return seen + + +# ---------------------------------------------------------------- YAML scan +def yaml_ops(text): + """-> {(METHOD, path): {'operationId', 'query', 'body_ref'}} from an OpenAPI YAML by indentation.""" + ops = {} + lines = text.splitlines() + in_paths = False + path = method = None + for i, line in enumerate(lines): + if re.match(r"^paths:\s*$", line): + in_paths = True + continue + if in_paths and re.match(r"^\S", line): + in_paths = False + if not in_paths: + continue + m = re.match(r"^ ['\"]?(/[^'\":]*)['\"]?:\s*$", line) + if m: + path = m.group(1) + method = None + continue + m = re.match(r"^ (get|post|put|patch|delete):\s*$", line) + if m and path: + method = m.group(1).upper() + ops[(method, path)] = {"operationId": "", "query": [], "body_ref": ""} + continue + if not method: + continue + cur = ops[(method, path)] + m = re.match(r"^ operationId:\s*(\S+)", line) + if m: + cur["operationId"] = m.group(1).strip("'\"") + m = re.match(r"^\s+- name:\s*(\S+)", line) + if m and any(re.match(r"^\s+in:\s*query", lines[k]) for k in range(i + 1, min(i + 5, len(lines))) + if not re.match(r"^\s+- name:", lines[k])): + cur["query"].append(m.group(1).strip("'\"")) + if "requestBody:" in line: + for k in range(i, min(i + 14, len(lines))): + mm = re.search(r"\$ref:\s*['\"]#/components/schemas/([^'\"]+)['\"]", lines[k]) + if mm: + cur["body_ref"] = mm.group(1) + break + return ops + + +def yaml_schema(text, name): + """-> (property names, required names) of components.schemas. by indentation.""" + lines = text.splitlines() + start = None + for i, line in enumerate(lines): + if re.match(r"^ %s:\s*$" % re.escape(name), line): + start = i + break + if start is None: + return [], [] + block = [] + for line in lines[start + 1:]: + if line.strip() and re.match(r"^ {0,4}\S", line): + break + block.append(line) + props, req = [], [] + i = 0 + while i < len(block): + line = block[i] + m = re.match(r"^( +)properties:\s*$", line) + if m: + base = len(m.group(1)) + j = i + 1 + while j < len(block): + lj = block[j] + if lj.strip() == "": + j += 1 + continue + ind = len(lj) - len(lj.lstrip()) + if ind <= base: + break + mm = re.match(r"^ {%d}['\"]?([A-Za-z0-9_\-\.\[\]]+)['\"]?:" % (base + 2), lj) + if mm: + props.append(mm.group(1)) + j += 1 + i = j + continue + m = re.match(r"^( +)required:\s*$", line) + if m: + j = i + 1 + while j < len(block) and re.match(r"^ +- ", block[j]): + req.append(block[j].strip()[2:].strip("'\"")) + j += 1 + i = j + continue + i += 1 + return props, req + + +def check_yaml_rows(provider, rows, text, host, source_prefix, exclude_ops=(), hosts=None): + ops = yaml_ops(text) + seen = set() + for (rid, method, path, h, body, fields, required, query, response, source) in rows: + if h not in (hosts or (host,)) or not source.startswith(source_prefix): + continue + op = ops.get((method, path)) + if op is None: + fail(provider, "%s %s (%s) is not in the spec" % (method, path, rid)) + continue + seen.add((method, path)) + count(provider, "rows checked") + opid = source.split(":", 1)[1] + if op["operationId"] and op["operationId"] != opid: + fail(provider, "%s: operationId is %s, table says %s" % (rid, op["operationId"], opid)) + for q in csv(query): + if q not in op["query"]: + fail(provider, "%s: query parameter %s not in spec (%s)" % (rid, q, ",".join(op["query"]))) + if body == "json" and fields != "*": + if not op["body_ref"]: + fail(provider, "%s: table has a body but the spec names no schema" % rid) + else: + props, req = yaml_schema(text, op["body_ref"]) + for f in csv(fields): + if props and f not in props: + fail(provider, "%s: body field %s not in schema %s" % (rid, f, op["body_ref"])) + for f in csv(required): + if req and f not in req: + fail(provider, "%s: required field %s is not required in schema %s" % (rid, f, op["body_ref"])) + for key in ops: + if key not in seen and key not in exclude_ops: + fail(provider, "spec operation %s %s is neither in the table nor excluded" % key) + else: + count(provider, "spec operations covered" if key in seen else "spec operations excluded") + return seen + + +def words_in(provider, what, words, text): + for w in words: + if not re.search(r'(?', api_md): + titles.add((m.group(1).upper(), norm_path(m.group(2)).rstrip("/") or "/")) + stats = fetch(CA_SDK + ".stats.yml") + n = int(re.search(r"configured_endpoints:\s*(\d+)", stats).group(1)) + if n != len(rows): + fail(p, ".stats.yml says %d configured endpoints, the table has %d rows" % (n, len(rows))) + count(p, "sdk endpoints (stats)", n) + mine = set() + for (rid, method, path, h, body, fields, required, query, response, source) in rows: + key = (method, norm_path(path).rstrip("/") or "/") + mine.add(key) + if key not in titles: + fail(p, "%s: %s %s is not in the SDK api.md" % (rid, method, path)) + count(p, "rows checked") + if rid in CA_PARAMS and fields != "*": + txt = fetch(CA_SDK + "src/cartesia/" + CA_PARAMS[rid]) + for f in csv(fields): + base = f.replace("[]", "").replace("[", "_").replace("]", "") + if not re.search(r"\b%s\b" % re.escape(f.split("[")[0].replace("-", "_")), txt) and not re.search(r"\b%s\b" % re.escape(base), txt): + fail(p, "%s: body field %s not in %s" % (rid, f, CA_PARAMS[rid])) + if rid in CA_QUERY_PARAMS: + txt = fetch(CA_SDK + "src/cartesia/" + CA_QUERY_PARAMS[rid]) + for q in csv(query): + if not re.search(r"^ %s: " % re.escape(q), txt, re.M): + fail(p, "%s: query parameter %s not in %s" % (rid, q, CA_QUERY_PARAMS[rid])) + for key in titles: + if key not in mine: + fail(p, "SDK endpoint %s %s is not in the table" % key) + ws_doc = fetch("https://docs.cartesia.ai/api-reference/tts/websocket.md") + if "address: /tts/websocket" not in ws_doc: + fail(p, "docs page no longer shows address /tts/websocket") + count(p, "ws names checked") + for fname, want in (("stt/auto_finalize_websocket_params.py", cpp_quoted_after(p, "stt.cpp", "q.add(\"encoding\"", "} else {")), + ("stt/manual_finalize_websocket_params.py", cpp_quoted_after(p, "stt.cpp", "} else {", "std::string base = url.empty()"))): + txt = fetch(CA_SDK + "src/cartesia/types/" + fname) + for w in want: + if re.fullmatch(r"[a-z_]+", w) and not re.search(r"^ %s: " % re.escape(w), txt, re.M): + fail(p, "stt query parameter %s not in %s" % (w, fname)) + count(p, "ws names checked") + resp = fetch(CA_SDK + "src/cartesia/types/websocket_response.py") + for w in ("chunk", "flush_done", "done", "timestamps", "phoneme_timestamps", "error"): + if 'Literal["%s"]' % w not in resp: + fail(p, "tts ws response type %s not in websocket_response.py" % w) + count(p, "ws names checked") + for w, fname in (("turn.start", "stt/stt_auto_finalize_turn_start.py"), ("turn.update", "stt/stt_auto_finalize_turn_update.py"), + ("turn.eager_end", "stt/stt_auto_finalize_turn_eager_end.py"), ("turn.end", "stt/stt_auto_finalize_turn_end.py"), + ("connected", "stt/stt_auto_finalize_connected.py"), ("transcript", "stt/stt_manual_finalize_transcript_response.py"), + ("flush_done", "stt/stt_manual_finalize_flush_done_response.py"), ("done", "stt/stt_manual_finalize_done_response.py")): + if 'Literal["%s"]' % w not in fetch(CA_SDK + "src/cartesia/types/" + fname): + fail(p, "stt event type %s not in %s" % (w, fname)) + count(p, "ws names checked") + req = fetch(CA_SDK + "src/cartesia/types/stt/stt_auto_finalize_websocket_request.py") + for w in ("close", "config", "eager_end_threshold", "end_threshold", "end_timeout_ms", "start_threshold"): + if w not in req: + fail(p, "auto-finalize request name %s not in the SDK" % w) + count(p, "ws names checked") + gen = fetch(CA_SDK + "src/cartesia/types/generation_request.py") + for w in ("context_id", "continue", "flush", "add_timestamps", "add_phoneme_timestamps", "max_buffer_delay_ms", "speed", "language"): + if w not in gen: + fail(p, "generation request field %s not in the SDK" % w) + count(p, "ws names checked") + + +SM_SDK = GH + "speechmatics/speechmatics-python-sdk/main/sdk/" +SM_DOCS = "https://docs.speechmatics.com/" + + +def check_speechmatics(): + p = "speechmatics" + rows = read_rows(p) + batch = fetch(SM_SDK + "batch/speechmatics/batch/_async_client.py") + fetch(SM_SDK + "batch/speechmatics/batch/_common.py") + tts = fetch(SM_SDK + "tts/speechmatics/tts/_async_client.py") + auth = fetch(SM_DOCS + "get-started/authentication.md") + for (rid, method, path, h, body, fields, required, query, response, source) in rows: + count(p, "rows checked") + if "docs:api-ref/" in source: + page = re.search(r"docs:(api-ref/[a-z\-/]+)", source).group(1) + md = fetch(SM_DOCS + page + ".md") + m = re.search(r"```\n([A-Z]+)\s*\n(/[^\n`]*)\n```", md) + if not m: + fail(p, "%s: docs page %s shows no method and path" % (rid, page)) + continue + if m.group(1) != method or norm_path(m.group(2)) != norm_path(path): + fail(p, "%s: docs page %s shows %s %s, table says %s %s" % (rid, page, m.group(1), m.group(2), method, path)) + elif "sdk/batch" in source: + key = re.sub(r"\{[^}]*\}", "", path) + if ('"%s' % key.rstrip("/")) not in batch and ('f"%s' % key.rstrip("/")) not in batch: + fail(p, "%s: path %s not in the batch SDK" % (rid, path)) + elif "sdk/tts" in source: + if "/generate/" not in tts or "output_format" not in tts: + fail(p, "%s: generate path or output_format not in the tts SDK" % rid) + elif "authentication" in source: + if "mp.speechmatics.com/v1/api_keys?type=" not in auth or "ttl" not in auth: + fail(p, "%s: temporary key endpoint not on the authentication page" % rid) + else: + fail(p, "%s: unrecognised source %s" % (rid, source)) + rt = fetch(SM_SDK + "rt/speechmatics/rt/_models.py") + vocab = cpp_quoted_after(p, "realtime.cpp", "Vocabulary RealtimeSession::vocabulary()", "RealtimeSession::RealtimeSession") + for w in [w for w in vocab if w != "message"]: + if '"%s"' % w not in rt: + fail(p, "realtime message %s not in rt/_models.py" % w) + count(p, "ws names checked") + base = fetch(SM_SDK + "rt/speechmatics/rt/_base_client.py") + for w in ("StartRecognition", "audio_format", "transcription_config"): + if w not in rt + base: + fail(p, "realtime name %s not in the SDK" % w) + count(p, "ws names checked") + tcfg = rt[rt.index("class TranscriptionConfig"):] + for w in ("language", "enable_partials", "max_delay", "diarization", "operating_point", "additional_vocab"): + if not re.search(r"^ %s:" % w, tcfg, re.M): + fail(p, "transcription_config field %s not in the SDK" % w) + count(p, "ws names checked") + voices = fetch(SM_SDK + "tts/speechmatics/tts/_models.py") + for v in ("sarah", "theo", "megan", "jack", "wav_16000", "pcm_16000"): + if '"%s"' % v not in voices: + fail(p, "tts enum value %s not in the SDK" % v) + count(p, "enum values checked") + + +HU = "https://dev.hume.ai/openapi/" +HU_SDK = GH + "HumeAI/hume-python-sdk/main/src/hume/" + + +def check_hume(): + p = "hume" + rows = read_rows(p) + for fname, prefix in (("text-to-speech-tts.json", "openapi-tts:"), ("speech-to-speech-evi.json", "openapi-evi:"), + ("expression-measurement-api.json", "openapi-expression:")): + api = OpenApi(json.loads(fetch(HU + fname))) + seen = check_json_rows(p, rows, api, prefix) + for path, item in api.d["paths"].items(): + for m in ("get", "post", "put", "patch", "delete"): + if m in item: + if (m.upper(), path) not in seen: + fail(p, "spec operation %s %s (%s) is not in the table" % (m.upper(), path, fname)) + count(p, "spec operations covered") + raw = fetch(HU_SDK + "tts/raw_client.py") + for (rid, method, path, h, body, fields, required, query, response, source) in rows: + if not source.startswith("sdk:"): + continue + fn = source.split()[-1] + m = re.search(r"def %s\((.*?)\) ->" % re.escape(fn), raw, re.S) + if not m: + fail(p, "%s: SDK function %s not found" % (rid, fn)) + continue + sig = m.group(1) + i = raw.index("def %s(" % fn) + j = raw.find("def ", i + 10) + chunk = raw[i:j if j > 0 else len(raw)] + if '"%s"' % path.lstrip("/") not in chunk: + fail(p, "%s: path %s not used by %s" % (rid, path, fn)) + for f in csv(fields): + if not re.search(r"\b%s:" % re.escape(f), sig): + fail(p, "%s: field %s not a parameter of %s" % (rid, f, fn)) + count(p, "rows checked") + pub = fetch(HU_SDK + "tts/types/publish_tts.py") + for f in ("text", "voice", "description", "speed", "trailing_silence", "flush", "close"): + if not re.search(r"^ %s:" % f, pub, re.M): + fail(p, "publish_tts field %s not in the SDK" % f) + count(p, "ws names checked") + si = fetch(HU_SDK + "tts/stream_input/raw_client.py") + for q in ("format_type", "context_generation_id", "include_timestamp_types", "instant_mode", "no_binary", "strip_headers", "version"): + if '"%s"' % q not in si: + fail(p, "stream/input query %s not in the SDK" % q) + count(p, "ws names checked") + chat = fetch(HU_SDK + "empathic_voice/chat/raw_client.py") + for q in ("config_id", "config_version", "event_limit", "resumed_chat_group_id", "verbose_transcription"): + if '"%s"' % q not in chat: + fail(p, "evi chat query %s not in the SDK" % q) + count(p, "ws names checked") + for fname, lit in (("audio_input", "audio_input"), ("session_settings", "session_settings"), ("user_input", "user_input"), + ("assistant_input", "assistant_input"), ("audio_output", "audio_output"), ("chat_metadata", "chat_metadata"), + ("assistant_message", "assistant_message"), ("user_message", "user_message"), ("assistant_end", "assistant_end"), + ("web_socket_error", "error"), ("user_interruption", "user_interruption"), ("tool_call_message", "tool_call"), + ("assistant_prosody", "assistant_prosody"), ("pause_assistant_message", "pause_assistant_message"), + ("resume_assistant_message", "resume_assistant_message")): + txt = fetch(HU_SDK + "empathic_voice/types/%s.py" % fname) + if 'typing.Literal["%s"]' % lit not in txt: + fail(p, "evi message type %s not in types/%s.py" % (lit, fname)) + count(p, "ws names checked") + ch = fetch(HU_SDK + "tts/types/snippet_audio_chunk.py") + for w in ("audio", "is_last_chunk", "generation_id"): + if w not in ch: + fail(p, "snippet audio chunk field %s not in the SDK" % w) + count(p, "ws names checked") + + +CHECKS = {"elevenlabs": check_elevenlabs, "assemblyai": check_assemblyai, "deepgram": check_deepgram, + "cartesia": check_cartesia, "speechmatics": check_speechmatics, "hume": check_hume} + + +def main(): + wanted = sys.argv[1:] or list(CHECKS) + for name in wanted: + if name not in CHECKS: + raise SystemExit("unknown provider %s (have: %s)" % (name, ", ".join(CHECKS))) + n_before = len(BAD) + CHECKS[name]() + print("%-13s %s %s" % (name, "ok " if len(BAD) == n_before else "FAIL", ", ".join("%s %d" % (k, v) for k, v in sorted(STATS.get(name, {}).items())))) + if BAD: + print("\n%d problem(s):" % len(BAD)) + for b in BAD: + print(" - " + b) + sys.exit(1) + print("check_spec: all clean (%s)" % time.strftime("%Y-%m-%d")) + + +if __name__ == "__main__": + main() diff --git a/tests/example/ai/audio/confirm_test.cpp b/tests/example/ai/audio/confirm_test.cpp new file mode 100644 index 0000000..266733d --- /dev/null +++ b/tests/example/ai/audio/confirm_test.cpp @@ -0,0 +1,34 @@ +// Dry runs: anything that generates audio, writes or deletes does not run +// without "confirm":true, and answers 200 with a preview even without a key. +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + for (const char* k : {"ELEVENLABS_API_KEY", "ASSEMBLYAI_API_KEY", "DEEPGRAM_API_KEY", "CARTESIA_API_KEY", + "SPEECHMATICS_API_KEY", "SPEECHMATICS_MANAGEMENT_TOKEN", "HUME_API_KEY"}) + unsetenv(k); + Fixture f; + auto post = [&](const std::string& path, const std::string& body) { return f.client->post(path, body, "application/json"); }; + auto tts = post("/ai/audio/tts", "{\"provider\":\"elevenlabs\",\"text\":\"Hello. This is a test.\",\"voice\":\"v1\"}"); + check(tts.status == 200 && contains(tts.body, "\"dry_run\":true"), "tts without confirm is a dry run"); + check(contains(tts.body, "\"chunks\":1") && contains(tts.body, "confirm"), "dry run reports chunks and how to run it"); + auto stt = post("/ai/audio/stt", "{\"provider\":\"assemblyai\",\"url\":\"https://example.com/a.mp3\"}"); + check(stt.status == 200 && contains(stt.body, "\"dry_run\":true"), "stt without confirm is a dry run"); + auto session = post("/ai/audio/session", "{\"provider\":\"hume\",\"text\":\"hi\"}"); + check(session.status == 200 && contains(session.body, "\"dry_run\":true"), "streaming session without confirm is a dry run"); + auto write = post("/ai/audio/call", "{\"provider\":\"elevenlabs\",\"id\":\"text_to_speech_full\",\"segments\":[\"v1\"],\"body\":{\"text\":\"x\"}}"); + check(write.status == 200 && contains(write.body, "\"dry_run\":true") && contains(write.body, "POST /v1/text-to-speech/v1"), "generation call is a dry run with the path"); + auto del = post("/ai/audio/call", "{\"provider\":\"elevenlabs\",\"id\":\"delete_voice\",\"segments\":[\"v1\"]}"); + check(del.status == 200 && contains(del.body, "\"dry_run\":true") && contains(del.body, "DELETE /v1/voices/v1"), "delete is a dry run"); + auto job = post("/ai/audio/call", "{\"provider\":\"speechmatics\",\"id\":\"jobs_delete\",\"segments\":[\"job1\"],\"query\":{\"force\":true}}"); + check(job.status == 200 && contains(job.body, "force=true"), "speechmatics delete previews its query"); + check(post("/ai/audio/call", "{\"provider\":\"elevenlabs\",\"id\":\"delete_voice\",\"segments\":[\"v1\"],\"confirm\":true}").status == 503, "confirmed delete needs the key"); + check(post("/ai/audio/tts", "{\"provider\":\"elevenlabs\",\"text\":\"x\",\"voice\":\"v1\",\"confirm\":true}").status == 503, "confirmed tts needs the key"); + check(post("/ai/audio/call", "{\"provider\":\"elevenlabs\",\"id\":\"delete_voice\",\"segments\":[\"a/b\"]}").status == 422, "bad id is 422 before the dry run"); + check(post("/ai/audio/call", "{\"provider\":\"elevenlabs\",\"id\":\"text_to_speech_full\",\"segments\":[\"v1\"],\"body\":{\"nope\":1}}").status == 422, "bad body field is 422 before the dry run"); + check(post("/ai/audio/call", "{\"provider\":\"elevenlabs\",\"id\":\"text_to_speech_full\",\"segments\":[\"v1\"]}").status == 422, "missing required body is 422"); + return finish("ai/audio/confirm"); +} diff --git a/tests/example/ai/audio/local_test.cpp b/tests/example/ai/audio/local_test.cpp new file mode 100644 index 0000000..d86a595 --- /dev/null +++ b/tests/example/ai/audio/local_test.cpp @@ -0,0 +1,34 @@ +// Local routes: POST /ai/audio/split and /ai/audio/ws/preview. No key, no network. +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto post = [&](const std::string& path, const std::string& body) { return f.client->post(path, body, "application/json"); }; + auto split = post("/ai/audio/split", "{\"text\":\"One sentence. Another sentence. A third one.\",\"max_chars\":20}"); + check(split.status == 200 && contains(split.body, "\"count\":3"), "splits at sentence ends"); + check(contains(split.body, "One sentence."), "chunks are returned"); + check(post("/ai/audio/split", "{\"text\":\"\"}").status == 422, "empty text is 422"); + check(post("/ai/audio/split", "{\"text\":\"x\",\"max_chars\":0}").status == 422, "max_chars 0 is 422"); + check(post("/ai/audio/split", "{\"text\":\"x\",\"max_chars\":100001}").status == 422, "max_chars over the cap is 422"); + std::string many; + for (int i = 0; i < 100; ++i) many += "word. "; + check(post("/ai/audio/split", "{\"text\":\"" + many + "\",\"max_chars\":6}").status == 422, "more than 64 chunks is 422"); + auto el = post("/ai/audio/ws/preview", "{\"provider\":\"elevenlabs\",\"channel\":\"tts-stream-input\",\"params\":{\"voice_id\":\"v1\",\"model_id\":\"eleven_flash_v2_5\"}}"); + check(el.status == 200 && contains(el.body, "wss://api.elevenlabs.io/v1/text-to-speech/v1/stream-input?model_id=eleven_flash_v2_5"), "elevenlabs url"); + check(contains(el.body, "never in the URL") && !contains(el.body, "preview-only"), "no credential in the preview"); + auto dg = post("/ai/audio/ws/preview", "{\"provider\":\"deepgram\",\"channel\":\"listen-v1\",\"params\":{\"model\":\"nova-3\"}}"); + check(dg.status == 200 && contains(dg.body, "wss://api.deepgram.com/v1/listen?model=nova-3"), "deepgram url"); + check(contains(post("/ai/audio/ws/preview", "{\"provider\":\"speechmatics\",\"channel\":\"realtime\"}").body, "rt.speechmatics.com/v2"), "speechmatics url"); + check(contains(post("/ai/audio/ws/preview", "{\"provider\":\"cartesia\",\"channel\":\"tts-websocket\"}").body, "cartesia_version=2026-08-14"), "cartesia pins the version"); + check(contains(post("/ai/audio/ws/preview", "{\"provider\":\"hume\",\"channel\":\"evi-chat\",\"params\":{\"config_id\":\"cfg1\"}}").body, "wss://api.hume.ai/v0/evi/chat?config_id=cfg1"), "hume evi url"); + check(contains(post("/ai/audio/ws/preview", "{\"provider\":\"assemblyai\",\"channel\":\"streaming\",\"params\":{\"sample_rate\":\"16000\"}}").body, "streaming.assemblyai.com/v3/ws?sample_rate=16000"), "assemblyai url"); + check(post("/ai/audio/ws/preview", "{\"provider\":\"elevenlabs\",\"channel\":\"nope\"}").status == 422, "unknown channel is 422"); + check(post("/ai/audio/ws/preview", "{\"provider\":\"elevenlabs\",\"channel\":\"tts-stream-input\",\"params\":{\"voice_id\":\"../x\"}}").status == 422, "bad voice id is 422"); + check(post("/ai/audio/ws/preview", "{\"provider\":\"deepgram\",\"channel\":\"listen-v1\",\"params\":{\"nonsense\":\"1\"}}").status == 422, "unknown stream parameter is 422"); + check(f.client->get("/ai/audio/split").status == 405 && f.client->get("/ai/audio/ws/preview").status == 405, "get is 405"); + return finish("ai/audio/local"); +} diff --git a/tests/example/ai/audio/page_test.cpp b/tests/example/ai/audio/page_test.cpp new file mode 100644 index 0000000..0eba20f --- /dev/null +++ b/tests/example/ai/audio/page_test.cpp @@ -0,0 +1,21 @@ +// Route: GET /ai/audio (page, no credential needed) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto res = f.client->get("/ai/audio"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "text/html"), "html"); + check(contains(res.body, "Audio") && contains(res.body, "viewport"), "page and mobile viewport"); + check(contains(res.body, "/embedded/ai_audio.js"), "script wired"); + check(contains(res.body, "