2026-10-25 09:14:26 +01:00
|
|
|
#include "../AiDemos.hpp"
|
|
|
|
|
|
2026-10-25 15:36:57 +01:00
|
|
|
#include <algorithm>
|
|
|
|
|
|
2026-10-25 09:14:26 +01:00
|
|
|
#include "../../fw/Router.hpp"
|
2026-10-25 15:36:57 +01:00
|
|
|
#include "../../fw/Templates.hpp"
|
|
|
|
|
#include "llm/api_client.h"
|
|
|
|
|
#include "llm/config.h"
|
|
|
|
|
#include "security/crypto/Base64.h"
|
|
|
|
|
#include "vendors/audio/assemblyai/client.h"
|
|
|
|
|
#include "vendors/audio/assemblyai/transcripts.h"
|
|
|
|
|
#include "vendors/audio/cartesia/client.h"
|
|
|
|
|
#include "vendors/audio/deepgram/client.h"
|
|
|
|
|
#include "vendors/audio/elevenlabs/client.h"
|
|
|
|
|
#include "vendors/audio/hume/client.h"
|
|
|
|
|
#include "vendors/audio/audio_service.h"
|
|
|
|
|
#include "vendors/audio/cartesia/tts_socket.h"
|
|
|
|
|
#include "vendors/audio/deepgram/streams.h"
|
|
|
|
|
#include "vendors/audio/elevenlabs/tts_stream.h"
|
|
|
|
|
#include "vendors/audio/hume/sockets.h"
|
|
|
|
|
#include "vendors/audio/providers.h"
|
|
|
|
|
#include "vendors/audio/speech.h"
|
|
|
|
|
#include "vendors/audio/assemblyai/stream.h"
|
|
|
|
|
#include "vendors/audio/elevenlabs/scribe_stream.h"
|
|
|
|
|
#include "vendors/audio/speechmatics/batch.h"
|
|
|
|
|
#include "vendors/audio/speechmatics/realtime.h"
|
2026-10-25 09:14:26 +01:00
|
|
|
|
|
|
|
|
namespace molo::example::app {
|
|
|
|
|
|
2026-10-25 15:36:57 +01:00
|
|
|
// audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics and Hume
|
|
|
|
|
// through src/vendors/audio. GET routes need no credential. Every route that
|
|
|
|
|
// calls a provider is a POST: 503 not_configured without the provider's key,
|
|
|
|
|
// 422 on bad input. Anything that generates audio, writes, or deletes is a
|
|
|
|
|
// dry run unless the body says "confirm":true.
|
|
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
|
|
namespace au = audio;
|
|
|
|
|
using Json = clients::body::Json;
|
|
|
|
|
|
|
|
|
|
constexpr std::size_t kMaxBody = 4u * 1024u * 1024u;
|
|
|
|
|
constexpr std::size_t kMaxAudioEcho = 8u * 1024u * 1024u; // audio returned to the browser as base64
|
|
|
|
|
|
|
|
|
|
const char kPage[] = R"AUDIOP({% extends "base.html" %}
|
|
|
|
|
{% block content %}
|
|
|
|
|
<h1>Audio</h1>
|
|
|
|
|
<p class="hint">Speech synthesis and recognition across ElevenLabs, AssemblyAI, Deepgram, Cartesia,
|
|
|
|
|
Speechmatics and Hume. Anything that generates audio, writes or deletes is a dry run until the body says
|
|
|
|
|
<code>"confirm":true</code>.</p>
|
|
|
|
|
<p id="au-config" class="hint">Checking provider keys...</p>
|
|
|
|
|
<section class="au">
|
|
|
|
|
<label>Route
|
|
|
|
|
<select id="au-route">
|
|
|
|
|
<option value="tts">Text to speech (neutral seam)</option>
|
|
|
|
|
<option value="stt">Speech to text (neutral seam)</option>
|
|
|
|
|
<option value="voices">Voices of a provider</option>
|
|
|
|
|
<option value="call">Any REST call by table id</option>
|
|
|
|
|
<option value="session">Streaming text to speech over WebSocket (paid)</option>
|
|
|
|
|
<option value="ws/preview">WebSocket URL preview (local)</option>
|
|
|
|
|
<option value="split">Split long text (local)</option>
|
|
|
|
|
</select></label>
|
|
|
|
|
<label>Provider
|
|
|
|
|
<select id="au-provider">
|
|
|
|
|
<option>elevenlabs</option><option>deepgram</option><option>cartesia</option>
|
|
|
|
|
<option>speechmatics</option><option>hume</option><option>assemblyai</option>
|
|
|
|
|
</select></label>
|
|
|
|
|
<label>Body (JSON object) <textarea id="au-body" rows="9"></textarea></label>
|
|
|
|
|
<div class="au-actions"><button id="au-send" type="button" class="primary">Send</button></div>
|
|
|
|
|
<p id="au-error" class="error" role="alert"></p>
|
|
|
|
|
<audio id="au-audio" controls hidden></audio>
|
|
|
|
|
<pre id="au-out" class="au-out"></pre>
|
|
|
|
|
</section>
|
|
|
|
|
<style>
|
|
|
|
|
.au label{display:block;margin:.5rem 0}
|
|
|
|
|
.au select,.au textarea{width:100%;box-sizing:border-box}
|
|
|
|
|
.au-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
|
|
|
|
.au-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
|
|
|
|
|
.au audio{width:100%}
|
|
|
|
|
</style>
|
|
|
|
|
<script src="/embedded/ai_audio.js"></script>
|
|
|
|
|
{% endblock %}
|
|
|
|
|
)AUDIOP";
|
|
|
|
|
|
|
|
|
|
fw::Response json_out(const Json& j, int status = 200) {
|
|
|
|
|
fw::Response res = fw::JsonResponse(j.dump(), status);
|
|
|
|
|
res.headers.set("Cache-Control", "no-store");
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
|
|
|
|
|
|
|
|
|
|
template <typename F>
|
|
|
|
|
auto guarded(F&& f) -> decltype(f()) {
|
|
|
|
|
try {
|
|
|
|
|
return f();
|
|
|
|
|
} catch (const llm::NotConfiguredError& e) {
|
|
|
|
|
throw fw::HttpError(503, "not_configured", e.message());
|
|
|
|
|
} catch (const llm::ApiError& e) {
|
|
|
|
|
if (e.http_status() == 400 && e.type() == "INVALID_ARGUMENT") bad(e.message());
|
|
|
|
|
throw fw::HttpError(502, "audio_error",
|
|
|
|
|
"HTTP " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
|
|
|
|
|
} catch (const clients::body::JsonError&) {
|
|
|
|
|
bad("body is not valid JSON");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
class Body {
|
|
|
|
|
public:
|
|
|
|
|
explicit Body(const fw::Request& req) {
|
|
|
|
|
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
|
|
|
|
if (req.body.empty()) return;
|
|
|
|
|
try {
|
|
|
|
|
doc_ = Json::parse(req.body);
|
|
|
|
|
} catch (const clients::body::JsonError&) {
|
|
|
|
|
bad("body is not valid JSON");
|
|
|
|
|
}
|
|
|
|
|
if (!doc_.is_object()) bad("body must be a JSON object");
|
|
|
|
|
}
|
|
|
|
|
std::string text(const char* key, bool required = false) const {
|
|
|
|
|
const Json* v = doc_.find(key);
|
|
|
|
|
if (!v || v->is_null()) {
|
|
|
|
|
if (required) bad(std::string(key) + " is required");
|
|
|
|
|
return "";
|
|
|
|
|
}
|
|
|
|
|
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
|
|
|
|
if (required && v->as_string().empty()) bad(std::string(key) + " is required");
|
|
|
|
|
return v->as_string();
|
|
|
|
|
}
|
|
|
|
|
double real(const char* key, double fallback = 0) const {
|
|
|
|
|
const Json* v = doc_.find(key);
|
|
|
|
|
if (!v || v->is_null()) return fallback;
|
|
|
|
|
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
|
|
|
|
return v->as_number();
|
|
|
|
|
}
|
|
|
|
|
long number(const char* key, long fallback = 0) const { return static_cast<long>(real(key, static_cast<double>(fallback))); }
|
|
|
|
|
bool flag(const char* key) const {
|
|
|
|
|
const Json* v = doc_.find(key);
|
|
|
|
|
if (!v || v->is_null()) return false;
|
|
|
|
|
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
|
|
|
|
return v->as_bool();
|
|
|
|
|
}
|
|
|
|
|
Json object(const char* key) const {
|
|
|
|
|
const Json* v = doc_.find(key);
|
|
|
|
|
if (!v || v->is_null()) return Json();
|
|
|
|
|
if (!v->is_object()) bad(std::string(key) + " must be an object");
|
|
|
|
|
return *v;
|
|
|
|
|
}
|
|
|
|
|
std::vector<std::string> strings(const char* key) const {
|
|
|
|
|
std::vector<std::string> out;
|
|
|
|
|
const Json* v = doc_.find(key);
|
|
|
|
|
if (!v || v->is_null()) return out;
|
|
|
|
|
if (!v->is_array()) bad(std::string(key) + " must be an array");
|
|
|
|
|
for (const Json& s : v->as_array()) {
|
|
|
|
|
if (!s.is_string()) bad(std::string(key) + " must hold strings");
|
|
|
|
|
out.push_back(s.as_string());
|
|
|
|
|
}
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
std::string bytes(const char* key) const {
|
|
|
|
|
std::string b64 = text(key), raw;
|
|
|
|
|
if (b64.empty()) return raw;
|
|
|
|
|
if (!security::crypto::Base64::decode(b64, raw)) bad(std::string(key) + " is not valid base64");
|
|
|
|
|
return raw;
|
|
|
|
|
}
|
|
|
|
|
bool confirm() const { return flag("confirm"); }
|
|
|
|
|
const Json& doc() const { return doc_; }
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
Json doc_ = Json::object();
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
std::string b64(const std::string& raw) { return security::crypto::Base64::encode(raw); }
|
|
|
|
|
|
|
|
|
|
const au::ProviderInfo& provider_of(const Body& b) {
|
|
|
|
|
std::string name = b.text("provider", true);
|
|
|
|
|
const au::ProviderInfo* p = au::Providers::find(name);
|
|
|
|
|
if (!p) bad("provider must be one of elevenlabs, assemblyai, deepgram, cartesia, speechmatics, hume");
|
|
|
|
|
return *p;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Json dry_run(const std::string& provider, const std::string& what, const std::string& request) {
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("dry_run", Json(true));
|
|
|
|
|
o.set("provider", Json(provider));
|
|
|
|
|
o.set("what", Json(what));
|
|
|
|
|
o.set("request", Json(request));
|
|
|
|
|
o.set("hint", Json("send \"confirm\":true to run it"));
|
|
|
|
|
return o;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Json audio_json(const std::string& audio, const std::string& content_type) {
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("content_type", Json(content_type));
|
|
|
|
|
o.set("bytes", Json(static_cast<int>(audio.size())));
|
|
|
|
|
if (audio.size() <= kMaxAudioEcho) o.set("audio_base64", Json(b64(audio)));
|
|
|
|
|
else o.set("audio_omitted", Json("over the 8 MiB echo cap"));
|
|
|
|
|
return o;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The service for a provider's table, built from the environment.
|
|
|
|
|
std::unique_ptr<au::AudioService> service_for(const std::string& name) {
|
|
|
|
|
if (name == "elevenlabs") return std::make_unique<au::AudioService>(au::elevenlabs::calls(), au::elevenlabs::ElevenEndpoints::from_env());
|
|
|
|
|
if (name == "assemblyai") return std::make_unique<au::AudioService>(au::assemblyai::calls(), au::assemblyai::AssemblyEndpoints::from_env());
|
|
|
|
|
if (name == "deepgram") return std::make_unique<au::AudioService>(au::deepgram::calls(), au::deepgram::DeepgramEndpoints::from_env());
|
|
|
|
|
if (name == "cartesia") return std::make_unique<au::AudioService>(au::cartesia::calls(), au::cartesia::CartesiaEndpoints::from_env());
|
|
|
|
|
if (name == "speechmatics") return std::make_unique<au::AudioService>(au::speechmatics::calls(), au::speechmatics::SpeechmaticsEndpoints::from_env());
|
|
|
|
|
return std::make_unique<au::AudioService>(au::hume::calls(), au::hume::HumeEndpoints::from_env());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
template <typename F>
|
|
|
|
|
auto with_segments(const std::vector<std::string>& s, F&& f) -> decltype(f(std::initializer_list<std::string>{})) {
|
|
|
|
|
switch (s.size()) {
|
|
|
|
|
case 0: return f({});
|
|
|
|
|
case 1: return f({s[0]});
|
|
|
|
|
case 2: return f({s[0], s[1]});
|
|
|
|
|
case 3: return f({s[0], s[1], s[2]});
|
|
|
|
|
case 4: return f({s[0], s[1], s[2], s[3]});
|
|
|
|
|
default: bad("at most 4 path segments");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
class AudioRoutes {
|
|
|
|
|
public:
|
|
|
|
|
Json tts(const Body& b) const {
|
|
|
|
|
const au::ProviderInfo& p = provider_of(b);
|
|
|
|
|
if (!p.synthesizer) bad(std::string(p.title) + " has no speech synthesis product");
|
|
|
|
|
au::SpeechRequest r;
|
|
|
|
|
r.text = b.text("text", true);
|
|
|
|
|
r.voice = b.text("voice");
|
|
|
|
|
r.model = b.text("model");
|
|
|
|
|
r.language = b.text("language");
|
|
|
|
|
if (!b.text("format").empty()) r.format = b.text("format");
|
|
|
|
|
r.sample_rate = static_cast<int>(b.number("sample_rate"));
|
|
|
|
|
r.speed = b.real("speed");
|
|
|
|
|
return guarded([&]() -> Json {
|
|
|
|
|
au::validate(r);
|
|
|
|
|
auto synth = std::make_shared<au::ChunkedSynthesizer>(au::SpeechRegistry::synthesizer(p.name));
|
|
|
|
|
if (!b.confirm()) {
|
|
|
|
|
std::size_t chunks = au::split_text(r.text, std::min(synth->max_chars(), std::size_t{5000})).size();
|
|
|
|
|
Json o = dry_run(p.name, "synthesize", std::to_string(r.text.size()) + " characters, " + r.format);
|
|
|
|
|
o.set("chunks", Json(static_cast<int>(chunks)));
|
|
|
|
|
o.set("max_chars_per_call", Json(static_cast<int>(synth->max_chars())));
|
|
|
|
|
return o;
|
|
|
|
|
}
|
|
|
|
|
au::SpeechResult out = synth->synthesize(r);
|
|
|
|
|
Json o = audio_json(out.audio, out.content_type);
|
|
|
|
|
o.set("provider", Json(out.provider));
|
|
|
|
|
o.set("format", Json(out.format));
|
|
|
|
|
return o;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Json stt(const Body& b) const {
|
|
|
|
|
const au::ProviderInfo& p = provider_of(b);
|
|
|
|
|
if (!p.recognizer) bad(std::string(p.title) + " has no speech recognition product");
|
|
|
|
|
au::TranscribeRequest r;
|
|
|
|
|
r.url = b.text("url");
|
|
|
|
|
r.audio = b.bytes("data_base64");
|
|
|
|
|
if (!b.text("filename").empty()) r.filename = b.text("filename");
|
|
|
|
|
if (!b.text("content_type").empty()) r.content_type = b.text("content_type");
|
|
|
|
|
r.language = b.text("language");
|
|
|
|
|
r.model = b.text("model");
|
|
|
|
|
r.diarize = b.flag("diarize");
|
|
|
|
|
r.keywords = b.strings("keywords");
|
|
|
|
|
if (b.number("max_wait_seconds") > 0) r.max_wait_seconds = static_cast<int>(b.number("max_wait_seconds"));
|
|
|
|
|
return guarded([&]() -> Json {
|
|
|
|
|
au::validate(r);
|
|
|
|
|
if (!b.confirm()) {
|
|
|
|
|
return dry_run(p.name, "transcribe",
|
|
|
|
|
(r.url.empty() ? std::to_string(r.audio.size()) + " bytes of audio" : r.url) + (r.diarize ? ", diarized" : ""));
|
|
|
|
|
}
|
|
|
|
|
au::Transcript t = au::SpeechRegistry::recognizer(p.name)->transcribe(r);
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("provider", Json(t.provider));
|
|
|
|
|
o.set("text", Json(t.text));
|
|
|
|
|
o.set("language", Json(t.language));
|
|
|
|
|
o.set("duration", Json(t.duration));
|
|
|
|
|
Json words = Json::array();
|
|
|
|
|
for (const au::TranscriptWord& w : t.words) {
|
|
|
|
|
if (words.as_array().size() >= 2000) break;
|
|
|
|
|
Json j = Json::object();
|
|
|
|
|
j.set("text", Json(w.text));
|
|
|
|
|
j.set("start", Json(w.start));
|
|
|
|
|
j.set("end", Json(w.end));
|
|
|
|
|
if (!w.speaker.empty()) j.set("speaker", Json(w.speaker));
|
|
|
|
|
words.push(j);
|
|
|
|
|
}
|
|
|
|
|
o.set("words", words);
|
|
|
|
|
return o;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Json voices(const Body& b) const {
|
|
|
|
|
const au::ProviderInfo& p = provider_of(b);
|
|
|
|
|
if (!p.synthesizer) bad(std::string(p.title) + " has no voices");
|
|
|
|
|
return guarded([&]() -> Json { return au::SpeechRegistry::synthesizer(p.name)->voices(); });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Any row of a provider's call table by id. Reads run; everything else is a dry run unless confirmed.
|
|
|
|
|
Json call(const Body& b) const {
|
|
|
|
|
const au::ProviderInfo& p = provider_of(b);
|
|
|
|
|
std::string id = b.text("id", true);
|
|
|
|
|
std::vector<std::string> segments = b.strings("segments");
|
|
|
|
|
Json query = b.object("query");
|
|
|
|
|
Json body = b.object("body");
|
|
|
|
|
return guarded([&]() -> Json {
|
|
|
|
|
auto svc = service_for(p.name);
|
|
|
|
|
const au::CallSpec& spec = svc->table().get(id);
|
|
|
|
|
std::string kind = spec.body;
|
|
|
|
|
au::Query q = svc->query_from(id, query);
|
|
|
|
|
std::string method = spec.method;
|
|
|
|
|
std::string preview = with_segments(segments, [&](std::initializer_list<std::string> s) {
|
|
|
|
|
return svc->preview(id, s, kind == "json" && body.is_object() ? &body : nullptr, q);
|
|
|
|
|
});
|
|
|
|
|
bool write = method != "GET";
|
|
|
|
|
if (write && !b.confirm()) return dry_run(p.name, id, preview);
|
|
|
|
|
Json out = Json::object();
|
|
|
|
|
out.set("provider", Json(p.name));
|
|
|
|
|
out.set("id", Json(id));
|
|
|
|
|
if (kind == "bytes") {
|
|
|
|
|
std::string data = b.bytes("data_base64");
|
|
|
|
|
std::string ct = b.text("content_type");
|
|
|
|
|
llm::RawResponse raw = with_segments(segments, [&](std::initializer_list<std::string> s) {
|
|
|
|
|
return svc->call_bytes(id, s, data, ct, q);
|
|
|
|
|
});
|
|
|
|
|
out.set("response", Json(raw.body.substr(0, 65536)));
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
if (kind == "form") {
|
|
|
|
|
clients::body::MultipartBody form;
|
|
|
|
|
Json texts = b.object("form_text");
|
|
|
|
|
if (texts.is_object())
|
|
|
|
|
for (const auto& [k, v] : texts.as_object()) {
|
|
|
|
|
if (!v.is_string()) bad("form_text values must be strings");
|
|
|
|
|
form.add_text(k, v.as_string());
|
|
|
|
|
}
|
|
|
|
|
std::string data = b.bytes("data_base64");
|
|
|
|
|
if (!data.empty()) {
|
|
|
|
|
std::string field = b.text("file_field").empty() ? "file" : b.text("file_field");
|
|
|
|
|
std::string ct = b.text("content_type").empty() ? "application/octet-stream" : b.text("content_type");
|
|
|
|
|
form.add_file(field, b.text("filename").empty() ? "upload" : b.text("filename"), ct, data);
|
|
|
|
|
}
|
|
|
|
|
llm::RawResponse raw = with_segments(segments, [&](std::initializer_list<std::string> s) {
|
|
|
|
|
return svc->call_form_raw(id, s, form, q);
|
|
|
|
|
});
|
|
|
|
|
out.set("response", Json(raw.body.substr(0, 65536)));
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
if (au::is_destructive(spec)) {
|
|
|
|
|
out.set("response", with_segments(segments, [&](std::initializer_list<std::string> s) {
|
|
|
|
|
return svc->call_destructive(id, s, kind == "json" ? &body : nullptr, q, true);
|
|
|
|
|
}));
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
std::string resp = spec.response;
|
|
|
|
|
if (resp == "bytes" || resp == "text") {
|
|
|
|
|
llm::RawResponse raw = with_segments(segments, [&](std::initializer_list<std::string> s) {
|
|
|
|
|
return svc->call_raw(id, s, kind == "json" ? &body : nullptr, q);
|
|
|
|
|
});
|
|
|
|
|
Json a = resp == "text" ? Json::object() : audio_json(raw.body, raw.content_type);
|
|
|
|
|
if (resp == "text") a.set("text", Json(raw.body.substr(0, 65536)));
|
|
|
|
|
out.set("response", a);
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
out.set("response", with_segments(segments, [&](std::initializer_list<std::string> s) {
|
|
|
|
|
return svc->call(id, s, kind == "json" ? &body : nullptr, q);
|
|
|
|
|
}));
|
|
|
|
|
return out;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// One shot streaming synthesis over the provider's WebSocket.
|
|
|
|
|
Json session(const Body& b) const {
|
|
|
|
|
const au::ProviderInfo& p = provider_of(b);
|
|
|
|
|
std::string text = b.text("text", true);
|
|
|
|
|
std::string voice = b.text("voice");
|
|
|
|
|
std::string name = p.name;
|
|
|
|
|
if (name != "elevenlabs" && name != "deepgram" && name != "cartesia" && name != "hume")
|
|
|
|
|
bad("streaming synthesis is available for elevenlabs, deepgram, cartesia and hume");
|
|
|
|
|
if (name != "hume" && name != "deepgram" && voice.empty()) bad("voice is required");
|
|
|
|
|
return guarded([&]() -> Json {
|
|
|
|
|
std::string key = llm::Config::get(p.key_env);
|
|
|
|
|
if (!b.confirm()) return dry_run(name, "streaming synthesis", std::to_string(text.size()) + " characters over WebSocket");
|
|
|
|
|
if (key.empty()) throw llm::NotConfiguredError(name, p.key_env);
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("provider", Json(name));
|
|
|
|
|
if (name == "elevenlabs") {
|
|
|
|
|
au::elevenlabs::TtsStreamOptions opt;
|
|
|
|
|
opt.voice_id = voice;
|
|
|
|
|
opt.model_id = b.text("model");
|
|
|
|
|
opt.output_format = b.text("output_format");
|
|
|
|
|
au::elevenlabs::TtsInputStream s(key, opt);
|
|
|
|
|
s.connect();
|
|
|
|
|
au::elevenlabs::TtsStreamResult r = s.speak(text, 20000);
|
|
|
|
|
s.close();
|
|
|
|
|
if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error);
|
|
|
|
|
Json a = audio_json(r.audio, "audio/mpeg");
|
|
|
|
|
a.set("chunks", Json(static_cast<int>(r.chunks)));
|
|
|
|
|
a.set("final", Json(r.is_final));
|
|
|
|
|
return a;
|
|
|
|
|
}
|
|
|
|
|
if (name == "deepgram") {
|
|
|
|
|
au::deepgram::StreamParams params;
|
|
|
|
|
params.add("model", voice.empty() ? "aura-2-thalia-en" : voice).add("encoding", "linear16").add("sample_rate", "24000");
|
|
|
|
|
au::deepgram::SpeakStream s(key, params);
|
|
|
|
|
s.connect();
|
|
|
|
|
s.speak(text);
|
|
|
|
|
s.flush();
|
|
|
|
|
au::deepgram::SpeakStreamResult r = s.collect(20000);
|
|
|
|
|
s.end();
|
|
|
|
|
s.close();
|
|
|
|
|
if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error);
|
|
|
|
|
Json a = audio_json(r.audio, "audio/pcm");
|
|
|
|
|
a.set("chunks", Json(static_cast<int>(r.chunks)));
|
|
|
|
|
a.set("flushed", Json(r.flushed));
|
|
|
|
|
return a;
|
|
|
|
|
}
|
|
|
|
|
if (name == "cartesia") {
|
|
|
|
|
au::cartesia::TtsSocket s(key);
|
|
|
|
|
s.connect();
|
|
|
|
|
au::cartesia::TtsSocketRequest r;
|
|
|
|
|
r.context_id = "demo-1";
|
|
|
|
|
r.transcript = text;
|
|
|
|
|
r.voice_id = voice;
|
|
|
|
|
r.continue_ = 0;
|
|
|
|
|
s.generate(r);
|
|
|
|
|
au::cartesia::TtsSocketResult res = s.collect("demo-1", 20000);
|
|
|
|
|
s.close();
|
|
|
|
|
if (!res.error.empty()) throw llm::ApiError(name, 502, "stream_error", res.error);
|
|
|
|
|
Json a = audio_json(res.audio, "audio/pcm");
|
|
|
|
|
a.set("chunks", Json(static_cast<int>(res.chunks)));
|
|
|
|
|
a.set("done", Json(res.done));
|
|
|
|
|
return a;
|
|
|
|
|
}
|
|
|
|
|
au::hume::HumeStreamOptions opt;
|
|
|
|
|
opt.format_type = "mp3";
|
|
|
|
|
opt.no_binary = 1;
|
|
|
|
|
au::hume::HumeTtsStream s(key, opt);
|
|
|
|
|
s.connect();
|
|
|
|
|
s.send_text(text, voice);
|
|
|
|
|
s.finish();
|
|
|
|
|
au::hume::HumeStreamResult r = s.collect(20000);
|
|
|
|
|
s.close();
|
|
|
|
|
if (!r.error.empty()) throw llm::ApiError(name, 502, "stream_error", r.error);
|
|
|
|
|
Json a = audio_json(r.audio, "audio/mpeg");
|
|
|
|
|
a.set("chunks", Json(static_cast<int>(r.chunks)));
|
|
|
|
|
a.set("last_chunk", Json(r.last_chunk));
|
|
|
|
|
return a;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Local: the URL a WebSocket channel would open, with the credential left out.
|
|
|
|
|
Json ws_preview(const Body& b) const {
|
|
|
|
|
const au::ProviderInfo& p = provider_of(b);
|
|
|
|
|
std::string channel = b.text("channel", true);
|
|
|
|
|
Json params = b.object("params");
|
|
|
|
|
auto str = [&](const char* k) {
|
|
|
|
|
const Json* v = params.is_object() ? params.find(k) : nullptr;
|
|
|
|
|
return v && v->is_string() ? v->as_string() : std::string();
|
|
|
|
|
};
|
|
|
|
|
return guarded([&]() -> Json {
|
|
|
|
|
std::string url;
|
|
|
|
|
std::string key = "preview-only";
|
|
|
|
|
if (std::string(p.name) == "elevenlabs" && channel == "tts-stream-input") {
|
|
|
|
|
au::elevenlabs::TtsStreamOptions o;
|
|
|
|
|
o.voice_id = str("voice_id");
|
|
|
|
|
o.model_id = str("model_id");
|
|
|
|
|
o.output_format = str("output_format");
|
|
|
|
|
url = au::elevenlabs::TtsInputStream(key, o).url();
|
|
|
|
|
} else if (std::string(p.name) == "elevenlabs" && channel == "stt-realtime") {
|
|
|
|
|
au::elevenlabs::ScribeStreamOptions o;
|
|
|
|
|
o.model_id = str("model_id");
|
|
|
|
|
o.audio_format = str("audio_format");
|
|
|
|
|
o.commit_strategy = str("commit_strategy");
|
|
|
|
|
url = au::elevenlabs::ScribeStream(key, o).url();
|
|
|
|
|
} else if (std::string(p.name) == "deepgram" && (channel == "listen-v1" || channel == "listen-v2")) {
|
|
|
|
|
au::deepgram::StreamParams sp;
|
|
|
|
|
if (params.is_object())
|
|
|
|
|
for (const auto& [k, v] : params.as_object())
|
|
|
|
|
if (v.is_string()) sp.add(k, v.as_string());
|
|
|
|
|
url = au::deepgram::ListenStream(key, sp, channel == "listen-v2").url();
|
|
|
|
|
} else if (std::string(p.name) == "deepgram" && (channel == "speak-v1" || channel == "speak-v2")) {
|
|
|
|
|
au::deepgram::StreamParams sp;
|
|
|
|
|
if (params.is_object())
|
|
|
|
|
for (const auto& [k, v] : params.as_object())
|
|
|
|
|
if (v.is_string()) sp.add(k, v.as_string());
|
|
|
|
|
url = au::deepgram::SpeakStream(key, sp, channel == "speak-v2").url();
|
|
|
|
|
} else if (std::string(p.name) == "assemblyai" && channel == "streaming") {
|
|
|
|
|
au::assemblyai::StreamParams sp;
|
|
|
|
|
if (params.is_object())
|
|
|
|
|
for (const auto& [k, v] : params.as_object())
|
|
|
|
|
if (v.is_string()) sp.add(k, v.as_string());
|
|
|
|
|
url = au::assemblyai::TranscriptStream(key, sp).url();
|
|
|
|
|
} else if (std::string(p.name) == "cartesia" && channel == "tts-websocket") {
|
|
|
|
|
url = au::cartesia::TtsSocket(key).url();
|
|
|
|
|
} else if (std::string(p.name) == "speechmatics" && channel == "realtime") {
|
|
|
|
|
au::speechmatics::RealtimeConfig c;
|
|
|
|
|
if (!str("language").empty()) c.language = str("language");
|
|
|
|
|
url = au::speechmatics::RealtimeSession(key, c).url();
|
|
|
|
|
} else if (std::string(p.name) == "hume" && channel == "tts-stream-input") {
|
|
|
|
|
url = au::hume::HumeTtsStream(key, au::hume::HumeStreamOptions()).url();
|
|
|
|
|
} else if (std::string(p.name) == "hume" && channel == "evi-chat") {
|
|
|
|
|
au::hume::EviOptions o;
|
|
|
|
|
o.config_id = str("config_id");
|
|
|
|
|
url = au::hume::EviChat(key, o).url();
|
|
|
|
|
} else {
|
|
|
|
|
bad(std::string("no such channel for ") + p.name + ": " + channel);
|
|
|
|
|
}
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("provider", Json(p.name));
|
|
|
|
|
o.set("channel", Json(channel));
|
|
|
|
|
o.set("url", Json(url));
|
|
|
|
|
o.set("credential", Json("sent as a header, never in the URL"));
|
|
|
|
|
return o;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Local: sentence aware splitting with the package's caps.
|
|
|
|
|
Json split(const Body& b) const {
|
|
|
|
|
std::string text = b.text("text", true);
|
|
|
|
|
long max_chars = b.number("max_chars", 500);
|
|
|
|
|
if (max_chars < 1 || max_chars > 100000) bad("max_chars must be 1..100000");
|
|
|
|
|
return guarded([&]() -> Json {
|
|
|
|
|
Json arr = Json::array();
|
|
|
|
|
for (const std::string& c : au::split_text(text, static_cast<std::size_t>(max_chars))) arr.push(Json(c));
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("chunks", arr);
|
|
|
|
|
o.set("count", Json(static_cast<int>(arr.as_array().size())));
|
|
|
|
|
return o;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
|
|
2026-10-25 09:14:26 +01:00
|
|
|
void register_ai_audio_demo(fw::Router& router, Services& services) {
|
2026-10-25 15:36:57 +01:00
|
|
|
fw::Environment* env = services.templates.get();
|
|
|
|
|
services.templates->add_template("ai_audio.html", kPage);
|
|
|
|
|
services.add_feature("ai-audio", "/ai/audio");
|
|
|
|
|
auto routes = std::make_shared<AudioRoutes>();
|
|
|
|
|
|
|
|
|
|
router.get("/ai/audio", "ai-audio-page", [env](fw::Request req) {
|
|
|
|
|
(void)req;
|
|
|
|
|
fw::TemplateCtx ctx;
|
|
|
|
|
ctx["title"] = std::string("Audio");
|
|
|
|
|
return fw::Response(200, env->render("ai_audio.html", ctx), "text/html; charset=utf-8");
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
router.get("/ai/audio/status", "ai-audio-status", [](fw::Request req) {
|
|
|
|
|
(void)req;
|
|
|
|
|
Json providers = Json::array();
|
|
|
|
|
for (const au::ProviderInfo& p : au::Providers::all()) {
|
|
|
|
|
Json j = Json::object();
|
|
|
|
|
j.set("name", Json(p.name));
|
|
|
|
|
j.set("title", Json(p.title));
|
|
|
|
|
j.set("key_env", Json(p.key_env));
|
|
|
|
|
j.set("key_configured", Json(!llm::Config::get(p.key_env).empty()));
|
|
|
|
|
if (std::string(p.extra_env).size()) {
|
|
|
|
|
j.set("extra_env", Json(p.extra_env));
|
|
|
|
|
j.set("extra_configured", Json(!llm::Config::get(p.extra_env).empty()));
|
|
|
|
|
}
|
|
|
|
|
j.set("key_page", Json(p.key_page));
|
|
|
|
|
j.set("synthesizer", Json(p.synthesizer));
|
|
|
|
|
j.set("recognizer", Json(p.recognizer));
|
|
|
|
|
j.set("websockets", Json(p.websockets));
|
|
|
|
|
j.set("calls", Json(static_cast<int>(p.table().size())));
|
|
|
|
|
providers.push(j);
|
|
|
|
|
}
|
|
|
|
|
Json absent = Json::array();
|
|
|
|
|
for (const au::Providers::Absent& a : au::Providers::absent()) {
|
|
|
|
|
Json j = Json::object();
|
|
|
|
|
j.set("name", Json(a.name));
|
|
|
|
|
j.set("reason", Json(a.reason));
|
|
|
|
|
absent.push(j);
|
|
|
|
|
}
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("providers", providers);
|
|
|
|
|
o.set("absent", absent);
|
|
|
|
|
o.set("calls_total", Json(static_cast<int>(au::Providers::call_count())));
|
|
|
|
|
o.set("max_body_bytes", Json(static_cast<int>(kMaxBody)));
|
|
|
|
|
return json_out(o);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
router.get("/ai/audio/calls", "ai-audio-calls", [](fw::Request req) {
|
|
|
|
|
Json arr = Json::array();
|
|
|
|
|
std::string want;
|
|
|
|
|
if (auto it = req.query.find("provider"); it != req.query.end()) want = it->second;
|
|
|
|
|
for (const au::ProviderInfo& p : au::Providers::all()) {
|
|
|
|
|
if (!want.empty() && want != p.name) continue;
|
|
|
|
|
for (std::size_t i = 0; i < p.table().size(); ++i) {
|
|
|
|
|
const au::CallSpec& s = p.table().at(i);
|
|
|
|
|
Json row = Json::object();
|
|
|
|
|
row.set("provider", Json(p.name));
|
|
|
|
|
row.set("id", Json(s.id));
|
|
|
|
|
row.set("http", Json(s.method));
|
|
|
|
|
row.set("path", Json(s.path));
|
|
|
|
|
row.set("host", Json(s.host));
|
|
|
|
|
row.set("body", Json(s.body));
|
|
|
|
|
row.set("response", Json(s.response));
|
|
|
|
|
arr.push(row);
|
|
|
|
|
if (arr.as_array().size() >= 2000) break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Json o = Json::object();
|
|
|
|
|
o.set("calls", arr);
|
|
|
|
|
o.set("truncated", Json(want.empty() && au::Providers::call_count() > 2000));
|
|
|
|
|
return json_out(o);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
using Handler = Json (AudioRoutes::*)(const Body&) const;
|
|
|
|
|
struct Route { const char* path; const char* name; Handler handler; };
|
|
|
|
|
const Route table[] = {
|
|
|
|
|
{"/ai/audio/tts", "ai-audio-tts", &AudioRoutes::tts},
|
|
|
|
|
{"/ai/audio/stt", "ai-audio-stt", &AudioRoutes::stt},
|
|
|
|
|
{"/ai/audio/voices", "ai-audio-voices", &AudioRoutes::voices},
|
|
|
|
|
{"/ai/audio/call", "ai-audio-call", &AudioRoutes::call},
|
|
|
|
|
{"/ai/audio/session", "ai-audio-session", &AudioRoutes::session},
|
|
|
|
|
{"/ai/audio/ws/preview", "ai-audio-ws-preview", &AudioRoutes::ws_preview},
|
|
|
|
|
{"/ai/audio/split", "ai-audio-split", &AudioRoutes::split},
|
|
|
|
|
};
|
|
|
|
|
for (const Route& r : table) {
|
|
|
|
|
Handler h = r.handler;
|
|
|
|
|
router.post(r.path, r.name, [routes, h](fw::Request req) {
|
|
|
|
|
Body body(req);
|
|
|
|
|
return json_out(((*routes).*h)(body));
|
|
|
|
|
});
|
|
|
|
|
}
|
2026-10-25 09:14:26 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
} // namespace molo::example::app
|