feat(clients): google cloud voice: generic client, TTS, STT, Gemini audio, live voice page
src/google: GoogleClient over molo::clients (key in x-goog-api-key, never in
URLs), Service base, TextToSpeech, SpeechToText, Generative (replies, audio
in, audio out), Recognizer/Synthesizer seams with cloud, gemini and vertex
backends, VoicePipeline with bounded history. Example: /voice live
conversation page, /voice/{say,hear,turn,reset}, /google/status. Keys load
from env, .env.json.local, .env.json. Paid routes are excluded from locust.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
#include "App.hpp"
|
||||
|
||||
#include <cctype>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "../fw/Router.hpp"
|
||||
#include "../fw/Templates.hpp"
|
||||
#include "../fw/Validate.hpp"
|
||||
#include "google/credentials.h"
|
||||
#include "google/endpoints.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// Google Cloud voice over the real src/google part: text-to-speech,
|
||||
// speech-to-text, a Gemini reply, and the live conversation page.
|
||||
// Gap: turns are request/response over REST; streaming recognition and the
|
||||
// Gemini Live API are listed in src/google/SPEC.md.
|
||||
|
||||
namespace {
|
||||
|
||||
std::string env_or(const char* name, const char* fallback) {
|
||||
std::string v = google::Credentials::env(name);
|
||||
return v.empty() ? std::string(fallback) : v;
|
||||
}
|
||||
|
||||
// "nl-NL", "en-US": two or three lowercase letters, dash, two uppercase.
|
||||
bool valid_language(const std::string& s) {
|
||||
std::size_t dash = s.find('-');
|
||||
if (dash < 2 || dash > 3 || s.size() != dash + 3) return false;
|
||||
for (std::size_t i = 0; i < dash; ++i)
|
||||
if (!std::islower((unsigned char)s[i])) return false;
|
||||
return std::isupper((unsigned char)s[dash + 1]) &&
|
||||
std::isupper((unsigned char)s[dash + 2]);
|
||||
}
|
||||
|
||||
std::string language_of(const std::string& value) {
|
||||
std::string lang = value.empty() ? env_or("GOOGLE_VOICE_LANG", "nl-NL") : value;
|
||||
if (!valid_language(lang))
|
||||
throw fw::HttpError(422, "validation_error", "lang must look like nl-NL");
|
||||
return lang;
|
||||
}
|
||||
|
||||
bool valid_voice(const std::string& s) {
|
||||
if (s.size() > 64) return false;
|
||||
for (char c : s)
|
||||
if (!std::isalnum((unsigned char)c) && c != '-' && c != '_') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string voice_of(const std::string& value) {
|
||||
std::string voice = value.empty() ? google::Credentials::env("GOOGLE_VOICE_NAME") : value;
|
||||
if (!valid_voice(voice))
|
||||
throw fw::HttpError(422, "validation_error", "voice has invalid characters");
|
||||
return voice;
|
||||
}
|
||||
|
||||
// Maps Google part errors onto the example's JSON error format.
|
||||
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||||
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "google_not_configured", e.message());
|
||||
// Local input validation carries no service name; upstream errors do.
|
||||
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
throw fw::HttpError(502, "google_error",
|
||||
e.status() + ": " + e.message());
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
auto guarded(F&& f) -> decltype(f()) {
|
||||
try {
|
||||
return f();
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
std::string quote(const std::string& s) {
|
||||
return "\"" + fw::json_escape(s) + "\"";
|
||||
}
|
||||
|
||||
fw::Response no_store(fw::Response res) {
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_voice_demo(fw::Router& router, Services& services) {
|
||||
if (!services.google) services.google = google::GoogleClient::from_env();
|
||||
auto gclient = services.google;
|
||||
google::VoiceBackend backend =
|
||||
services.voice_backend.value_or(google::voice_backend_from_env());
|
||||
google::VoiceParts parts = google::make_voice_parts(gclient, backend);
|
||||
auto tts = parts.synthesizer;
|
||||
auto stt = parts.recognizer;
|
||||
services.voice = std::make_shared<google::VoicePipeline>(
|
||||
stt, parts.responder, tts, std::make_shared<google::VoiceSessions>());
|
||||
auto pipeline = services.voice;
|
||||
fw::Environment* env = services.templates.get();
|
||||
|
||||
services.add_feature("google", "/google/status");
|
||||
services.add_feature("voice", "/voice");
|
||||
|
||||
router.get("/google/status", "google-status", [gclient, backend](fw::Request req) {
|
||||
(void)req;
|
||||
std::string out = std::string("{\"configured\":") +
|
||||
(gclient->configured() ? "true" : "false") +
|
||||
",\"credential\":\"x-goog-api-key\",\"backend\":" +
|
||||
quote(google::voice_backend_name(backend)) + ",\"model\":" +
|
||||
quote(google::Generative::default_model()) +
|
||||
",\"language\":" + quote(env_or("GOOGLE_VOICE_LANG", "nl-NL")) +
|
||||
",\"services\":[";
|
||||
bool first = true;
|
||||
for (const auto& e : google::endpoints::catalog()) {
|
||||
out += std::string(first ? "" : ",") + "{\"name\":" + quote(e.name) +
|
||||
",\"base_url\":" + quote(e.base_url) + "}";
|
||||
first = false;
|
||||
}
|
||||
return no_store(fw::JsonResponse(out + "]}", 200));
|
||||
});
|
||||
|
||||
router.get("/voice", "voice-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Voice conversation");
|
||||
return fw::Response(200, env->render("voice.html", ctx),
|
||||
"text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
// Text in, spoken audio (mp3) out.
|
||||
router.post("/voice/say", "voice-say", [tts](fw::Request req) {
|
||||
auto in = fw::parse_json_object_flat(req.body);
|
||||
google::SynthesisRequest speak;
|
||||
speak.text = in.count("text") ? in["text"] : "";
|
||||
speak.language = language_of(in.count("lang") ? in["lang"] : "");
|
||||
speak.voice = voice_of(in.count("voice") ? in["voice"] : "");
|
||||
return guarded([&] {
|
||||
google::Audio audio = tts->synthesize(speak);
|
||||
return no_store(fw::Response(200, audio.bytes, audio.mime));
|
||||
});
|
||||
});
|
||||
|
||||
// Audio bytes in (Content-Type names the encoding), transcript out.
|
||||
router.post("/voice/hear", "voice-hear", [stt](fw::Request req) {
|
||||
google::RecognitionRequest heard;
|
||||
heard.audio = req.body;
|
||||
if (!google::speech_encoding_from_mime(req.headers.get("Content-Type"),
|
||||
heard.encoding))
|
||||
throw fw::HttpError(415, "unsupported_media_type",
|
||||
"send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg");
|
||||
heard.language = language_of(req.query_or("lang", ""));
|
||||
return guarded([&] {
|
||||
google::Transcript t = stt->recognize(heard);
|
||||
return no_store(fw::JsonResponse(
|
||||
"{\"text\":" + quote(t.text) +
|
||||
",\"confidence\":" + std::to_string(t.confidence) + "}",
|
||||
200));
|
||||
});
|
||||
});
|
||||
|
||||
// One conversation turn. Audio body = spoken turn; JSON body {"text"} =
|
||||
// typed turn. Both answer {transcript, reply, audio_b64, mime, language};
|
||||
// silence answers an empty transcript and no audio.
|
||||
router.post("/voice/turn", "voice-turn", [pipeline](fw::Request req) {
|
||||
std::string cid = req.query_or("cid", "");
|
||||
if (!google::VoiceSessions::valid_id(cid))
|
||||
throw fw::HttpError(422, "validation_error",
|
||||
"cid must be 1-64 chars of A-Z a-z 0-9 - _");
|
||||
std::string lang = language_of(req.query_or("lang", ""));
|
||||
std::string voice = voice_of(req.query_or("voice", ""));
|
||||
std::string ctype = req.headers.get("Content-Type");
|
||||
return guarded([&] {
|
||||
google::VoiceTurn turn;
|
||||
if (ctype.find("json") != std::string::npos) {
|
||||
auto in = fw::parse_json_object_flat(req.body);
|
||||
turn = pipeline->turn_text(cid, in.count("text") ? in["text"] : "",
|
||||
lang, voice);
|
||||
} else {
|
||||
google::RecognitionRequest heard;
|
||||
heard.audio = req.body;
|
||||
heard.language = lang;
|
||||
if (!google::speech_encoding_from_mime(ctype, heard.encoding))
|
||||
throw fw::HttpError(415, "unsupported_media_type",
|
||||
"send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg");
|
||||
turn = pipeline->turn(cid, heard, voice);
|
||||
}
|
||||
return no_store(fw::JsonResponse(
|
||||
"{\"transcript\":" + quote(turn.transcript) +
|
||||
",\"reply\":" + quote(turn.reply) + ",\"language\":" +
|
||||
quote(turn.language) + ",\"mime\":" + quote(turn.audio.mime) +
|
||||
",\"audio_b64\":\"" +
|
||||
security::crypto::Base64::encode(turn.audio.bytes) + "\"}",
|
||||
200));
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/voice/reset", "voice-reset", [pipeline](fw::Request req) {
|
||||
std::string cid = req.query_or("cid", "");
|
||||
if (!google::VoiceSessions::valid_id(cid))
|
||||
throw fw::HttpError(422, "validation_error",
|
||||
"cid must be 1-64 chars of A-Z a-z 0-9 - _");
|
||||
pipeline->sessions().reset(cid);
|
||||
return fw::JsonResponse("{\"reset\":true}", 200);
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
Reference in New Issue
Block a user