feat(clients): google cloud voice: generic client, TTS, STT, Gemini audio, live voice page
src/google: GoogleClient over molo::clients (key in x-goog-api-key, never in
URLs), Service base, TextToSpeech, SpeechToText, Generative (replies, audio
in, audio out), Recognizer/Synthesizer seams with cloud, gemini and vertex
backends, VoicePipeline with bounded history. Example: /voice live
conversation page, /voice/{say,hear,turn,reset}, /google/status. Keys load
from env, .env.json.local, .env.json. Paid routes are excluded from locust.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -67,6 +67,10 @@ std::shared_ptr<fw::Application> build_application(Services& services) {
|
||||
register_resilience_demo(resilience, services);
|
||||
app->include_router("", resilience);
|
||||
|
||||
fw::Router voice;
|
||||
register_voice_demo(voice, services);
|
||||
app->include_router("", voice);
|
||||
|
||||
register_chain_demo(app, services);
|
||||
return app;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
// Gap: real molo::di::Container and plugin registry in 05-request-routing.md.
|
||||
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
@@ -18,6 +19,9 @@
|
||||
#include "../fw/Observ.hpp"
|
||||
#include "../fw/Stats.hpp"
|
||||
#include "../fw/Templates.hpp"
|
||||
#include "google/client.h"
|
||||
#include "google/voice.h"
|
||||
#include "google/voice_backend.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
@@ -49,6 +53,12 @@ struct Services {
|
||||
std::shared_ptr<fw::OpenApi> openapi = std::make_shared<fw::OpenApi>();
|
||||
std::shared_ptr<fw::Environment> templates =
|
||||
std::make_shared<fw::Environment>();
|
||||
// Google Cloud client (null: built from GOOGLE_API_KEY on first use) and
|
||||
// the voice pipeline wired on it. Tests inject a scripted client.
|
||||
std::shared_ptr<google::GoogleClient> google;
|
||||
std::shared_ptr<google::VoicePipeline> voice;
|
||||
// Unset: chosen from GOOGLE_API_BACKEND / the key prefix (see src/google).
|
||||
std::optional<google::VoiceBackend> voice_backend;
|
||||
// Feature matrix rows (feature, route) shown on the index page. Each
|
||||
// register_*_demo adds its own rows with add_feature().
|
||||
std::vector<std::pair<std::string, std::string>> features;
|
||||
@@ -76,6 +86,7 @@ void register_security_extra_demo(fw::Router& router, Services& services);
|
||||
void register_platform_demo(fw::Router& router, Services& services);
|
||||
void register_enterprise_demo(fw::Router& router, Services& services);
|
||||
void register_resilience_demo(fw::Router& router, Services& services);
|
||||
void register_voice_demo(fw::Router& router, Services& services);
|
||||
|
||||
void load_templates(Services& services);
|
||||
void load_embedded(Services& services);
|
||||
|
||||
@@ -22,6 +22,7 @@ const char kBase[] = R"BASE(<!DOCTYPE html>
|
||||
<a href="/templates/page">Templates</a>
|
||||
<a href="/forms/contact">Forms</a>
|
||||
<a href="/stats">Stats</a>
|
||||
<a href="/voice">Voice</a>
|
||||
<a href="/docs">Docs</a>
|
||||
<a href="/health">Health</a>
|
||||
</nav>
|
||||
@@ -32,6 +33,7 @@ const char kBase[] = R"BASE(<!DOCTYPE html>
|
||||
<a href="/templates/page">Templates</a>
|
||||
<a href="/forms/contact">Forms</a>
|
||||
<a href="/stats">Stats</a>
|
||||
<a href="/voice">Voice</a>
|
||||
<a href="/docs">Docs</a>
|
||||
<a href="/health">Health</a>
|
||||
</nav>
|
||||
@@ -168,6 +170,37 @@ after its connection cap, so auto refresh is off by default.</p>
|
||||
{% endblock %}
|
||||
)STATS";
|
||||
|
||||
const char kVoice[] = R"VOICE({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Voice conversation</h1>
|
||||
<p class="hint">Talk, pause, and the assistant answers out loud. Google
|
||||
speech-to-text, Gemini, and Google text-to-speech run on the server.
|
||||
Speak over the answer to interrupt it.</p>
|
||||
<p id="v-config" class="hint">Checking Google credentials...</p>
|
||||
<section class="voice">
|
||||
<div class="voice-controls">
|
||||
<button id="v-toggle" type="button" class="primary" disabled>Start conversation</button>
|
||||
<label class="inline">Language
|
||||
<select id="v-lang">
|
||||
<option value="nl-NL">Nederlands</option>
|
||||
<option value="en-US">English</option>
|
||||
<option value="de-DE">Deutsch</option>
|
||||
</select></label>
|
||||
</div>
|
||||
<div class="voice-state"><span id="v-state" class="pill">idle</span>
|
||||
<div class="meter"><div id="v-level"></div></div></div>
|
||||
<p id="v-error" class="error" role="alert"></p>
|
||||
<ol id="v-log" class="voice-log"></ol>
|
||||
<form id="v-form" class="voice-form">
|
||||
<input id="v-text" type="text" maxlength="500" placeholder="Or type a message" autocomplete="off">
|
||||
<button type="submit">Send</button>
|
||||
<button id="v-reset" type="button">Reset</button>
|
||||
</form>
|
||||
</section>
|
||||
<script src="/embedded/voice.js"></script>
|
||||
{% endblock %}
|
||||
)VOICE";
|
||||
|
||||
const char kDocs[] = R"DOCS({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>API docs</h1>
|
||||
@@ -198,6 +231,7 @@ void load_templates(Services& services) {
|
||||
services.templates->add_template("page.html", kPage);
|
||||
services.templates->add_template("form_page.html", kFormPage);
|
||||
services.templates->add_template("docs.html", kDocs);
|
||||
services.templates->add_template("voice.html", kVoice);
|
||||
services.templates->add_template("stats.html", kStats);
|
||||
services.templates->add_template("error.html", kError);
|
||||
services.templates->add_template("nav_hint.html", kNavHint);
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
#include "App.hpp"
|
||||
|
||||
#include <cctype>
|
||||
#include <cstdlib>
|
||||
|
||||
#include "../fw/Router.hpp"
|
||||
#include "../fw/Templates.hpp"
|
||||
#include "../fw/Validate.hpp"
|
||||
#include "google/credentials.h"
|
||||
#include "google/endpoints.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// Google Cloud voice over the real src/google part: text-to-speech,
|
||||
// speech-to-text, a Gemini reply, and the live conversation page.
|
||||
// Gap: turns are request/response over REST; streaming recognition and the
|
||||
// Gemini Live API are listed in src/google/SPEC.md.
|
||||
|
||||
namespace {
|
||||
|
||||
std::string env_or(const char* name, const char* fallback) {
|
||||
std::string v = google::Credentials::env(name);
|
||||
return v.empty() ? std::string(fallback) : v;
|
||||
}
|
||||
|
||||
// "nl-NL", "en-US": two or three lowercase letters, dash, two uppercase.
|
||||
bool valid_language(const std::string& s) {
|
||||
std::size_t dash = s.find('-');
|
||||
if (dash < 2 || dash > 3 || s.size() != dash + 3) return false;
|
||||
for (std::size_t i = 0; i < dash; ++i)
|
||||
if (!std::islower((unsigned char)s[i])) return false;
|
||||
return std::isupper((unsigned char)s[dash + 1]) &&
|
||||
std::isupper((unsigned char)s[dash + 2]);
|
||||
}
|
||||
|
||||
std::string language_of(const std::string& value) {
|
||||
std::string lang = value.empty() ? env_or("GOOGLE_VOICE_LANG", "nl-NL") : value;
|
||||
if (!valid_language(lang))
|
||||
throw fw::HttpError(422, "validation_error", "lang must look like nl-NL");
|
||||
return lang;
|
||||
}
|
||||
|
||||
bool valid_voice(const std::string& s) {
|
||||
if (s.size() > 64) return false;
|
||||
for (char c : s)
|
||||
if (!std::isalnum((unsigned char)c) && c != '-' && c != '_') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string voice_of(const std::string& value) {
|
||||
std::string voice = value.empty() ? google::Credentials::env("GOOGLE_VOICE_NAME") : value;
|
||||
if (!valid_voice(voice))
|
||||
throw fw::HttpError(422, "validation_error", "voice has invalid characters");
|
||||
return voice;
|
||||
}
|
||||
|
||||
// Maps Google part errors onto the example's JSON error format.
|
||||
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||||
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "google_not_configured", e.message());
|
||||
// Local input validation carries no service name; upstream errors do.
|
||||
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
throw fw::HttpError(502, "google_error",
|
||||
e.status() + ": " + e.message());
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
auto guarded(F&& f) -> decltype(f()) {
|
||||
try {
|
||||
return f();
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
std::string quote(const std::string& s) {
|
||||
return "\"" + fw::json_escape(s) + "\"";
|
||||
}
|
||||
|
||||
fw::Response no_store(fw::Response res) {
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_voice_demo(fw::Router& router, Services& services) {
|
||||
if (!services.google) services.google = google::GoogleClient::from_env();
|
||||
auto gclient = services.google;
|
||||
google::VoiceBackend backend =
|
||||
services.voice_backend.value_or(google::voice_backend_from_env());
|
||||
google::VoiceParts parts = google::make_voice_parts(gclient, backend);
|
||||
auto tts = parts.synthesizer;
|
||||
auto stt = parts.recognizer;
|
||||
services.voice = std::make_shared<google::VoicePipeline>(
|
||||
stt, parts.responder, tts, std::make_shared<google::VoiceSessions>());
|
||||
auto pipeline = services.voice;
|
||||
fw::Environment* env = services.templates.get();
|
||||
|
||||
services.add_feature("google", "/google/status");
|
||||
services.add_feature("voice", "/voice");
|
||||
|
||||
router.get("/google/status", "google-status", [gclient, backend](fw::Request req) {
|
||||
(void)req;
|
||||
std::string out = std::string("{\"configured\":") +
|
||||
(gclient->configured() ? "true" : "false") +
|
||||
",\"credential\":\"x-goog-api-key\",\"backend\":" +
|
||||
quote(google::voice_backend_name(backend)) + ",\"model\":" +
|
||||
quote(google::Generative::default_model()) +
|
||||
",\"language\":" + quote(env_or("GOOGLE_VOICE_LANG", "nl-NL")) +
|
||||
",\"services\":[";
|
||||
bool first = true;
|
||||
for (const auto& e : google::endpoints::catalog()) {
|
||||
out += std::string(first ? "" : ",") + "{\"name\":" + quote(e.name) +
|
||||
",\"base_url\":" + quote(e.base_url) + "}";
|
||||
first = false;
|
||||
}
|
||||
return no_store(fw::JsonResponse(out + "]}", 200));
|
||||
});
|
||||
|
||||
router.get("/voice", "voice-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Voice conversation");
|
||||
return fw::Response(200, env->render("voice.html", ctx),
|
||||
"text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
// Text in, spoken audio (mp3) out.
|
||||
router.post("/voice/say", "voice-say", [tts](fw::Request req) {
|
||||
auto in = fw::parse_json_object_flat(req.body);
|
||||
google::SynthesisRequest speak;
|
||||
speak.text = in.count("text") ? in["text"] : "";
|
||||
speak.language = language_of(in.count("lang") ? in["lang"] : "");
|
||||
speak.voice = voice_of(in.count("voice") ? in["voice"] : "");
|
||||
return guarded([&] {
|
||||
google::Audio audio = tts->synthesize(speak);
|
||||
return no_store(fw::Response(200, audio.bytes, audio.mime));
|
||||
});
|
||||
});
|
||||
|
||||
// Audio bytes in (Content-Type names the encoding), transcript out.
|
||||
router.post("/voice/hear", "voice-hear", [stt](fw::Request req) {
|
||||
google::RecognitionRequest heard;
|
||||
heard.audio = req.body;
|
||||
if (!google::speech_encoding_from_mime(req.headers.get("Content-Type"),
|
||||
heard.encoding))
|
||||
throw fw::HttpError(415, "unsupported_media_type",
|
||||
"send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg");
|
||||
heard.language = language_of(req.query_or("lang", ""));
|
||||
return guarded([&] {
|
||||
google::Transcript t = stt->recognize(heard);
|
||||
return no_store(fw::JsonResponse(
|
||||
"{\"text\":" + quote(t.text) +
|
||||
",\"confidence\":" + std::to_string(t.confidence) + "}",
|
||||
200));
|
||||
});
|
||||
});
|
||||
|
||||
// One conversation turn. Audio body = spoken turn; JSON body {"text"} =
|
||||
// typed turn. Both answer {transcript, reply, audio_b64, mime, language};
|
||||
// silence answers an empty transcript and no audio.
|
||||
router.post("/voice/turn", "voice-turn", [pipeline](fw::Request req) {
|
||||
std::string cid = req.query_or("cid", "");
|
||||
if (!google::VoiceSessions::valid_id(cid))
|
||||
throw fw::HttpError(422, "validation_error",
|
||||
"cid must be 1-64 chars of A-Z a-z 0-9 - _");
|
||||
std::string lang = language_of(req.query_or("lang", ""));
|
||||
std::string voice = voice_of(req.query_or("voice", ""));
|
||||
std::string ctype = req.headers.get("Content-Type");
|
||||
return guarded([&] {
|
||||
google::VoiceTurn turn;
|
||||
if (ctype.find("json") != std::string::npos) {
|
||||
auto in = fw::parse_json_object_flat(req.body);
|
||||
turn = pipeline->turn_text(cid, in.count("text") ? in["text"] : "",
|
||||
lang, voice);
|
||||
} else {
|
||||
google::RecognitionRequest heard;
|
||||
heard.audio = req.body;
|
||||
heard.language = lang;
|
||||
if (!google::speech_encoding_from_mime(ctype, heard.encoding))
|
||||
throw fw::HttpError(415, "unsupported_media_type",
|
||||
"send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg");
|
||||
turn = pipeline->turn(cid, heard, voice);
|
||||
}
|
||||
return no_store(fw::JsonResponse(
|
||||
"{\"transcript\":" + quote(turn.transcript) +
|
||||
",\"reply\":" + quote(turn.reply) + ",\"language\":" +
|
||||
quote(turn.language) + ",\"mime\":" + quote(turn.audio.mime) +
|
||||
",\"audio_b64\":\"" +
|
||||
security::crypto::Base64::encode(turn.audio.bytes) + "\"}",
|
||||
200));
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/voice/reset", "voice-reset", [pipeline](fw::Request req) {
|
||||
std::string cid = req.query_or("cid", "");
|
||||
if (!google::VoiceSessions::valid_id(cid))
|
||||
throw fw::HttpError(422, "validation_error",
|
||||
"cid must be 1-64 chars of A-Z a-z 0-9 - _");
|
||||
pipeline->sessions().reset(cid);
|
||||
return fw::JsonResponse("{\"reset\":true}", 200);
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
Reference in New Issue
Block a user