209 lines
8.0 KiB
C++
209 lines
8.0 KiB
C++
#include "App.hpp"
|
|||
|
|
|
||
|
|
#include <cctype>
|
||
|
|
#include <cstdlib>
|
||
|
|
|
||
|
|
#include "../fw/Router.hpp"
|
||
|
|
#include "../fw/Templates.hpp"
|
||
|
|
#include "../fw/Validate.hpp"
|
||
|
|
#include "google/credentials.h"
|
||
|
|
#include "google/endpoints.h"
|
||
|
|
#include "security/crypto/Base64.h"
|
||
|
|
|
||
|
|
namespace molo::example::app {
|
||
|
|
|
||
|
|
// Google Cloud voice over the real src/google part: text-to-speech,
|
||
|
|
// speech-to-text, a Gemini reply, and the live conversation page.
|
||
|
|
// Gap: turns are request/response over REST; streaming recognition and the
|
||
|
|
// Gemini Live API are listed in src/google/SPEC.md.
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
|
||
|
|
std::string env_or(const char* name, const char* fallback) {
|
||
|
|
std::string v = google::Credentials::env(name);
|
||
|
|
return v.empty() ? std::string(fallback) : v;
|
||
|
|
}
|
||
|
|
|
||
|
|
// "nl-NL", "en-US": two or three lowercase letters, dash, two uppercase.
|
||
|
|
bool valid_language(const std::string& s) {
|
||
|
|
std::size_t dash = s.find('-');
|
||
|
|
if (dash < 2 || dash > 3 || s.size() != dash + 3) return false;
|
||
|
|
for (std::size_t i = 0; i < dash; ++i)
|
||
|
|
if (!std::islower((unsigned char)s[i])) return false;
|
||
|
|
return std::isupper((unsigned char)s[dash + 1]) &&
|
||
|
|
std::isupper((unsigned char)s[dash + 2]);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string language_of(const std::string& value) {
|
||
|
|
std::string lang = value.empty() ? env_or("GOOGLE_VOICE_LANG", "nl-NL") : value;
|
||
|
|
if (!valid_language(lang))
|
||
|
|
throw fw::HttpError(422, "validation_error", "lang must look like nl-NL");
|
||
|
|
return lang;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool valid_voice(const std::string& s) {
|
||
|
|
if (s.size() > 64) return false;
|
||
|
|
for (char c : s)
|
||
|
|
if (!std::isalnum((unsigned char)c) && c != '-' && c != '_') return false;
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string voice_of(const std::string& value) {
|
||
|
|
std::string voice = value.empty() ? google::Credentials::env("GOOGLE_VOICE_NAME") : value;
|
||
|
|
if (!valid_voice(voice))
|
||
|
|
throw fw::HttpError(422, "validation_error", "voice has invalid characters");
|
||
|
|
return voice;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Maps Google part errors onto the example's JSON error format.
|
||
|
|
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||
|
|
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||
|
|
throw fw::HttpError(503, "google_not_configured", e.message());
|
||
|
|
// Local input validation carries no service name; upstream errors do.
|
||
|
|
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||
|
|
throw fw::HttpError(422, "validation_error", e.message());
|
||
|
|
throw fw::HttpError(502, "google_error",
|
||
|
|
e.status() + ": " + e.message());
|
||
|
|
}
|
||
|
|
|
||
|
|
template <typename F>
|
||
|
|
auto guarded(F&& f) -> decltype(f()) {
|
||
|
|
try {
|
||
|
|
return f();
|
||
|
|
} catch (const google::GoogleError& e) {
|
||
|
|
rethrow(e);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string quote(const std::string& s) {
|
||
|
|
return "\"" + fw::json_escape(s) + "\"";
|
||
|
|
}
|
||
|
|
|
||
|
|
fw::Response no_store(fw::Response res) {
|
||
|
|
res.headers.set("Cache-Control", "no-store");
|
||
|
|
return res;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
void register_voice_demo(fw::Router& router, Services& services) {
|
||
|
|
if (!services.google) services.google = google::GoogleClient::from_env();
|
||
|
|
auto gclient = services.google;
|
||
|
|
google::VoiceBackend backend =
|
||
|
|
services.voice_backend.value_or(google::voice_backend_from_env());
|
||
|
|
google::VoiceParts parts = google::make_voice_parts(gclient, backend);
|
||
|
|
auto tts = parts.synthesizer;
|
||
|
|
auto stt = parts.recognizer;
|
||
|
|
services.voice = std::make_shared<google::VoicePipeline>(
|
||
|
|
stt, parts.responder, tts, std::make_shared<google::VoiceSessions>());
|
||
|
|
auto pipeline = services.voice;
|
||
|
|
fw::Environment* env = services.templates.get();
|
||
|
|
|
||
|
|
services.add_feature("google", "/google/status");
|
||
|
|
services.add_feature("voice", "/voice");
|
||
|
|
|
||
|
|
router.get("/google/status", "google-status", [gclient, backend](fw::Request req) {
|
||
|
|
(void)req;
|
||
|
|
std::string out = std::string("{\"configured\":") +
|
||
|
|
(gclient->configured() ? "true" : "false") +
|
||
|
|
",\"credential\":\"x-goog-api-key\",\"backend\":" +
|
||
|
|
quote(google::voice_backend_name(backend)) + ",\"model\":" +
|
||
|
|
quote(google::Generative::default_model()) +
|
||
|
|
",\"language\":" + quote(env_or("GOOGLE_VOICE_LANG", "nl-NL")) +
|
||
|
|
",\"services\":[";
|
||
|
|
bool first = true;
|
||
|
|
for (const auto& e : google::endpoints::catalog()) {
|
||
|
|
out += std::string(first ? "" : ",") + "{\"name\":" + quote(e.name) +
|
||
|
|
",\"base_url\":" + quote(e.base_url) + "}";
|
||
|
|
first = false;
|
||
|
|
}
|
||
|
|
return no_store(fw::JsonResponse(out + "]}", 200));
|
||
|
|
});
|
||
|
|
|
||
|
|
router.get("/voice", "voice-page", [env](fw::Request req) {
|
||
|
|
(void)req;
|
||
|
|
fw::TemplateCtx ctx;
|
||
|
|
ctx["title"] = std::string("Voice conversation");
|
||
|
|
return fw::Response(200, env->render("voice.html", ctx),
|
||
|
|
"text/html; charset=utf-8");
|
||
|
|
});
|
||
|
|
|
||
|
|
// Text in, spoken audio (mp3) out.
|
||
|
|
router.post("/voice/say", "voice-say", [tts](fw::Request req) {
|
||
|
|
auto in = fw::parse_json_object_flat(req.body);
|
||
|
|
google::SynthesisRequest speak;
|
||
|
|
speak.text = in.count("text") ? in["text"] : "";
|
||
|
|
speak.language = language_of(in.count("lang") ? in["lang"] : "");
|
||
|
|
speak.voice = voice_of(in.count("voice") ? in["voice"] : "");
|
||
|
|
return guarded([&] {
|
||
|
|
google::Audio audio = tts->synthesize(speak);
|
||
|
|
return no_store(fw::Response(200, audio.bytes, audio.mime));
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// Audio bytes in (Content-Type names the encoding), transcript out.
|
||
|
|
router.post("/voice/hear", "voice-hear", [stt](fw::Request req) {
|
||
|
|
google::RecognitionRequest heard;
|
||
|
|
heard.audio = req.body;
|
||
|
|
if (!google::speech_encoding_from_mime(req.headers.get("Content-Type"),
|
||
|
|
heard.encoding))
|
||
|
|
throw fw::HttpError(415, "unsupported_media_type",
|
||
|
|
"send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg");
|
||
|
|
heard.language = language_of(req.query_or("lang", ""));
|
||
|
|
return guarded([&] {
|
||
|
|
google::Transcript t = stt->recognize(heard);
|
||
|
|
return no_store(fw::JsonResponse(
|
||
|
|
"{\"text\":" + quote(t.text) +
|
||
|
|
",\"confidence\":" + std::to_string(t.confidence) + "}",
|
||
|
|
200));
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// One conversation turn. Audio body = spoken turn; JSON body {"text"} =
|
||
|
|
// typed turn. Both answer {transcript, reply, audio_b64, mime, language};
|
||
|
|
// silence answers an empty transcript and no audio.
|
||
|
|
router.post("/voice/turn", "voice-turn", [pipeline](fw::Request req) {
|
||
|
|
std::string cid = req.query_or("cid", "");
|
||
|
|
if (!google::VoiceSessions::valid_id(cid))
|
||
|
|
throw fw::HttpError(422, "validation_error",
|
||
|
|
"cid must be 1-64 chars of A-Z a-z 0-9 - _");
|
||
|
|
std::string lang = language_of(req.query_or("lang", ""));
|
||
|
|
std::string voice = voice_of(req.query_or("voice", ""));
|
||
|
|
std::string ctype = req.headers.get("Content-Type");
|
||
|
|
return guarded([&] {
|
||
|
|
google::VoiceTurn turn;
|
||
|
|
if (ctype.find("json") != std::string::npos) {
|
||
|
|
auto in = fw::parse_json_object_flat(req.body);
|
||
|
|
turn = pipeline->turn_text(cid, in.count("text") ? in["text"] : "",
|
||
|
|
lang, voice);
|
||
|
|
} else {
|
||
|
|
google::RecognitionRequest heard;
|
||
|
|
heard.audio = req.body;
|
||
|
|
heard.language = lang;
|
||
|
|
if (!google::speech_encoding_from_mime(ctype, heard.encoding))
|
||
|
|
throw fw::HttpError(415, "unsupported_media_type",
|
||
|
|
"send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg");
|
||
|
|
turn = pipeline->turn(cid, heard, voice);
|
||
|
|
}
|
||
|
|
return no_store(fw::JsonResponse(
|
||
|
|
"{\"transcript\":" + quote(turn.transcript) +
|
||
|
|
",\"reply\":" + quote(turn.reply) + ",\"language\":" +
|
||
|
|
quote(turn.language) + ",\"mime\":" + quote(turn.audio.mime) +
|
||
|
|
",\"audio_b64\":\"" +
|
||
|
|
security::crypto::Base64::encode(turn.audio.bytes) + "\"}",
|
||
|
|
200));
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
router.post("/voice/reset", "voice-reset", [pipeline](fw::Request req) {
|
||
|
|
std::string cid = req.query_or("cid", "");
|
||
|
|
if (!google::VoiceSessions::valid_id(cid))
|
||
|
|
throw fw::HttpError(422, "validation_error",
|
||
|
|
"cid must be 1-64 chars of A-Z a-z 0-9 - _");
|
||
|
|
pipeline->sessions().reset(cid);
|
||
|
|
return fw::JsonResponse("{\"reset\":true}", 200);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace molo::example::app
|