diff --git a/.gitignore b/.gitignore index 79b039d..2f49025 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build/ *.o *.a +.env.json.local diff --git a/AGENTS.md b/AGENTS.md index 808785c..f345cea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ make test # whole repo make -f example/example.mk example-test # this part only, standalone ``` -Expected: `example suite: 105 passed, 0 failed`. Failure action: reproduce +Expected: `example suite: 111 passed, 0 failed`. Failure action: reproduce through the real stack first (failing route-level test), then fix; the reproduction stays in the suite. Conventions: `tests/README.md`. diff --git a/Makefile b/Makefile index d98bae9..92365cd 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ clean:: -include src/realtime/realtime.mk -include src/supervisor/supervisor.mk -include src/clients/clients.mk +-include src/google/google.mk # NOTE: supplychain.mk and operability.mk precede cli.mk and supervisor.mk # so SUPPLY_LIB / OPERABILITY_LIB are defined when those add link edges # (supply PATCHES section 2, operability PATCHES sections 5-6 and 10). @@ -53,6 +54,7 @@ all:: data-all all:: app-build all:: $(REALTIME_LIB) all:: clients +all:: google all:: $(MOLO_BIN) all:: tenancy-tests all:: audit-tests @@ -69,6 +71,7 @@ test:: data-test test:: app-test test:: realtime-test test:: clients-test +test:: google-test test:: cli-test test:: tenancy-check test:: audit-check diff --git a/README.md b/README.md index 25fad8f..4529dcd 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,13 @@ is not defined in the Makefile yet. ## The living example -A route-level demo of every feature, with 105 integration tests on the real +A route-level demo of every feature, with 111 integration tests on the real stack (no mocks). Standalone, no other part needed. ```sh make -f example/example.mk example # build ./build/example/living-server 8080 # run -make -f example/example.mk example-test # expect: 105 passed, 0 failed +make -f example/example.mk example-test # expect: 111 passed, 0 failed ``` Open `http://127.0.0.1:8080/` and click through the matrix. Narrow the @@ -54,6 +54,20 @@ curl localhost:8080/health # {"status":"ok","tls":"stub-no-wolfssl"} Open `/stats` for live Chart.js charts of requests, latency, bytes and connections (Chart.js is embedded into the binary at build time). The feature -> route -> test -> plan matrix lives in `example/README.md`. + +Open `/voice` for a live spoken conversation (needs a Google API key with +Text-to-Speech, Speech-to-Text and Generative Language enabled): + +```sh +GOOGLE_API_KEY=... ./build/example/living-server 8080 # then open /voice +``` + +Click Start, talk, pause: the answer is spoken back, and speaking over it +interrupts it. Optional env: `GOOGLE_VOICE_LANG` (default `nl-NL`), +`GOOGLE_VOICE_NAME`, `GOOGLE_GENAI_MODEL` (default `gemini-3.5-flash`), `GOOGLE_STT_MODEL`, `GOOGLE_TTS_MODEL` (default `gemini-2.5-flash-tts`), `GOOGLE_API_BACKEND`. +The Google layer is `src/google` (spec: `src/google/SPEC.md`); its route +tests script the Google side, `MOLO_LIVE_NET=1 make google-test` adds the +opt-in real-network checks. Test conventions (one test file per URL, Arrange/Act/Assert, integration only) are in `tests/README.md`. @@ -183,6 +197,7 @@ src/app application, routers, views, forms, validation, templates src/realtime WebSocket, SSE, Socket.IO, streaming, WebDAV, expose tunnels src/supervisor process supervisor, tasks, replication src/clients HTTP/REST client, OpenAPI, test double +src/google Google Cloud APIs: generic client, TTS, STT, Gemini, voice pipeline src/cli molo binary: CLI, dev server, embedder, AI, ops, i18n, CSP src/tenancy multi-tenancy src/audit hash-chained audit log, GDPR export/erasure, retention diff --git a/example/README.md b/example/README.md index 6b4ad14..55a3c0b 100644 --- a/example/README.md +++ b/example/README.md @@ -14,7 +14,7 @@ Open `http://127.0.0.1:8080/` and click through the matrix; `/stats` has the charts (Chart.js is embedded in the binary, no CDN). Narrow the window to 360px: the top nav collapses into a dropdown menu. -Run the suite (105 route-level integration tests, real stack, no mocks): +Run the suite (111 route-level integration tests, real stack, no mocks): ```sh make -f example/example.mk example-test @@ -114,6 +114,12 @@ on a missing row. | statistics | `/stats` | `tests/example/stats/page_test.cpp` | `plans/02-observability.md` | | statistics data | `/stats/data` | `tests/example/stats/data_test.cpp` | `plans/02-observability.md` | +| google status | `/google/status` | `tests/example/google/status_test.cpp` | `src/google/SPEC.md` | +| voice conversation page | `/voice` | `tests/example/voice/page_test.cpp` | `src/google/SPEC.md` | +| voice text to speech | `/voice/say` | `tests/example/voice/say_test.cpp` | `src/google/SPEC.md` | +| voice speech to text | `/voice/hear` | `tests/example/voice/hear_test.cpp` | `src/google/SPEC.md` | +| voice turn | `/voice/turn` | `tests/example/voice/turn_test.cpp` | `src/google/SPEC.md` | +| voice reset | `/voice/reset` | `tests/example/voice/reset_test.cpp` | `src/google/SPEC.md` | ### Extended features @@ -203,7 +209,7 @@ example/ fw/ # skeletal framework public API (namespace molo::example::fw) app/ # demo routes (namespace molo::example::app) static/ # disk static assets (responsive CSS, tiny JS) - assets/ # embedded at build time (Chart.js, stats.js) via src/cli/embed + assets/ # embedded at build time (Chart.js, stats.js, voice.js) via src/cli/embed tools/ # build helper wrapping the embedder tests/example/ # route-level integration tests mirroring URLs ``` diff --git a/example/SPEC.md b/example/SPEC.md index f9cce07..4ef1fd8 100644 --- a/example/SPEC.md +++ b/example/SPEC.md @@ -45,6 +45,20 @@ owning sibling plan and the file that will close it. builder, singleflight, ttl store). Each is a skeleton of the real `src/` module; per-module gaps are in the header of its `fw/` file. +16. Voice (`src/google`, the real framework part, not a skeleton): + `GET /voice` is a hands-free conversation page. The browser detects + speech from the microphone level, records one utterance, posts it to + `POST /voice/turn`, plays the spoken answer, and listens again; + speaking over the answer interrupts it. The server runs Google + speech-to-text, a Gemini reply and Google text-to-speech per turn, with + bounded per-conversation history. `POST /voice/say` and `/voice/hear` + expose the two speech services alone; `GET /google/status` shows + credential state and known services (never the key). No + `GOOGLE_API_KEY`: 503 `google_not_configured`. Upstream failure: 502 + `google_error` with Google's status. Bad input: 422 or 415 before any + network call. Routes and tests use a scripted transport; the key lives + only in the `x-goog-api-key` header. + ## Caps All ceilings live as literals in `example/fw/Caps.hpp`: 1 MiB request body, @@ -76,11 +90,13 @@ auth, 403 bad credentials, 404 unknown, 405 wrong method, 406 no format, ## Isolation -May depend on: C++23 stdlib, POSIX sockets, `example/fw/` only. Build-time -only: the embedder sources in `src/cli/embed` (asset embedding). May NOT -depend on: any `src/` tree (does not exist yet in this worktree), wolfSSL -(stub behind `ITlsContext`), real Postgres/SQLite (in-memory doubles), -real network except loopback. Namespaces mirror directories: +May depend on: C++23 stdlib, POSIX sockets, `example/fw/`, and the real +framework parts it links on purpose: `src/google` over `src/clients` (HTTPS +via wolfSSL) and `src/operability`. Build-time only: the embedder sources in +`src/cli/embed` (asset embedding). May NOT +depend on: other `src/` trees, real Postgres/SQLite (in-memory doubles), +real network except Google through `src/google` (tests script that +transport; only the opt-in `MOLO_LIVE_NET=1` tests touch the network). Namespaces mirror directories: `molo::example::fw` and `molo::example::app`. ## Gaps @@ -98,6 +114,11 @@ real network except loopback. Namespaces mirror directories: - Full validation coercions, template sandbox, DI container, flash messages: owner 05-request-routing.md, files `example/fw/Validate.*`, `example/fw/Templates.*`, `example/app/App.*`. +- Voice turns are request/response over REST (about 2 to 4 s per turn). + The loopback server accepts sequentially, so one in-flight turn blocks + other requests meanwhile. Streaming recognition and the Gemini Live API: + owner `src/google/SPEC.md`, gaps section. Owner of concurrent accept: + 01-runtime.md, `example/fw/Server.*`. - Server accepts sequentially and stops after 512 connections (cap above); the stats page shows connections left. Owner: 01-runtime.md, `example/fw/Server.*`. diff --git a/example/app/App.cpp b/example/app/App.cpp index 0de7af3..44c4bc8 100644 --- a/example/app/App.cpp +++ b/example/app/App.cpp @@ -67,6 +67,10 @@ std::shared_ptr build_application(Services& services) { register_resilience_demo(resilience, services); app->include_router("", resilience); + fw::Router voice; + register_voice_demo(voice, services); + app->include_router("", voice); + register_chain_demo(app, services); return app; } diff --git a/example/app/App.hpp b/example/app/App.hpp index fe819f2..fe625e7 100644 --- a/example/app/App.hpp +++ b/example/app/App.hpp @@ -5,6 +5,7 @@ // Gap: real molo::di::Container and plugin registry in 05-request-routing.md. #include +#include #include #include #include @@ -18,6 +19,9 @@ #include "../fw/Observ.hpp" #include "../fw/Stats.hpp" #include "../fw/Templates.hpp" +#include "google/client.h" +#include "google/voice.h" +#include "google/voice_backend.h" namespace molo::example::app { @@ -49,6 +53,12 @@ struct Services { std::shared_ptr openapi = std::make_shared(); std::shared_ptr templates = std::make_shared(); + // Google Cloud client (null: built from GOOGLE_API_KEY on first use) and + // the voice pipeline wired on it. Tests inject a scripted client. + std::shared_ptr google; + std::shared_ptr voice; + // Unset: chosen from GOOGLE_API_BACKEND / the key prefix (see src/google). + std::optional voice_backend; // Feature matrix rows (feature, route) shown on the index page. Each // register_*_demo adds its own rows with add_feature(). std::vector> features; @@ -76,6 +86,7 @@ void register_security_extra_demo(fw::Router& router, Services& services); void register_platform_demo(fw::Router& router, Services& services); void register_enterprise_demo(fw::Router& router, Services& services); void register_resilience_demo(fw::Router& router, Services& services); +void register_voice_demo(fw::Router& router, Services& services); void load_templates(Services& services); void load_embedded(Services& services); diff --git a/example/app/Templates.cpp b/example/app/Templates.cpp index 4f20768..cbe9967 100644 --- a/example/app/Templates.cpp +++ b/example/app/Templates.cpp @@ -22,6 +22,7 @@ const char kBase[] = R"BASE( Templates Forms Stats +Voice Docs Health @@ -32,6 +33,7 @@ const char kBase[] = R"BASE( Templates Forms Stats +Voice Docs Health @@ -168,6 +170,37 @@ after its connection cap, so auto refresh is off by default.

{% endblock %} )STATS"; +const char kVoice[] = R"VOICE({% extends "base.html" %} +{% block content %} +

Voice conversation

+

Talk, pause, and the assistant answers out loud. Google +speech-to-text, Gemini, and Google text-to-speech run on the server. +Speak over the answer to interrupt it.

+

Checking Google credentials...

+
+
+ + +
+
idle +
+ +
    +
    + + + +
    +
    + +{% endblock %} +)VOICE"; + const char kDocs[] = R"DOCS({% extends "base.html" %} {% block content %}

    API docs

    @@ -198,6 +231,7 @@ void load_templates(Services& services) { services.templates->add_template("page.html", kPage); services.templates->add_template("form_page.html", kFormPage); services.templates->add_template("docs.html", kDocs); + services.templates->add_template("voice.html", kVoice); services.templates->add_template("stats.html", kStats); services.templates->add_template("error.html", kError); services.templates->add_template("nav_hint.html", kNavHint); diff --git a/example/app/VoiceDemo.cpp b/example/app/VoiceDemo.cpp new file mode 100644 index 0000000..f5a25dc --- /dev/null +++ b/example/app/VoiceDemo.cpp @@ -0,0 +1,208 @@ +#include "App.hpp" + +#include +#include + +#include "../fw/Router.hpp" +#include "../fw/Templates.hpp" +#include "../fw/Validate.hpp" +#include "google/credentials.h" +#include "google/endpoints.h" +#include "security/crypto/Base64.h" + +namespace molo::example::app { + +// Google Cloud voice over the real src/google part: text-to-speech, +// speech-to-text, a Gemini reply, and the live conversation page. +// Gap: turns are request/response over REST; streaming recognition and the +// Gemini Live API are listed in src/google/SPEC.md. + +namespace { + +std::string env_or(const char* name, const char* fallback) { + std::string v = google::Credentials::env(name); + return v.empty() ? std::string(fallback) : v; +} + +// "nl-NL", "en-US": two or three lowercase letters, dash, two uppercase. +bool valid_language(const std::string& s) { + std::size_t dash = s.find('-'); + if (dash < 2 || dash > 3 || s.size() != dash + 3) return false; + for (std::size_t i = 0; i < dash; ++i) + if (!std::islower((unsigned char)s[i])) return false; + return std::isupper((unsigned char)s[dash + 1]) && + std::isupper((unsigned char)s[dash + 2]); +} + +std::string language_of(const std::string& value) { + std::string lang = value.empty() ? env_or("GOOGLE_VOICE_LANG", "nl-NL") : value; + if (!valid_language(lang)) + throw fw::HttpError(422, "validation_error", "lang must look like nl-NL"); + return lang; +} + +bool valid_voice(const std::string& s) { + if (s.size() > 64) return false; + for (char c : s) + if (!std::isalnum((unsigned char)c) && c != '-' && c != '_') return false; + return true; +} + +std::string voice_of(const std::string& value) { + std::string voice = value.empty() ? google::Credentials::env("GOOGLE_VOICE_NAME") : value; + if (!valid_voice(voice)) + throw fw::HttpError(422, "validation_error", "voice has invalid characters"); + return voice; +} + +// Maps Google part errors onto the example's JSON error format. +[[noreturn]] void rethrow(const google::GoogleError& e) { + if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED") + throw fw::HttpError(503, "google_not_configured", e.message()); + // Local input validation carries no service name; upstream errors do. + if (e.service().empty() && e.status() == "INVALID_ARGUMENT") + throw fw::HttpError(422, "validation_error", e.message()); + throw fw::HttpError(502, "google_error", + e.status() + ": " + e.message()); +} + +template +auto guarded(F&& f) -> decltype(f()) { + try { + return f(); + } catch (const google::GoogleError& e) { + rethrow(e); + } +} + +std::string quote(const std::string& s) { + return "\"" + fw::json_escape(s) + "\""; +} + +fw::Response no_store(fw::Response res) { + res.headers.set("Cache-Control", "no-store"); + return res; +} + +} // namespace + +void register_voice_demo(fw::Router& router, Services& services) { + if (!services.google) services.google = google::GoogleClient::from_env(); + auto gclient = services.google; + google::VoiceBackend backend = + services.voice_backend.value_or(google::voice_backend_from_env()); + google::VoiceParts parts = google::make_voice_parts(gclient, backend); + auto tts = parts.synthesizer; + auto stt = parts.recognizer; + services.voice = std::make_shared( + stt, parts.responder, tts, std::make_shared()); + auto pipeline = services.voice; + fw::Environment* env = services.templates.get(); + + services.add_feature("google", "/google/status"); + services.add_feature("voice", "/voice"); + + router.get("/google/status", "google-status", [gclient, backend](fw::Request req) { + (void)req; + std::string out = std::string("{\"configured\":") + + (gclient->configured() ? "true" : "false") + + ",\"credential\":\"x-goog-api-key\",\"backend\":" + + quote(google::voice_backend_name(backend)) + ",\"model\":" + + quote(google::Generative::default_model()) + + ",\"language\":" + quote(env_or("GOOGLE_VOICE_LANG", "nl-NL")) + + ",\"services\":["; + bool first = true; + for (const auto& e : google::endpoints::catalog()) { + out += std::string(first ? "" : ",") + "{\"name\":" + quote(e.name) + + ",\"base_url\":" + quote(e.base_url) + "}"; + first = false; + } + return no_store(fw::JsonResponse(out + "]}", 200)); + }); + + router.get("/voice", "voice-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Voice conversation"); + return fw::Response(200, env->render("voice.html", ctx), + "text/html; charset=utf-8"); + }); + + // Text in, spoken audio (mp3) out. + router.post("/voice/say", "voice-say", [tts](fw::Request req) { + auto in = fw::parse_json_object_flat(req.body); + google::SynthesisRequest speak; + speak.text = in.count("text") ? in["text"] : ""; + speak.language = language_of(in.count("lang") ? in["lang"] : ""); + speak.voice = voice_of(in.count("voice") ? in["voice"] : ""); + return guarded([&] { + google::Audio audio = tts->synthesize(speak); + return no_store(fw::Response(200, audio.bytes, audio.mime)); + }); + }); + + // Audio bytes in (Content-Type names the encoding), transcript out. + router.post("/voice/hear", "voice-hear", [stt](fw::Request req) { + google::RecognitionRequest heard; + heard.audio = req.body; + if (!google::speech_encoding_from_mime(req.headers.get("Content-Type"), + heard.encoding)) + throw fw::HttpError(415, "unsupported_media_type", + "send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg"); + heard.language = language_of(req.query_or("lang", "")); + return guarded([&] { + google::Transcript t = stt->recognize(heard); + return no_store(fw::JsonResponse( + "{\"text\":" + quote(t.text) + + ",\"confidence\":" + std::to_string(t.confidence) + "}", + 200)); + }); + }); + + // One conversation turn. Audio body = spoken turn; JSON body {"text"} = + // typed turn. Both answer {transcript, reply, audio_b64, mime, language}; + // silence answers an empty transcript and no audio. + router.post("/voice/turn", "voice-turn", [pipeline](fw::Request req) { + std::string cid = req.query_or("cid", ""); + if (!google::VoiceSessions::valid_id(cid)) + throw fw::HttpError(422, "validation_error", + "cid must be 1-64 chars of A-Z a-z 0-9 - _"); + std::string lang = language_of(req.query_or("lang", "")); + std::string voice = voice_of(req.query_or("voice", "")); + std::string ctype = req.headers.get("Content-Type"); + return guarded([&] { + google::VoiceTurn turn; + if (ctype.find("json") != std::string::npos) { + auto in = fw::parse_json_object_flat(req.body); + turn = pipeline->turn_text(cid, in.count("text") ? in["text"] : "", + lang, voice); + } else { + google::RecognitionRequest heard; + heard.audio = req.body; + heard.language = lang; + if (!google::speech_encoding_from_mime(ctype, heard.encoding)) + throw fw::HttpError(415, "unsupported_media_type", + "send audio/webm, audio/ogg, audio/flac, audio/wav or audio/mpeg"); + turn = pipeline->turn(cid, heard, voice); + } + return no_store(fw::JsonResponse( + "{\"transcript\":" + quote(turn.transcript) + + ",\"reply\":" + quote(turn.reply) + ",\"language\":" + + quote(turn.language) + ",\"mime\":" + quote(turn.audio.mime) + + ",\"audio_b64\":\"" + + security::crypto::Base64::encode(turn.audio.bytes) + "\"}", + 200)); + }); + }); + + router.post("/voice/reset", "voice-reset", [pipeline](fw::Request req) { + std::string cid = req.query_or("cid", ""); + if (!google::VoiceSessions::valid_id(cid)) + throw fw::HttpError(422, "validation_error", + "cid must be 1-64 chars of A-Z a-z 0-9 - _"); + pipeline->sessions().reset(cid); + return fw::JsonResponse("{\"reset\":true}", 200); + }); +} + +} // namespace molo::example::app diff --git a/example/assets/voice.js b/example/assets/voice.js new file mode 100644 index 0000000..bac2f5d --- /dev/null +++ b/example/assets/voice.js @@ -0,0 +1,244 @@ +// Voice conversation page: hands-free loop in the browser. +// listen -> detect speech (mic level) -> record one utterance -> POST +// /voice/turn -> play the spoken answer -> listen again. Speaking over the +// answer interrupts it. All limits are constants below. +(function () { + "use strict"; + var LEVEL_START = 0.04; // RMS that counts as speech + var LEVEL_BARGE = 0.09; // RMS that interrupts the assistant's voice + var ONSET_MS = 80; // speech must last this long to start recording + var BARGE_MS = 300; // ... and this long to interrupt playback + var SILENCE_MS = 900; // quiet this long ends the utterance + var MIN_SPEECH_MS = 300; // shorter utterances are noise, discarded + var MAX_UTTERANCE_MS = 30000; + var TICK_MS = 40; + + var $ = function (id) { return document.getElementById(id); }; + var toggle = $("v-toggle"), langSel = $("v-lang"), stateEl = $("v-state"); + var levelEl = $("v-level"), errorEl = $("v-error"), logEl = $("v-log"); + var form = $("v-form"), textEl = $("v-text"), resetBtn = $("v-reset"); + + var cid = newId(); + var stream = null, ctx = null, analyser = null, buf = null, timer = null; + var recorder = null, chunks = [], recStart = 0, voicedMs = 0, quietMs = 0; + var onsetMs = 0, bargeMs = 0; + var player = null; + var state = "idle"; // idle | listening | hearing | thinking | speaking + + function newId() { + var a = new Uint8Array(12); + crypto.getRandomValues(a); + return Array.prototype.map.call(a, function (b) { + return ("0" + b.toString(16)).slice(-2); + }).join(""); + } + function setState(s) { + state = s; + stateEl.textContent = s; + stateEl.className = "pill " + s; + } + function showError(msg) { errorEl.textContent = msg || ""; } + function addLog(who, text) { + var li = document.createElement("li"); + li.className = who; + var b = document.createElement("b"); + b.textContent = who === "you" ? "You" : "Assistant"; + li.appendChild(b); + li.appendChild(document.createTextNode(": " + text)); + logEl.appendChild(li); + li.scrollIntoView({ block: "nearest" }); + } + function errorText(res, body) { + try { + var j = JSON.parse(body); + return j.message || j.error || ("HTTP " + res.status); + } catch (e) { return "HTTP " + res.status; } + } + + function rms() { + analyser.getByteTimeDomainData(buf); + var sum = 0; + for (var i = 0; i < buf.length; i++) { + var v = (buf[i] - 128) / 128; + sum += v * v; + } + return Math.sqrt(sum / buf.length); + } + + function pickMime() { + var list = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"]; + for (var i = 0; i < list.length; i++) + if (window.MediaRecorder && MediaRecorder.isTypeSupported(list[i])) return list[i]; + return ""; + } + + function startRecording() { + chunks = []; + var mime = pickMime(); + recorder = mime ? new MediaRecorder(stream, { mimeType: mime }) : new MediaRecorder(stream); + recorder.ondataavailable = function (e) { if (e.data.size) chunks.push(e.data); }; + recorder.start(250); + recStart = Date.now(); + voicedMs = 0; quietMs = 0; + setState("hearing"); + } + + function stopRecording(send) { + var r = recorder, voiced = voicedMs; + recorder = null; + r.onstop = function () { + var blob = new Blob(chunks, { type: r.mimeType || "audio/webm" }); + chunks = []; + if (send && voiced >= MIN_SPEECH_MS && state !== "idle") sendAudio(blob); + else if (state !== "idle") setState("listening"); + }; + r.stop(); + } + + function turnUrl() { + return "/voice/turn?cid=" + encodeURIComponent(cid) + + "&lang=" + encodeURIComponent(langSel.value); + } + + function sendAudio(blob) { + setState("thinking"); + post(turnUrl(), blob, blob.type || "audio/webm"); + } + + function post(url, body, type) { + showError(""); + fetch(url, { method: "POST", headers: { "Content-Type": type }, body: body }) + .then(function (res) { return res.text().then(function (t) { return [res, t]; }); }) + .then(function (pair) { + var res = pair[0], text = pair[1]; + if (!res.ok) { + showError(errorText(res, text)); + if (res.status === 503) { stop(); return; } + if (state !== "idle") setState(stream ? "listening" : "idle"); + return; + } + handleTurn(JSON.parse(text)); + }) + .catch(function (err) { + showError("Network error: " + err.message); + if (state !== "idle") setState(stream ? "listening" : "idle"); + }); + } + + function handleTurn(turn) { + if (!turn.transcript) { // nothing intelligible was heard + if (state !== "idle") setState(stream ? "listening" : "idle"); + return; + } + addLog("you", turn.transcript); + addLog("bot", turn.reply); + if (state === "idle" && !stream) return; + play(turn); + } + + function play(turn) { + stopPlayback(); + player = new Audio("data:" + turn.mime + ";base64," + turn.audio_b64); + var mine = player; + mine.onended = function () { + if (player === mine) { player = null; if (stream) setState("listening"); else setState("idle"); } + }; + mine.onerror = function () { + if (player === mine) { player = null; showError("Could not play audio"); if (stream) setState("listening"); } + }; + setState(stream ? "speaking" : "idle"); + mine.play().catch(function (e) { showError("Playback blocked: " + e.message); if (stream) setState("listening"); }); + } + + function stopPlayback() { + if (player) { var p = player; player = null; p.onended = null; p.pause(); } + } + + function tick() { + var level = rms(); + levelEl.style.width = Math.min(100, Math.round(level * 400)) + "%"; + if (state === "listening") { + onsetMs = level > LEVEL_START ? onsetMs + TICK_MS : 0; + if (onsetMs >= ONSET_MS) { onsetMs = 0; startRecording(); } + } else if (state === "speaking") { + bargeMs = level > LEVEL_BARGE ? bargeMs + TICK_MS : 0; + if (bargeMs >= BARGE_MS) { bargeMs = 0; stopPlayback(); startRecording(); } + } else if (state === "hearing") { + if (level > LEVEL_START) { voicedMs += TICK_MS; quietMs = 0; } + else quietMs += TICK_MS; + if (quietMs >= SILENCE_MS || Date.now() - recStart >= MAX_UTTERANCE_MS) + stopRecording(true); + } + } + + function start() { + showError(""); + if (!navigator.mediaDevices || !window.MediaRecorder) { + showError("This browser has no microphone recording support."); + return; + } + navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } + }).then(function (s) { + stream = s; + ctx = new (window.AudioContext || window.webkitAudioContext)(); + var src = ctx.createMediaStreamSource(stream); + analyser = ctx.createAnalyser(); + analyser.fftSize = 1024; + buf = new Uint8Array(analyser.fftSize); + src.connect(analyser); + onsetMs = 0; bargeMs = 0; + timer = setInterval(tick, TICK_MS); + toggle.textContent = "Stop conversation"; + toggle.classList.add("live"); + setState("listening"); + }).catch(function (err) { + showError("Microphone unavailable: " + err.message); + }); + } + + function stop() { + if (timer) { clearInterval(timer); timer = null; } + if (recorder) { var r = recorder; recorder = null; r.onstop = null; try { r.stop(); } catch (e) {} } + stopPlayback(); + if (stream) { stream.getTracks().forEach(function (t) { t.stop(); }); stream = null; } + if (ctx) { ctx.close(); ctx = null; } + levelEl.style.width = "0"; + toggle.textContent = "Start conversation"; + toggle.classList.remove("live"); + setState("idle"); + } + + toggle.addEventListener("click", function () { if (stream) stop(); else start(); }); + + form.addEventListener("submit", function (ev) { + ev.preventDefault(); + var text = textEl.value.trim(); + if (!text || state === "thinking") return; + textEl.value = ""; + stopPlayback(); + setState("thinking"); + post(turnUrl(), JSON.stringify({ text: text }), "application/json"); + }); + + resetBtn.addEventListener("click", function () { + fetch("/voice/reset?cid=" + encodeURIComponent(cid), { method: "POST" }); + cid = newId(); + logEl.innerHTML = ""; + showError(""); + }); + + fetch("/google/status").then(function (r) { return r.json(); }).then(function (j) { + var cfg = $("v-config"); + if (j.configured) { + cfg.textContent = "Google credentials found. Model: " + j.model + "."; + cfg.className = "ok"; + toggle.disabled = false; + } else { + cfg.textContent = "No Google API key. Start the server with GOOGLE_API_KEY set."; + cfg.className = "error"; + } + }).catch(function () { $("v-config").textContent = "Status unavailable."; }); + + setState("idle"); +})(); diff --git a/example/check_locust_coverage.py b/example/check_locust_coverage.py index f95eb08..a7fa96e 100644 --- a/example/check_locust_coverage.py +++ b/example/check_locust_coverage.py @@ -12,6 +12,10 @@ HERE = Path(__file__).resolve().parent APP = HERE / "app" LOCUST = HERE / "locustfile.py" +# Routes that call paid Google APIs: never load tested. The check fails if +# locust references one, and does not require them to be covered. +EXCLUDED = {"/voice/say", "/voice/hear", "/voice/turn", "/voice/reset"} + FILLS = {"name": "locust", "id": "1", "status": "404", "tenant": "acme"} @@ -55,7 +59,7 @@ def main() -> int: routes = collect_routes() urls, methods = collect_locust_urls() missing: list[str] = [] - for r in sorted(routes): + for r in sorted(routes - EXCLUDED): c = fill(r) ok = ( c in urls @@ -68,6 +72,9 @@ def main() -> int: ) if not ok: missing.append(f"{r} -> {c}") + leaked = sorted(EXCLUDED & urls) + for r in leaked: + print("FORBIDDEN (costs money):", r) required_methods = {"get", "post", "put", "options"} missing_methods = required_methods - methods print(f"routes={len(routes)} urls={len(urls)} methods={sorted(methods)}") @@ -75,10 +82,13 @@ def main() -> int: print("MISSING METHODS:", sorted(missing_methods)) for m in missing: print("MISS", m) + if leaked: + print(f"FAIL: {len(leaked)} paid Google routes in locust") + return 1 if missing or missing_methods: print(f"FAIL: {len(missing)} missing routes, {len(missing_methods)} missing methods") return 1 - print(f"OK: {len(routes)}/{len(routes)} routes covered, methods {sorted(methods)}") + print(f"OK: {len(routes) - len(EXCLUDED)} routes covered, {len(EXCLUDED)} paid Google routes excluded, methods {sorted(methods)}") return 0 diff --git a/example/example.mk b/example/example.mk index 58f6688..781db29 100644 --- a/example/example.mk +++ b/example/example.mk @@ -52,7 +52,8 @@ EXAMPLE_SRCS = \ $(EXAMPLE_DIR)/app/SecurityExtraDemo.cpp \ $(EXAMPLE_DIR)/app/PlatformDemo.cpp \ $(EXAMPLE_DIR)/app/EnterpriseDemo.cpp \ - $(EXAMPLE_DIR)/app/ResilienceDemo.cpp + $(EXAMPLE_DIR)/app/ResilienceDemo.cpp \ + $(EXAMPLE_DIR)/app/VoiceDemo.cpp # Tests are discovered: every tests/example/**/*_test.cpp is one route test. EXAMPLE_TEST_JOBS ?= 8 @@ -68,10 +69,15 @@ EXAMPLE_ASSETS = \ $(EXAMPLE_DIR)/assets/app.css \ $(EXAMPLE_DIR)/assets/chart.umd.min.js \ $(EXAMPLE_DIR)/assets/hello.txt \ - $(EXAMPLE_DIR)/assets/stats.js -EXAMPLE_INC = -I$(EXAMPLE_DIR) -I$(EXAMPLE_BUILD) + $(EXAMPLE_DIR)/assets/stats.js \ + $(EXAMPLE_DIR)/assets/voice.js +EXAMPLE_INC = -I$(EXAMPLE_DIR) -I$(EXAMPLE_BUILD) -Isrc +# Real framework parts the example links: Google Cloud (src/google) over the +# HTTP client (src/clients, wolfSSL TLS) and the operability breaker. +EXAMPLE_REAL_LIBS = build/google/libgoogle.a build/clients/libclients.a \ + build/liboperability.a -.PHONY: example example-assets example-test example-clean example-locust +.PHONY: example example-real-libs example-assets example-test example-clean example-locust # Asset embedding is part of the build: example/assets (Chart.js included) # becomes generated C++ under build/example/gen via the MoloApp embedder. @@ -86,7 +92,12 @@ $(EXAMPLE_GEN)/assets.cpp: $(EXAMPLE_ASSETS) $(EXAMPLE_DIR)/tools/embed_assets.c $(EXAMPLE_BUILD)/embed_assets $(EXAMPLE_DIR)/assets \ $(EXAMPLE_GEN)/assets.cpp $(EXAMPLE_GEN)/assets.hpp molo::example::gen -example: example-assets +example-real-libs: + $(MAKE) -f src/operability.mk build/liboperability.a + $(MAKE) -f src/clients/clients.mk clients + $(MAKE) -f src/google/google.mk google + +example: example-assets example-real-libs mkdir -p $(EXAMPLE_BUILD)/obj for s in $(EXAMPLE_SRCS); do \ o=$(EXAMPLE_BUILD)/obj/`echo $$s | tr '/' '_'`.o; \ @@ -98,7 +109,7 @@ example: example-assets $(MAKE) -f src/supervisor/supervisor.mk build/supervisor/libsupervisor.a $(CXX) $(EXAMPLE_CXXFLAGS) -I$(EXAMPLE_DIR) -Isrc $(EXAMPLE_DIR)/main.cpp \ $(EXAMPLE_BUILD)/libexample.a build/supervisor/libsupervisor.a \ - -o $(EXAMPLE_BUILD)/living-server + $(EXAMPLE_REAL_LIBS) -lwolfssl -o $(EXAMPLE_BUILD)/living-server example-test: example rm -rf $(EXAMPLE_BUILD)/tests @@ -107,8 +118,8 @@ example-test: example xargs -P $(EXAMPLE_TEST_JOBS) -I{} sh -c ' \ t={}; n=`echo $$t | sed "s|$(EXAMPLE_TESTDIR)/||; s|/|_|g; s|\\.cpp$$||"`; \ b=$(EXAMPLE_BUILD)/tests/$$n; \ - if $(CXX) $(EXAMPLE_CXXFLAGS) -I$(EXAMPLE_DIR) -I$(EXAMPLE_TESTDIR) \ - $$t $(EXAMPLE_BUILD)/libexample.a -o $$b > $$b.log 2>&1; \ + if $(CXX) $(EXAMPLE_CXXFLAGS) -I$(EXAMPLE_DIR) -I$(EXAMPLE_TESTDIR) -Isrc \ + $$t $(EXAMPLE_BUILD)/libexample.a $(EXAMPLE_REAL_LIBS) -lwolfssl -o $$b > $$b.log 2>&1; \ then $$b >> $$b.log 2>&1; else echo "COMPILE FAILED: $$t" >> $$b.log; false; fi; \ echo $$? > $$b.rc' < $(EXAMPLE_BUILD)/tests/list.txt pass=0; fail=0; \ diff --git a/example/locustfile.py b/example/locustfile.py index e75138d..463a439 100644 --- a/example/locustfile.py +++ b/example/locustfile.py @@ -39,6 +39,7 @@ GETS = [ "/csp/page", "/csp/report", "/di/services", "/di/resolve/locust", "/embedded/chart.umd.min.js", "/embedded/hello.txt", "/embedded/stats.js", + "/embedded/voice.js", "/google/status", "/voice", "/events/fire", "/expose/record", "/expose/stats", "/files/", "/files/welcome.txt", "/flash", "/flash/set", @@ -87,6 +88,8 @@ GETS = [ "/webdav/files/test.txt", ] +# NOT load tested on purpose: /voice/say, /voice/hear, /voice/turn and +# /voice/reset call paid Google APIs (check_locust_coverage.py enforces this). POSTS = [ ("/audit/log", {"event": "locust"}), ("/audit/retention/hold", {"hold": True}), diff --git a/example/static/app.css b/example/static/app.css index 01de16f..bb213ec 100644 --- a/example/static/app.css +++ b/example/static/app.css @@ -76,3 +76,29 @@ label.inline input { padding: 0; } .nav-wide { display: flex; } .nav-narrow { display: none; } } + +/* Voice conversation page */ +.voice { display: flex; flex-direction: column; gap: 0.75rem; } +.voice-controls { display: flex; flex-wrap: wrap; align-items: center; gap: 0.75rem; } +.voice-controls button.primary { flex: 1 1 12rem; padding: 0.8rem 1rem; font-weight: 600; + border: 0; border-radius: 6px; background: var(--series-1); color: #fff; } +.voice-controls button.primary.live { background: var(--series-8); } +.voice-controls button:disabled { opacity: 0.5; } +.voice-controls select { font-size: 1rem; padding: 0.4rem; } +.voice-state { display: flex; align-items: center; gap: 0.75rem; } +.pill { min-width: 6rem; text-align: center; padding: 0.2rem 0.6rem; border-radius: 1rem; + border: 1px solid var(--border); color: var(--text-secondary); } +.pill.listening { border-color: var(--series-3); color: var(--series-3); } +.pill.hearing { border-color: var(--series-2); color: var(--series-2); } +.pill.thinking { border-color: var(--series-4); color: var(--series-4); } +.pill.speaking { border-color: var(--series-1); color: var(--series-1); } +.meter { flex: 1; height: 0.6rem; border: 1px solid var(--border); border-radius: 0.3rem; overflow: hidden; } +.meter > div { height: 100%; width: 0; background: var(--series-3); } +.voice-log { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; + gap: 0.4rem; max-height: 22rem; overflow-y: auto; } +.voice-log li { padding: 0.4rem 0.6rem; border: 1px solid var(--border); border-radius: 6px; + overflow-wrap: anywhere; } +.voice-log li.you { border-left: 4px solid var(--series-2); } +.voice-log li.bot { border-left: 4px solid var(--series-1); } +.voice-form { flex-direction: row; flex-wrap: wrap; max-width: none; } +.voice-form input[type="text"] { flex: 1 1 12rem; min-width: 0; } diff --git a/src/google/SPEC.md b/src/google/SPEC.md new file mode 100644 index 0000000..6f2f4bd --- /dev/null +++ b/src/google/SPEC.md @@ -0,0 +1,90 @@ +# Google Cloud APIs - Specification + +Owned directory: `src/google/`. Namespace `molo::google`. Depends on +`molo::clients` (HTTP client, JSON DOM, transports) and the Base64 codec in +`src/security/crypto`. No other part is included. + +Purpose: one generic way to call any Google Cloud REST API, plus concrete +services built on it. Voice (text-to-speech, speech-to-text, Gemini reply and audio, +conversation pipeline) is the first set. Adding a service means one +`ServiceEndpoint` constant and one `Service` subclass; no change to the +client, credentials, errors, or caps. + +## 1. Module map (one class or concern per file) + +| File | Concern | +|---|---| +| `caps.h` | Static ceilings, never autotuned | +| `errors.h/.cpp` | `GoogleError` (http status, Google status, message), `NotConfiguredError`, parser for Google's `{"error":{...}}` body | +| `credentials.h/.cpp` | `Credentials`: API key and OAuth bearer providers, env loader | +| `endpoints.h/.cpp` | `ServiceEndpoint` plus the catalog of known services | +| `client.h/.cpp` | `GoogleClient`: auth, JSON in/out, error mapping, over any `clients::Transport` | +| `service.h/.cpp` | `Service` base class: holds client plus endpoint, one `call()` | +| `text_to_speech.h/.cpp` | `TextToSpeech::synthesize`, `voices` | +| `speech_to_text.h/.cpp` | `SpeechToText::recognize` | +| `generative.h/.cpp` | `Generative::generate` (Gemini `generateContent`) | +| `voice_io.h/.cpp` | `Recognizer`, `Synthesizer` seams plus request and audio types | +| `gemini_voice.h/.cpp` | Gemini-based `Recognizer` and `Synthesizer` | +| `voice_backend.h/.cpp` | `cloud` / `gemini` / `vertex` selection and wiring | +| `wav.h/.cpp` | PCM16 to WAV | +| `voice.h/.cpp` | `Responder`, `GenerativeResponder`, `VoiceSessions`, `VoicePipeline` | + +## 2. Static caps (see caps.h) + +| Knob | Value | +|---|---| +| TTS text characters per request | 5000 | +| STT audio bytes per request | 10 MiB (Google sync limit) | +| Generation output tokens | 300 default, 1024 max | +| Conversation history | 20 turns per conversation | +| Conversations held in memory | 1000, oldest evicted | +| Conversation id | `[A-Za-z0-9_-]`, 1 to 64 chars | +| Message text per history entry | 4000 characters | +| Request timeout (total) | 60 s (client default) | + +## 3. Contracts + +- Credentials: `GOOGLE_API_KEY` is read once by `GoogleClient::from_env()` + from the process environment, else `.env.json.local` (git-ignored, put + secrets here), else `.env.json`, in the working directory. All env knobs + below follow the same order. + The key travels in the `x-goog-api-key` header, never in the URL, so it + never reaches a span, log line, or proxy URL. Bearer tokens (OAuth, for + APIs that refuse keys) use the same seam: `Credentials::bearer`. +- No key: `configured()` is false and every call throws `NotConfiguredError` + (HTTP 503, status `NOT_CONFIGURED`) before any network traffic. Loud, not + silent. +- Upstream failure: any non-2xx becomes `GoogleError` carrying the HTTP + status, Google's `status` string (`PERMISSION_DENIED`, `RESOURCE_EXHAUSTED`, + ...) and message. Transport failures become `GoogleError` with status + `UNAVAILABLE`. The key is never part of an error message. +- Invalid input (empty text, text over cap, empty or oversized audio, bad + conversation id) throws `GoogleError` 400 `INVALID_ARGUMENT` before any + network traffic. +- Transports are swappable: production uses `HttpTransport` (HTTPS via + wolfSSL), tests use `MockTransport`. Calling code does not change. +- Voice replies disable model thinking (`thinkingBudget` 0): reasoning tokens + count against the output cap and truncated spoken replies in practice. +- Backends (`voice_backend.h`): `cloud` (classic TTS/STT), `gemini` + (generativelanguage audio in/out), `vertex` (aiplatform, express keys + starting `AQ.`). `GOOGLE_API_BACKEND` overrides the key-prefix guess. + Gemini TTS returns raw PCM; the server wraps it as WAV. +- Env knobs: `GOOGLE_API_KEY`, `GOOGLE_VOICE_LANG` (default `nl-NL`), + `GOOGLE_VOICE_NAME` (default empty: Google picks the language default), + `GOOGLE_GENAI_MODEL` (default `gemini-3.5-flash`). +- `VoicePipeline::turn` = recognize, respond, synthesize. Silence (empty + transcript) returns an empty result with no reply and no synthesis, so a + quiet room costs one STT call and nothing else. +- History is bounded per conversation and conversations are bounded in + count; both evict oldest first. + +## 4. Gaps + +- Streaming recognition (gRPC/WebSocket) and the Gemini Live API are not + implemented; turns are request/response over REST. +- Service-account JWT exchange for OAuth tokens: callers pass a ready token + to `Credentials::bearer`. Owner: this part, `credentials.cpp`. +- Free-tier quota accounting is Google's; `RESOURCE_EXHAUSTED` is surfaced, + not tracked. +- Live behavior against Google needs a real `GOOGLE_API_KEY`; the suite + proves request shape and error mapping with scripted responses only. diff --git a/src/google/caps.h b/src/google/caps.h new file mode 100644 index 0000000..387ae2f --- /dev/null +++ b/src/google/caps.h @@ -0,0 +1,20 @@ +#pragma once + +// Static ceilings for the Google Cloud part. Never autotuned. + +#include + +namespace molo::google { + +struct Caps { + static constexpr std::size_t kMaxTtsChars = 5000; + static constexpr std::size_t kMaxSttBytes = 10u * 1024u * 1024u; + static constexpr int kDefaultOutputTokens = 300; + static constexpr int kMaxOutputTokens = 1024; + static constexpr std::size_t kMaxHistoryTurns = 20; + static constexpr std::size_t kMaxConversations = 1000; + static constexpr std::size_t kMaxConversationId = 64; + static constexpr std::size_t kMaxMessageChars = 4000; +}; + +} // namespace molo::google diff --git a/src/google/client.cpp b/src/google/client.cpp new file mode 100644 index 0000000..1a341f5 --- /dev/null +++ b/src/google/client.cpp @@ -0,0 +1,64 @@ +#include "google/client.h" + +#include "clients/transports/http_transport.h" +#include "google/credentials.h" + +namespace molo::google { + +namespace { +clients::ClientConfig make_config(std::shared_ptr auth) { + clients::ClientConfig config; + config.auth = std::move(auth); + config.persist_cookies = false; + return config; +} +} // namespace + +GoogleClient::GoogleClient(std::unique_ptr transport, + std::shared_ptr auth) + : client_(std::move(transport), make_config(auth)), + configured_(auth != nullptr) {} + +std::shared_ptr GoogleClient::from_env() { + return std::make_shared( + std::make_unique(), + Credentials::from_env()); +} + +clients::body::Json GoogleClient::post_json(const ServiceEndpoint& service, + const std::string& path, + const clients::body::Json& body) { + if (!configured_) throw NotConfiguredError(service.name); + try { + return finish(service, client_.post_json_blocking( + service.base_url + path, body)); + } catch (const GoogleError&) { + throw; + } catch (const std::exception& e) { + throw GoogleError(502, "UNAVAILABLE", e.what(), service.name); + } +} + +clients::body::Json GoogleClient::get_json(const ServiceEndpoint& service, + const std::string& path) { + if (!configured_) throw NotConfiguredError(service.name); + try { + return finish(service, client_.get_blocking(service.base_url + path)); + } catch (const GoogleError&) { + throw; + } catch (const std::exception& e) { + throw GoogleError(502, "UNAVAILABLE", e.what(), service.name); + } +} + +clients::body::Json GoogleClient::finish( + const ServiceEndpoint& service, const clients::ClientResponse& response) { + if (!response.is_success()) throw error_from_response(service.name, response); + try { + return response.json(); + } catch (const std::exception& e) { + throw GoogleError(502, "BAD_RESPONSE", e.what(), service.name); + } +} + +} // namespace molo::google diff --git a/src/google/client.h b/src/google/client.h new file mode 100644 index 0000000..c6f68b6 --- /dev/null +++ b/src/google/client.h @@ -0,0 +1,45 @@ +#pragma once + +// GoogleClient: the one place that talks to Google. Applies the credential, +// sends JSON, maps failures to GoogleError. Runs over any clients::Transport +// (HttpTransport in production, MockTransport in tests). + +#include +#include + +#include "clients/body/json.h" +#include "clients/client.h" +#include "google/endpoints.h" +#include "google/errors.h" + +namespace molo::google { + +class GoogleClient { + public: + // auth may be null: the client is then "not configured" and every call + // throws NotConfiguredError before touching the network. + GoogleClient(std::unique_ptr transport, + std::shared_ptr auth); + + // Real HTTPS transport plus GOOGLE_API_KEY from the environment. + static std::shared_ptr from_env(); + + bool configured() const { return configured_; } + + clients::body::Json post_json(const ServiceEndpoint& service, + const std::string& path, + const clients::body::Json& body); + clients::body::Json get_json(const ServiceEndpoint& service, + const std::string& path); + + clients::ClientStatsSnapshot stats() const { return client_.stats(); } + + private: + clients::body::Json finish(const ServiceEndpoint& service, + const clients::ClientResponse& response); + + clients::Client client_; + bool configured_; +}; + +} // namespace molo::google diff --git a/src/google/credentials.cpp b/src/google/credentials.cpp new file mode 100644 index 0000000..2490c4e --- /dev/null +++ b/src/google/credentials.cpp @@ -0,0 +1,55 @@ +#include "google/credentials.h" + +#include +#include +#include + +#include "clients/body/json.h" + +namespace molo::google { + +std::shared_ptr Credentials::api_key(std::string key) { + return std::make_shared(std::move(key), "x-goog-api-key"); +} + +std::shared_ptr Credentials::bearer(std::string token) { + return std::make_shared(std::move(token)); +} + +namespace { +constexpr std::size_t kMaxConfigBytes = 64 * 1024; + +// String value of `name` in a flat JSON object file, or empty (missing, +// oversized, malformed, or not a string: all treated as "not set"). +std::string from_json_file(const char* path, const char* name) { + std::ifstream in(path, std::ios::binary); + if (!in) return ""; + std::ostringstream text; + text << in.rdbuf(); + if (text.str().size() > kMaxConfigBytes) return ""; + try { + clients::body::Json doc = clients::body::Json::parse(text.str()); + const clients::body::Json* v = doc.is_object() ? doc.find(name) : nullptr; + return v && v->is_string() ? v->as_string() : ""; + } catch (const std::exception&) { + return ""; + } +} +} // namespace + +// Order: process environment, then .env.json.local (git-ignored, for +// secrets), then .env.json, both in the working directory. +std::string Credentials::env(const char* name) { + if (const char* v = std::getenv(name); v && *v) return v; + std::string local = from_json_file(".env.json.local", name); + if (!local.empty()) return local; + return from_json_file(".env.json", name); +} + +std::shared_ptr Credentials::from_env(const char* name) { + std::string key = env(name); + if (key.empty()) return nullptr; + return api_key(std::move(key)); +} + +} // namespace molo::google diff --git a/src/google/credentials.h b/src/google/credentials.h new file mode 100644 index 0000000..698d0fb --- /dev/null +++ b/src/google/credentials.h @@ -0,0 +1,27 @@ +#pragma once + +// Credential providers for Google APIs. A credential is a clients::Auth, so +// it is applied to every request by the client. API keys go in the +// x-goog-api-key header, never in the URL. + +#include +#include + +#include "clients/auth.h" + +namespace molo::google { + +class Credentials { + public: + static std::shared_ptr api_key(std::string key); + // OAuth access token for APIs that do not accept keys. + static std::shared_ptr bearer(std::string token); + // Value of `name` from the process environment, else .env.json.local, + // else .env.json (working directory); empty when unset everywhere. + static std::string env(const char* name = "GOOGLE_API_KEY"); + // api_key(env()) or nullptr when unset or empty. + static std::shared_ptr from_env( + const char* name = "GOOGLE_API_KEY"); +}; + +} // namespace molo::google diff --git a/src/google/endpoints.cpp b/src/google/endpoints.cpp new file mode 100644 index 0000000..0638919 --- /dev/null +++ b/src/google/endpoints.cpp @@ -0,0 +1,24 @@ +#include "google/endpoints.h" + +namespace molo::google::endpoints { + +ServiceEndpoint text_to_speech() { + return {"texttospeech", "https://texttospeech.googleapis.com/v1"}; +} +ServiceEndpoint speech_to_text() { + return {"speech", "https://speech.googleapis.com/v1"}; +} +ServiceEndpoint generative() { + return {"generativelanguage", + "https://generativelanguage.googleapis.com/v1beta"}; +} + +ServiceEndpoint vertex_generative() { + return {"vertex", "https://aiplatform.googleapis.com/v1/publishers/google"}; +} + +std::vector catalog() { + return {text_to_speech(), speech_to_text(), generative(), vertex_generative()}; +} + +} // namespace molo::google::endpoints diff --git a/src/google/endpoints.h b/src/google/endpoints.h new file mode 100644 index 0000000..002a8e4 --- /dev/null +++ b/src/google/endpoints.h @@ -0,0 +1,26 @@ +#pragma once + +// A Google service is a name plus a base URL (host and version prefix). +// New Google APIs are one constant here and one Service subclass. + +#include +#include + +namespace molo::google { + +struct ServiceEndpoint { + std::string name; + std::string base_url; // No trailing slash. +}; + +namespace endpoints { +ServiceEndpoint text_to_speech(); +ServiceEndpoint speech_to_text(); +ServiceEndpoint generative(); +// Gemini through Vertex AI (express-mode API keys start with "AQ."). +ServiceEndpoint vertex_generative(); +// Every endpoint this build knows, for status pages. +std::vector catalog(); +} // namespace endpoints + +} // namespace molo::google diff --git a/src/google/errors.cpp b/src/google/errors.cpp new file mode 100644 index 0000000..614a4f9 --- /dev/null +++ b/src/google/errors.cpp @@ -0,0 +1,43 @@ +#include "google/errors.h" + +#include "clients/body/json.h" + +namespace molo::google { + +GoogleError::GoogleError(int http_status, std::string status, + const std::string& message, std::string service) + : std::runtime_error("molo::google: " + status + ": " + message), + http_status_(http_status), + status_(std::move(status)), + service_(std::move(service)), + message_(message) {} + +GoogleError GoogleError::invalid(const std::string& message) { + return GoogleError(400, "INVALID_ARGUMENT", message); +} + +NotConfiguredError::NotConfiguredError(const std::string& service) + : GoogleError(503, "NOT_CONFIGURED", + "no Google credential configured (set GOOGLE_API_KEY)", + service) {} + +GoogleError error_from_response(const std::string& service, + const clients::ClientResponse& response) { + std::string status = "UNKNOWN"; + std::string message = "HTTP " + std::to_string(response.status()); + try { + clients::body::Json doc = response.json(); + const clients::body::Json* err = doc.find("error"); + if (err && err->is_object()) { + if (const auto* s = err->find("status"); s && s->is_string()) + status = s->as_string(); + if (const auto* m = err->find("message"); m && m->is_string()) + message = m->as_string(); + } + } catch (const std::exception&) { + // Non-JSON error body: keep the generic status and message. + } + return GoogleError(response.status(), status, message, service); +} + +} // namespace molo::google diff --git a/src/google/errors.h b/src/google/errors.h new file mode 100644 index 0000000..19362d9 --- /dev/null +++ b/src/google/errors.h @@ -0,0 +1,45 @@ +#pragma once + +// One error type for every Google call. Carries the HTTP status, Google's +// own status string and a message; never the credential. + +#include +#include + +#include "clients/response.h" + +namespace molo::google { + +class GoogleError : public std::runtime_error { + public: + GoogleError(int http_status, std::string status, const std::string& message, + std::string service = ""); + + int http_status() const noexcept { return http_status_; } + const std::string& status() const noexcept { return status_; } + const std::string& service() const noexcept { return service_; } + // The message alone, without the "molo::google: STATUS:" prefix of what(). + const std::string& message() const noexcept { return message_; } + + // Validation failure raised before any network traffic. + static GoogleError invalid(const std::string& message); + + private: + int http_status_; + std::string status_; + std::string service_; + std::string message_; +}; + +// Raised when no credential is configured. HTTP 503, status NOT_CONFIGURED. +class NotConfiguredError : public GoogleError { + public: + explicit NotConfiguredError(const std::string& service); +}; + +// Maps a non-2xx response (Google's {"error":{code,message,status}} body, or +// anything else) to a GoogleError. +GoogleError error_from_response(const std::string& service, + const clients::ClientResponse& response); + +} // namespace molo::google diff --git a/src/google/gemini_voice.cpp b/src/google/gemini_voice.cpp new file mode 100644 index 0000000..febdba0 --- /dev/null +++ b/src/google/gemini_voice.cpp @@ -0,0 +1,27 @@ +#include "google/gemini_voice.h" + +namespace molo::google { + +GeminiRecognizer::GeminiRecognizer(std::shared_ptr generative, + std::string model) + : generative_(std::move(generative)), model_(std::move(model)) {} + +Transcript GeminiRecognizer::recognize(const RecognitionRequest& request) const { + Transcript out; + out.text = generative_->transcribe( + model_.empty() ? Generative::default_stt_model() : model_, request.audio, + speech_encoding_mime(request.encoding), request.language); + return out; +} + +GeminiSynthesizer::GeminiSynthesizer(std::shared_ptr generative, + std::string model) + : generative_(std::move(generative)), model_(std::move(model)) {} + +Audio GeminiSynthesizer::synthesize(const SynthesisRequest& request) const { + return generative_->speak( + model_.empty() ? Generative::default_tts_model() : model_, request.text, + request.voice); +} + +} // namespace molo::google diff --git a/src/google/gemini_voice.h b/src/google/gemini_voice.h new file mode 100644 index 0000000..40c9a97 --- /dev/null +++ b/src/google/gemini_voice.h @@ -0,0 +1,37 @@ +#pragma once + +// Recognizer and Synthesizer over Gemini: audio understanding for speech to +// text, Gemini TTS models for text to speech. Works with any credential the +// Gemini endpoint accepts (AI Studio key, Vertex express key, OAuth). + +#include +#include + +#include "google/generative.h" +#include "google/voice_io.h" + +namespace molo::google { + +class GeminiRecognizer : public Recognizer { + public: + explicit GeminiRecognizer(std::shared_ptr generative, + std::string model = ""); + Transcript recognize(const RecognitionRequest& request) const override; + + private: + std::shared_ptr generative_; + std::string model_; +}; + +class GeminiSynthesizer : public Synthesizer { + public: + explicit GeminiSynthesizer(std::shared_ptr generative, + std::string model = ""); + Audio synthesize(const SynthesisRequest& request) const override; + + private: + std::shared_ptr generative_; + std::string model_; +}; + +} // namespace molo::google diff --git a/src/google/generative.cpp b/src/google/generative.cpp new file mode 100644 index 0000000..4839af0 --- /dev/null +++ b/src/google/generative.cpp @@ -0,0 +1,209 @@ +#include "google/generative.h" + +#include "google/caps.h" +#include "google/credentials.h" +#include "google/endpoints.h" +#include "google/wav.h" +#include "security/crypto/Base64.h" + +namespace molo::google { + +using clients::body::Json; + +Generative::Generative(std::shared_ptr client, + ServiceEndpoint endpoint) + : Service(std::move(client), std::move(endpoint)) {} + +std::string Generative::default_model() { + std::string m = Credentials::env("GOOGLE_GENAI_MODEL"); + return m.empty() ? "gemini-3.5-flash" : m; +} + +std::string Generative::default_stt_model() { + std::string m = Credentials::env("GOOGLE_STT_MODEL"); + return m.empty() ? default_model() : m; +} + +std::string Generative::default_tts_model() { + std::string m = Credentials::env("GOOGLE_TTS_MODEL"); + return m.empty() ? "gemini-2.5-flash-tts" : m; +} + +Json Generative::generate_content(const std::string& model, + const Json& body) const { + return call("/models/" + model + ":generateContent", body); +} + +std::string Generative::generate(const GenerationRequest& request) const { + if (request.messages.empty()) throw GoogleError::invalid("no messages"); + if (request.max_output_tokens < 1 || + request.max_output_tokens > Caps::kMaxOutputTokens) + throw GoogleError::invalid("max_output_tokens outside 1.." + + std::to_string(Caps::kMaxOutputTokens)); + + Json contents = Json::array(); + for (const ChatMessage& m : request.messages) { + if (m.role != "user" && m.role != "model") + throw GoogleError::invalid("role must be user or model"); + if (m.text.size() > Caps::kMaxMessageChars) + throw GoogleError::invalid("message over " + + std::to_string(Caps::kMaxMessageChars) + + " characters"); + Json part = Json::object(); + part.set("text", Json(m.text)); + Json parts = Json::array(); + parts.push(part); + Json content = Json::object(); + content.set("role", Json(m.role)); + content.set("parts", parts); + contents.push(content); + } + Json gen = Json::object(); + gen.set("maxOutputTokens", Json(request.max_output_tokens)); + gen.set("temperature", Json(request.temperature)); + if (request.thinking_budget >= 0) { + Json thinking = Json::object(); + thinking.set("thinkingBudget", Json(request.thinking_budget)); + gen.set("thinkingConfig", thinking); + } + Json body = Json::object(); + body.set("contents", contents); + body.set("generationConfig", gen); + if (!request.system.empty()) { + Json part = Json::object(); + part.set("text", Json(request.system)); + Json parts = Json::array(); + parts.push(part); + Json sys = Json::object(); + sys.set("parts", parts); + body.set("systemInstruction", sys); + } + + std::string model = request.model.empty() ? default_model() : request.model; + Json reply = generate_content(model, body); + + std::string text; + const Json* candidates = reply.find("candidates"); + if (candidates && candidates->is_array() && !candidates->as_array().empty()) { + const Json* content = candidates->as_array().front().find("content"); + const Json* parts = content ? content->find("parts") : nullptr; + if (parts && parts->is_array()) + for (const Json& p : parts->as_array()) text += string_at(p, "text"); + } + if (text.empty()) + throw GoogleError(502, "EMPTY_RESPONSE", "model returned no text", + endpoint_.name); + return text; +} + +std::string Generative::transcribe(const std::string& model, + const std::string& audio, + const std::string& mime, + const std::string& language) const { + if (audio.empty()) throw GoogleError::invalid("audio is empty"); + if (audio.size() > Caps::kMaxSttBytes) + throw GoogleError::invalid("audio over " + + std::to_string(Caps::kMaxSttBytes) + " bytes"); + Json inline_data = Json::object(); + inline_data.set("mimeType", Json(mime)); + inline_data.set("data", Json(security::crypto::Base64::encode(audio))); + Json audio_part = Json::object(); + audio_part.set("inlineData", inline_data); + Json text_part = Json::object(); + text_part.set("text", + Json("Transcribe the speech in this audio verbatim. The " + "language is probably " + language + ". Output only the " + "transcript. If there is no intelligible speech, output " + "nothing at all.")); + Json parts = Json::array(); + parts.push(audio_part); + parts.push(text_part); + Json content = Json::object(); + content.set("role", Json("user")); + content.set("parts", parts); + Json contents = Json::array(); + contents.push(content); + Json gen = Json::object(); + gen.set("temperature", Json(0.0)); + Json no_thinking = Json::object(); + no_thinking.set("thinkingBudget", Json(0)); + gen.set("thinkingConfig", no_thinking); + Json body = Json::object(); + body.set("contents", contents); + body.set("generationConfig", gen); + + Json reply = generate_content(model, body); + std::string text; + const Json* candidates = reply.find("candidates"); + if (candidates && candidates->is_array() && !candidates->as_array().empty()) { + const Json* c = candidates->as_array().front().find("content"); + const Json* p = c ? c->find("parts") : nullptr; + if (p && p->is_array()) + for (const Json& part : p->as_array()) text += string_at(part, "text"); + } + std::size_t a = text.find_first_not_of(" \t\r\n"); + std::size_t b = text.find_last_not_of(" \t\r\n"); + return a == std::string::npos ? "" : text.substr(a, b - a + 1); +} + +Audio Generative::speak(const std::string& model, const std::string& text, + const std::string& voice) const { + if (text.empty()) throw GoogleError::invalid("text is empty"); + if (text.size() > Caps::kMaxTtsChars) + throw GoogleError::invalid("text over " + + std::to_string(Caps::kMaxTtsChars) + " characters"); + Json part = Json::object(); + part.set("text", Json(text)); + Json parts = Json::array(); + parts.push(part); + Json content = Json::object(); + content.set("role", Json("user")); + content.set("parts", parts); + Json contents = Json::array(); + contents.push(content); + Json prebuilt = Json::object(); + prebuilt.set("voiceName", Json(voice.empty() ? kDefaultTtsVoice : voice)); + Json voice_config = Json::object(); + voice_config.set("prebuiltVoiceConfig", prebuilt); + Json speech = Json::object(); + speech.set("voiceConfig", voice_config); + Json modalities = Json::array(); + modalities.push(Json("AUDIO")); + Json gen = Json::object(); + gen.set("responseModalities", modalities); + gen.set("speechConfig", speech); + Json body = Json::object(); + body.set("contents", contents); + body.set("generationConfig", gen); + + Json reply = generate_content(model, body); + const Json* candidates = reply.find("candidates"); + if (candidates && candidates->is_array() && !candidates->as_array().empty()) { + const Json* c = candidates->as_array().front().find("content"); + const Json* p = c ? c->find("parts") : nullptr; + if (p && p->is_array()) + for (const Json& pt : p->as_array()) { + const Json* data = pt.find("inlineData"); + if (!data) continue; + std::string pcm; + if (!security::crypto::Base64::decode(string_at(*data, "data"), pcm) || + pcm.empty()) + continue; + std::string mime = string_at(*data, "mimeType"); + Audio out; + if (mime.find("L16") != std::string::npos || + mime.find("pcm") != std::string::npos) { + out.bytes = pcm16_to_wav(pcm, sample_rate_from_mime(mime)); + out.mime = "audio/wav"; + } else { + out.bytes = std::move(pcm); + out.mime = mime.empty() ? "audio/wav" : mime; + } + return out; + } + } + throw GoogleError(502, "EMPTY_RESPONSE", "model returned no audio", + endpoint_.name); +} + +} // namespace molo::google diff --git a/src/google/generative.h b/src/google/generative.h new file mode 100644 index 0000000..8128ec0 --- /dev/null +++ b/src/google/generative.h @@ -0,0 +1,54 @@ +#pragma once + +// Generative Language API (Gemini): models/{model}:generateContent. + +#include +#include + +#include "google/endpoints.h" +#include "google/service.h" +#include "google/voice_io.h" + +namespace molo::google { + +struct ChatMessage { + std::string role; // "user" or "model" + std::string text; +}; + +struct GenerationRequest { + std::string model; // Empty: GOOGLE_GENAI_MODEL or the built-in default. + std::string system; + std::vector messages; + int max_output_tokens = 300; + double temperature = 0.7; + // Reasoning tokens count against max_output_tokens on thinking models and + // can truncate a spoken reply. -1: model default; 0: no thinking. + int thinking_budget = -1; +}; + +class Generative : public Service { + public: + explicit Generative(std::shared_ptr client, + ServiceEndpoint endpoint = endpoints::generative()); + + static std::string default_model(); + static std::string default_stt_model(); // GOOGLE_STT_MODEL, else default_model(). + static std::string default_tts_model(); // GOOGLE_TTS_MODEL, else gemini-2.5-flash-tts. + static constexpr const char* kDefaultTtsVoice = "Kore"; + + std::string generate(const GenerationRequest& request) const; + // Audio in, text out (Gemini audio understanding). Empty string: no speech. + std::string transcribe(const std::string& model, const std::string& audio, + const std::string& mime, + const std::string& language) const; + // Text in, spoken WAV out (Gemini TTS models return raw 24 kHz PCM). + Audio speak(const std::string& model, const std::string& text, + const std::string& voice) const; + + private: + clients::body::Json generate_content(const std::string& model, + const clients::body::Json& body) const; +}; + +} // namespace molo::google diff --git a/src/google/google.mk b/src/google/google.mk new file mode 100644 index 0000000..4323b88 --- /dev/null +++ b/src/google/google.mk @@ -0,0 +1,50 @@ +# Per-part fragment for the Google Cloud APIs (owned dir: src/google). +# Included by the root Makefile, or run standalone: +# make -f src/google/google.mk [google|google-test|google-clean] +# Root wiring: `all` depends on `google`, `test` on `google-test`, +# `clean` on `google-clean`. + +ROOT ?= . +CXX ?= g++ +GOOGLE_CXXFLAGS ?= -std=c++23 -Wall -Wextra -Werror -DMOLO_HAS_WOLFSSL +CLIENTS_LIB ?= $(ROOT)/build/clients/libclients.a +OPERABILITY_LIB ?= build/liboperability.a +GOOGLE_INCS := -I$(ROOT)/src -I$(ROOT)/tests/src/clients + +GOOGLE_SRCS := $(sort $(wildcard $(ROOT)/src/google/*.cpp)) \ + $(ROOT)/src/security/crypto/Base64.cpp +GOOGLE_TEST_SRCS := $(sort $(wildcard $(ROOT)/tests/src/google/*.cpp)) + +GOOGLE_BUILD := $(ROOT)/build/google +GOOGLE_OBJDIR := $(GOOGLE_BUILD)/objs +GOOGLE_LIB := $(GOOGLE_BUILD)/libgoogle.a +GOOGLE_TEST_BIN := $(ROOT)/build/google_test + +GOOGLE_OBJS := $(patsubst $(ROOT)/%.cpp,$(GOOGLE_OBJDIR)/%.o,$(GOOGLE_SRCS)) + +# Header changes must rebuild dependents (a stale object with an old class +# layout corrupts the heap). +-include $(GOOGLE_OBJS:.o=.d) + +.PHONY: google google-test google-clean + +google: $(GOOGLE_LIB) + +$(GOOGLE_OBJDIR)/%.o: $(ROOT)/%.cpp + @mkdir -p $(dir $@) + $(CXX) $(GOOGLE_CXXFLAGS) -MMD -MP -I$(ROOT)/src -c $< -o $@ + +$(GOOGLE_LIB): $(GOOGLE_OBJS) + @mkdir -p $(dir $@) + $(AR) rcs $@ $^ + +# The clients test harness (framework.h plus main.cpp) is reused, not copied. +google-test: $(GOOGLE_TEST_BIN) + $(GOOGLE_TEST_BIN) + +$(GOOGLE_TEST_BIN): $(GOOGLE_OBJS) $(GOOGLE_TEST_SRCS) $(CLIENTS_LIB) $(OPERABILITY_LIB) $(ROOT)/tests/src/clients/main.cpp + @mkdir -p $(dir $@) + $(CXX) $(GOOGLE_CXXFLAGS) $(GOOGLE_INCS) $(GOOGLE_OBJS) $(GOOGLE_TEST_SRCS) $(ROOT)/tests/src/clients/main.cpp $(CLIENTS_LIB) $(OPERABILITY_LIB) -lwolfssl -o $@ + +google-clean: + rm -rf $(GOOGLE_BUILD) $(GOOGLE_TEST_BIN) diff --git a/src/google/service.cpp b/src/google/service.cpp new file mode 100644 index 0000000..0839248 --- /dev/null +++ b/src/google/service.cpp @@ -0,0 +1,30 @@ +#include "google/service.h" + +namespace molo::google { + +Service::Service(std::shared_ptr client, ServiceEndpoint endpoint) + : client_(std::move(client)), endpoint_(std::move(endpoint)) {} + +clients::body::Json Service::call(const std::string& path, + const clients::body::Json& body) const { + return client_->post_json(endpoint_, path, body); +} + +clients::body::Json Service::fetch(const std::string& path) const { + return client_->get_json(endpoint_, path); +} + +std::string Service::string_at(const clients::body::Json& doc, + const std::string& key, + const std::string& fallback) { + const auto* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_string() ? v->as_string() : fallback; +} + +double Service::number_at(const clients::body::Json& doc, + const std::string& key, double fallback) { + const auto* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_number() ? v->as_number() : fallback; +} + +} // namespace molo::google diff --git a/src/google/service.h b/src/google/service.h new file mode 100644 index 0000000..6b51424 --- /dev/null +++ b/src/google/service.h @@ -0,0 +1,38 @@ +#pragma once + +// Base class for every concrete Google service. A subclass owns its request +// and response shapes; transport, auth and errors live in GoogleClient. + +#include +#include + +#include "clients/body/json.h" +#include "google/client.h" + +namespace molo::google { + +class Service { + public: + Service(std::shared_ptr client, ServiceEndpoint endpoint); + virtual ~Service() = default; + + const ServiceEndpoint& endpoint() const { return endpoint_; } + bool configured() const { return client_->configured(); } + + protected: + clients::body::Json call(const std::string& path, + const clients::body::Json& body) const; + clients::body::Json fetch(const std::string& path) const; + + // Tolerant readers for response documents. + static std::string string_at(const clients::body::Json& doc, + const std::string& key, + const std::string& fallback = ""); + static double number_at(const clients::body::Json& doc, + const std::string& key, double fallback = 0.0); + + std::shared_ptr client_; + ServiceEndpoint endpoint_; +}; + +} // namespace molo::google diff --git a/src/google/speech_to_text.cpp b/src/google/speech_to_text.cpp new file mode 100644 index 0000000..082089e --- /dev/null +++ b/src/google/speech_to_text.cpp @@ -0,0 +1,51 @@ +#include "google/speech_to_text.h" + +#include "google/caps.h" +#include "google/endpoints.h" +#include "security/crypto/Base64.h" + +namespace molo::google { + +using clients::body::Json; + +SpeechToText::SpeechToText(std::shared_ptr client) + : Service(std::move(client), endpoints::speech_to_text()) {} + +Transcript SpeechToText::recognize(const RecognitionRequest& request) const { + if (request.audio.empty()) throw GoogleError::invalid("audio is empty"); + if (request.audio.size() > Caps::kMaxSttBytes) + throw GoogleError::invalid("audio over " + + std::to_string(Caps::kMaxSttBytes) + " bytes"); + + Json config = Json::object(); + config.set("languageCode", Json(request.language)); + config.set("encoding", Json(speech_encoding_name(request.encoding))); + config.set("enableAutomaticPunctuation", Json(request.punctuation)); + if (request.sample_rate_hz > 0) + config.set("sampleRateHertz", Json(request.sample_rate_hz)); + Json audio = Json::object(); + audio.set("content", Json(security::crypto::Base64::encode(request.audio))); + Json body = Json::object(); + body.set("config", config); + body.set("audio", audio); + + Json reply = call("/speech:recognize", body); + Transcript out; + const Json* results = reply.find("results"); + if (!results || !results->is_array()) return out; + bool first = true; + for (const Json& r : results->as_array()) { + const Json* alts = r.find("alternatives"); + if (!alts || !alts->is_array() || alts->as_array().empty()) continue; + const Json& best = alts->as_array().front(); + std::string piece = string_at(best, "transcript"); + if (piece.empty()) continue; + if (!out.text.empty()) out.text += " "; + out.text += piece; + if (first) out.confidence = number_at(best, "confidence"); + first = false; + } + return out; +} + +} // namespace molo::google diff --git a/src/google/speech_to_text.h b/src/google/speech_to_text.h new file mode 100644 index 0000000..9794e07 --- /dev/null +++ b/src/google/speech_to_text.h @@ -0,0 +1,19 @@ +#pragma once + +// Cloud Speech-to-Text: speech:recognize (synchronous, up to ~1 minute). + +#include + +#include "google/service.h" +#include "google/voice_io.h" + +namespace molo::google { + +class SpeechToText : public Service, public Recognizer { + public: + explicit SpeechToText(std::shared_ptr client); + + Transcript recognize(const RecognitionRequest& request) const override; +}; + +} // namespace molo::google diff --git a/src/google/text_to_speech.cpp b/src/google/text_to_speech.cpp new file mode 100644 index 0000000..54558a0 --- /dev/null +++ b/src/google/text_to_speech.cpp @@ -0,0 +1,65 @@ +#include "google/text_to_speech.h" + +#include "google/caps.h" +#include "google/endpoints.h" +#include "security/crypto/Base64.h" + +namespace molo::google { + +using clients::body::Json; + +TextToSpeech::TextToSpeech(std::shared_ptr client) + : Service(std::move(client), endpoints::text_to_speech()) {} + +Audio TextToSpeech::synthesize(const SynthesisRequest& request) const { + if (request.text.empty()) throw GoogleError::invalid("text is empty"); + if (request.text.size() > Caps::kMaxTtsChars) + throw GoogleError::invalid("text over " + + std::to_string(Caps::kMaxTtsChars) + " characters"); + if (request.speaking_rate < 0.25 || request.speaking_rate > 4.0) + throw GoogleError::invalid("speaking_rate outside 0.25..4.0"); + + Json input = Json::object(); + input.set(request.ssml ? "ssml" : "text", Json(request.text)); + Json voice = Json::object(); + voice.set("languageCode", Json(request.language)); + if (!request.voice.empty()) voice.set("name", Json(request.voice)); + Json audio_config = Json::object(); + audio_config.set("audioEncoding", Json(audio_encoding_name(request.encoding))); + audio_config.set("speakingRate", Json(request.speaking_rate)); + Json body = Json::object(); + body.set("input", input); + body.set("voice", voice); + body.set("audioConfig", audio_config); + + Json reply = call("/text:synthesize", body); + Audio out; + out.mime = audio_encoding_mime(request.encoding); + if (!security::crypto::Base64::decode(string_at(reply, "audioContent"), + out.bytes) || + out.bytes.empty()) + throw GoogleError(502, "BAD_RESPONSE", "missing or invalid audioContent", + endpoint_.name); + return out; +} + +std::vector TextToSpeech::voices(const std::string& language) const { + Json reply = fetch(language.empty() ? "/voices" + : "/voices?languageCode=" + language); + std::vector out; + const Json* list = reply.find("voices"); + if (!list || !list->is_array()) return out; + for (const Json& v : list->as_array()) { + Voice voice; + voice.name = string_at(v, "name"); + voice.gender = string_at(v, "ssmlGender"); + if (const Json* codes = v.find("languageCodes"); + codes && codes->is_array()) + for (const Json& c : codes->as_array()) + if (c.is_string()) voice.languages.push_back(c.as_string()); + out.push_back(std::move(voice)); + } + return out; +} + +} // namespace molo::google diff --git a/src/google/text_to_speech.h b/src/google/text_to_speech.h new file mode 100644 index 0000000..f3e37cf --- /dev/null +++ b/src/google/text_to_speech.h @@ -0,0 +1,27 @@ +#pragma once + +// Cloud Text-to-Speech: text:synthesize and voices.list. + +#include +#include + +#include "google/service.h" +#include "google/voice_io.h" + +namespace molo::google { + +struct Voice { + std::string name; + std::string gender; + std::vector languages; +}; + +class TextToSpeech : public Service, public Synthesizer { + public: + explicit TextToSpeech(std::shared_ptr client); + + Audio synthesize(const SynthesisRequest& request) const override; + std::vector voices(const std::string& language = "") const; +}; + +} // namespace molo::google diff --git a/src/google/voice.cpp b/src/google/voice.cpp new file mode 100644 index 0000000..6179c0b --- /dev/null +++ b/src/google/voice.cpp @@ -0,0 +1,144 @@ +#include "google/voice.h" + +#include "google/caps.h" + +namespace molo::google { + +GenerativeResponder::GenerativeResponder(std::shared_ptr generative, + std::string system) + : generative_(std::move(generative)), system_(std::move(system)) { + if (system_.empty()) + system_ = + "You are a friendly voice assistant in a live spoken conversation. " + "Answer in the language of the user, in one to three short sentences, " + "plain text only: no markdown, lists, or emoji."; +} + +std::string GenerativeResponder::reply(const std::vector& history, + const std::string& language) { + GenerationRequest request; + request.system = system_ + " Conversation language code: " + language + "."; + request.messages = history; + request.thinking_budget = 0; // Thinking would eat the spoken reply's budget. + return generative_->generate(request); +} + +bool VoiceSessions::valid_id(const std::string& id) { + if (id.empty() || id.size() > Caps::kMaxConversationId) return false; + for (char c : id) { + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '-' || c == '_'; + if (!ok) return false; + } + return true; +} + +std::vector VoiceSessions::append(const std::string& id, + ChatMessage message) { + if (!valid_id(id)) throw GoogleError::invalid("bad conversation id"); + std::lock_guard lock(mutex_); + auto it = entries_.find(id); + if (it == entries_.end()) { + if (entries_.size() >= Caps::kMaxConversations) { + entries_.erase(order_.front()); + order_.pop_front(); + } + order_.push_back(id); + Entry entry; + entry.order = std::prev(order_.end()); + it = entries_.emplace(id, std::move(entry)).first; + } else { + order_.splice(order_.end(), order_, it->second.order); + } + auto& history = it->second.history; + history.push_back(std::move(message)); + while (history.size() > Caps::kMaxHistoryTurns * 2) history.pop_front(); + // Gemini requires the first message to be from the user. + while (!history.empty() && history.front().role != "user") + history.pop_front(); + return std::vector(history.begin(), history.end()); +} + +void VoiceSessions::drop_last(const std::string& id) { + std::lock_guard lock(mutex_); + auto it = entries_.find(id); + if (it != entries_.end() && !it->second.history.empty()) + it->second.history.pop_back(); +} + +void VoiceSessions::reset(const std::string& id) { + std::lock_guard lock(mutex_); + auto it = entries_.find(id); + if (it == entries_.end()) return; + order_.erase(it->second.order); + entries_.erase(it); +} + +std::size_t VoiceSessions::size() const { + std::lock_guard lock(mutex_); + return entries_.size(); +} + +std::size_t VoiceSessions::history_size(const std::string& id) const { + std::lock_guard lock(mutex_); + auto it = entries_.find(id); + return it == entries_.end() ? 0 : it->second.history.size(); +} + +VoicePipeline::VoicePipeline(std::shared_ptr stt, + std::shared_ptr responder, + std::shared_ptr tts, + std::shared_ptr sessions) + : stt_(std::move(stt)), + responder_(std::move(responder)), + tts_(std::move(tts)), + sessions_(std::move(sessions)) {} + +VoiceTurn VoicePipeline::turn(const std::string& id, + const RecognitionRequest& heard, + const std::string& voice) { + if (!VoiceSessions::valid_id(id)) + throw GoogleError::invalid("bad conversation id"); + Transcript transcript = stt_->recognize(heard); + if (transcript.text.empty()) { + VoiceTurn silent; + silent.language = heard.language; + return silent; + } + return respond(id, transcript.text, heard.language, voice); +} + +VoiceTurn VoicePipeline::turn_text(const std::string& id, + const std::string& text, + const std::string& language, + const std::string& voice) { + if (!VoiceSessions::valid_id(id)) + throw GoogleError::invalid("bad conversation id"); + if (text.empty()) throw GoogleError::invalid("text is empty"); + return respond(id, text, language, voice); +} + +VoiceTurn VoicePipeline::respond(const std::string& id, const std::string& text, + const std::string& language, + const std::string& voice) { + VoiceTurn out; + out.transcript = text; + out.language = language; + std::string clipped = text.substr(0, Caps::kMaxMessageChars); + auto history = sessions_->append(id, {"user", clipped}); + try { + out.reply = responder_->reply(history, language); + SynthesisRequest speak; + speak.text = out.reply.substr(0, Caps::kMaxTtsChars); + speak.language = language; + speak.voice = voice; + out.audio = tts_->synthesize(speak); + } catch (...) { + sessions_->drop_last(id); // A failed turn leaves no half conversation. + throw; + } + sessions_->append(id, {"model", out.reply.substr(0, Caps::kMaxMessageChars)}); + return out; +} + +} // namespace molo::google diff --git a/src/google/voice.h b/src/google/voice.h new file mode 100644 index 0000000..9838148 --- /dev/null +++ b/src/google/voice.h @@ -0,0 +1,100 @@ +#pragma once + +// Voice conversation: recognize -> respond -> synthesize, with bounded +// per-conversation history. Responder is the seam: Gemini today, anything +// else tomorrow. + +#include +#include +#include +#include +#include +#include +#include + +#include "google/generative.h" +#include "google/speech_to_text.h" +#include "google/text_to_speech.h" +#include "google/voice_io.h" + +namespace molo::google { + +class Responder { + public: + virtual ~Responder() = default; + // history ends with the new user message. + virtual std::string reply(const std::vector& history, + const std::string& language) = 0; +}; + +class GenerativeResponder : public Responder { + public: + explicit GenerativeResponder(std::shared_ptr generative, + std::string system = ""); + std::string reply(const std::vector& history, + const std::string& language) override; + + private: + std::shared_ptr generative_; + std::string system_; +}; + +// Bounded conversation memory. Both history length and conversation count +// are capped; oldest entries are evicted first. +class VoiceSessions { + public: + static bool valid_id(const std::string& id); + + // Appends and returns the full history snapshot (including this message). + std::vector append(const std::string& id, ChatMessage message); + // Removes the last message (used to roll back a failed turn). + void drop_last(const std::string& id); + void reset(const std::string& id); + std::size_t size() const; + std::size_t history_size(const std::string& id) const; + + private: + struct Entry { + std::deque history; + std::list::iterator order; + }; + mutable std::mutex mutex_; + std::map entries_; + std::list order_; // Front is oldest. +}; + +struct VoiceTurn { + std::string transcript; // Empty: nothing was heard. + std::string reply; + Audio audio; + std::string language; +}; + +class VoicePipeline { + public: + VoicePipeline(std::shared_ptr stt, + std::shared_ptr responder, + std::shared_ptr tts, + std::shared_ptr sessions); + + // Full turn from audio. + VoiceTurn turn(const std::string& conversation_id, + const RecognitionRequest& heard, const std::string& voice = ""); + // Text turn (typed instead of spoken): respond and synthesize. + VoiceTurn turn_text(const std::string& conversation_id, + const std::string& text, const std::string& language, + const std::string& voice = ""); + + VoiceSessions& sessions() { return *sessions_; } + + private: + VoiceTurn respond(const std::string& id, const std::string& text, + const std::string& language, const std::string& voice); + + std::shared_ptr stt_; + std::shared_ptr responder_; + std::shared_ptr tts_; + std::shared_ptr sessions_; +}; + +} // namespace molo::google diff --git a/src/google/voice_backend.cpp b/src/google/voice_backend.cpp new file mode 100644 index 0000000..f5c6fd7 --- /dev/null +++ b/src/google/voice_backend.cpp @@ -0,0 +1,46 @@ +#include "google/voice_backend.h" + +#include "google/credentials.h" +#include "google/gemini_voice.h" + +namespace molo::google { + +const char* voice_backend_name(VoiceBackend backend) { + switch (backend) { + case VoiceBackend::kCloud: return "cloud"; + case VoiceBackend::kGemini: return "gemini"; + case VoiceBackend::kVertex: return "vertex"; + } + return "cloud"; +} + +VoiceBackend voice_backend_from_env() { + std::string pick = Credentials::env("GOOGLE_API_BACKEND"); + if (pick == "cloud") return VoiceBackend::kCloud; + if (pick == "gemini") return VoiceBackend::kGemini; + if (pick == "vertex") return VoiceBackend::kVertex; + return Credentials::env("GOOGLE_API_KEY").rfind("AQ.", 0) == 0 + ? VoiceBackend::kVertex + : VoiceBackend::kCloud; +} + +VoiceParts make_voice_parts(std::shared_ptr client, + VoiceBackend backend) { + VoiceParts parts; + if (backend == VoiceBackend::kCloud) { + auto generative = std::make_shared(client); + parts.recognizer = std::make_shared(client); + parts.synthesizer = std::make_shared(client); + parts.responder = std::make_shared(generative); + return parts; + } + auto generative = std::make_shared( + client, backend == VoiceBackend::kVertex ? endpoints::vertex_generative() + : endpoints::generative()); + parts.recognizer = std::make_shared(generative); + parts.synthesizer = std::make_shared(generative); + parts.responder = std::make_shared(generative); + return parts; +} + +} // namespace molo::google diff --git a/src/google/voice_backend.h b/src/google/voice_backend.h new file mode 100644 index 0000000..eab1f41 --- /dev/null +++ b/src/google/voice_backend.h @@ -0,0 +1,33 @@ +#pragma once + +// Picks the three voice parts for a credential. One switch, three backends: +// kCloud classic Cloud Text-to-Speech + Speech-to-Text (need OAuth or an +// old-style key) with Gemini on generativelanguage for replies +// kGemini Gemini audio in/out on generativelanguage (AI Studio key) +// kVertex Gemini audio in/out on Vertex AI (express key, "AQ." prefix) + +#include +#include + +#include "google/client.h" +#include "google/voice.h" + +namespace molo::google { + +enum class VoiceBackend { kCloud, kGemini, kVertex }; + +const char* voice_backend_name(VoiceBackend backend); +// GOOGLE_API_BACKEND (cloud|gemini|vertex) wins; else keys starting with +// "AQ." are Vertex express keys; everything else is classic cloud. +VoiceBackend voice_backend_from_env(); + +struct VoiceParts { + std::shared_ptr recognizer; + std::shared_ptr responder; + std::shared_ptr synthesizer; +}; + +VoiceParts make_voice_parts(std::shared_ptr client, + VoiceBackend backend); + +} // namespace molo::google diff --git a/src/google/voice_io.cpp b/src/google/voice_io.cpp new file mode 100644 index 0000000..7e9e21d --- /dev/null +++ b/src/google/voice_io.cpp @@ -0,0 +1,56 @@ +#include "google/voice_io.h" + +namespace molo::google { + +const char* audio_encoding_name(AudioEncoding encoding) { + switch (encoding) { + case AudioEncoding::kMp3: return "MP3"; + case AudioEncoding::kLinear16: return "LINEAR16"; + case AudioEncoding::kOggOpus: return "OGG_OPUS"; + } + return "MP3"; +} + +const char* audio_encoding_mime(AudioEncoding encoding) { + switch (encoding) { + case AudioEncoding::kMp3: return "audio/mpeg"; + case AudioEncoding::kLinear16: return "audio/wav"; + case AudioEncoding::kOggOpus: return "audio/ogg"; + } + return "audio/mpeg"; +} + +const char* speech_encoding_name(SpeechEncoding encoding) { + switch (encoding) { + case SpeechEncoding::kWebmOpus: return "WEBM_OPUS"; + case SpeechEncoding::kOggOpus: return "OGG_OPUS"; + case SpeechEncoding::kFlac: return "FLAC"; + case SpeechEncoding::kLinear16: return "LINEAR16"; + case SpeechEncoding::kMp3: return "MP3"; + } + return "WEBM_OPUS"; +} + +const char* speech_encoding_mime(SpeechEncoding encoding) { + switch (encoding) { + case SpeechEncoding::kWebmOpus: return "audio/webm"; + case SpeechEncoding::kOggOpus: return "audio/ogg"; + case SpeechEncoding::kFlac: return "audio/flac"; + case SpeechEncoding::kLinear16: return "audio/wav"; + case SpeechEncoding::kMp3: return "audio/mpeg"; + } + return "audio/webm"; +} + +bool speech_encoding_from_mime(const std::string& mime, SpeechEncoding& out) { + auto has = [&](const char* s) { return mime.find(s) != std::string::npos; }; + if (has("webm")) out = SpeechEncoding::kWebmOpus; + else if (has("ogg")) out = SpeechEncoding::kOggOpus; + else if (has("flac")) out = SpeechEncoding::kFlac; + else if (has("wav") || has("l16") || has("pcm")) out = SpeechEncoding::kLinear16; + else if (has("mpeg") || has("mp3")) out = SpeechEncoding::kMp3; + else return false; + return true; +} + +} // namespace molo::google diff --git a/src/google/voice_io.h b/src/google/voice_io.h new file mode 100644 index 0000000..cacb36a --- /dev/null +++ b/src/google/voice_io.h @@ -0,0 +1,63 @@ +#pragma once + +// The two speech seams of a voice pipeline. Backends implement them: +// classic Cloud TTS/STT (TextToSpeech, SpeechToText) or Gemini audio +// (GeminiSynthesizer, GeminiRecognizer). Callers never know which. + +#include + +namespace molo::google { + +enum class AudioEncoding { kMp3, kLinear16, kOggOpus }; + +const char* audio_encoding_name(AudioEncoding encoding); // Google enum name. +const char* audio_encoding_mime(AudioEncoding encoding); + +struct SynthesisRequest { + std::string text; + bool ssml = false; // Classic Cloud TTS only. + std::string language = "nl-NL"; + std::string voice; // Empty: backend default. + AudioEncoding encoding = AudioEncoding::kMp3; // Classic Cloud TTS only. + double speaking_rate = 1.0; // 0.25 .. 4.0, classic Cloud TTS only. +}; + +struct Audio { + std::string bytes; + std::string mime; +}; + +enum class SpeechEncoding { kWebmOpus, kOggOpus, kFlac, kLinear16, kMp3 }; + +const char* speech_encoding_name(SpeechEncoding encoding); // Google enum name. +const char* speech_encoding_mime(SpeechEncoding encoding); // Content type. +// Maps a browser/content type ("audio/webm;codecs=opus", "audio/ogg", ...) +// to an encoding. Returns false when unknown. +bool speech_encoding_from_mime(const std::string& mime, SpeechEncoding& out); + +struct RecognitionRequest { + std::string audio; // Raw bytes. + SpeechEncoding encoding = SpeechEncoding::kWebmOpus; + std::string language = "nl-NL"; + int sample_rate_hz = 0; // 0: taken from the container header. + bool punctuation = true; +}; + +struct Transcript { + std::string text; + double confidence = 0.0; // 0 when the backend does not report one. +}; + +class Synthesizer { + public: + virtual ~Synthesizer() = default; + virtual Audio synthesize(const SynthesisRequest& request) const = 0; +}; + +class Recognizer { + public: + virtual ~Recognizer() = default; + virtual Transcript recognize(const RecognitionRequest& request) const = 0; +}; + +} // namespace molo::google diff --git a/src/google/wav.cpp b/src/google/wav.cpp new file mode 100644 index 0000000..f4cabc8 --- /dev/null +++ b/src/google/wav.cpp @@ -0,0 +1,46 @@ +#include "google/wav.h" + +#include + +namespace molo::google { + +namespace { +void put16(std::string& out, std::uint16_t v) { + out.push_back(static_cast(v & 0xff)); + out.push_back(static_cast((v >> 8) & 0xff)); +} +void put32(std::string& out, std::uint32_t v) { + for (int i = 0; i < 4; ++i) out.push_back(static_cast((v >> (8 * i)) & 0xff)); +} +} // namespace + +std::string pcm16_to_wav(const std::string& pcm, int sample_rate_hz, + int channels) { + std::string out; + out.reserve(44 + pcm.size()); + out += "RIFF"; + put32(out, static_cast(36 + pcm.size())); + out += "WAVEfmt "; + put32(out, 16); + put16(out, 1); // PCM + put16(out, static_cast(channels)); + put32(out, static_cast(sample_rate_hz)); + put32(out, static_cast(sample_rate_hz * channels * 2)); + put16(out, static_cast(channels * 2)); + put16(out, 16); + out += "data"; + put32(out, static_cast(pcm.size())); + out += pcm; + return out; +} + +int sample_rate_from_mime(const std::string& mime, int fallback) { + std::size_t pos = mime.find("rate="); + if (pos == std::string::npos) return fallback; + int rate = 0; + for (std::size_t i = pos + 5; i < mime.size() && mime[i] >= '0' && mime[i] <= '9'; ++i) + rate = rate * 10 + (mime[i] - '0'); + return rate >= 8000 && rate <= 96000 ? rate : fallback; +} + +} // namespace molo::google diff --git a/src/google/wav.h b/src/google/wav.h new file mode 100644 index 0000000..2394282 --- /dev/null +++ b/src/google/wav.h @@ -0,0 +1,17 @@ +#pragma once + +// Gemini TTS returns raw 16-bit PCM; browsers need a container. + +#include + +namespace molo::google { + +// Wraps little-endian mono/stereo 16-bit PCM in a RIFF/WAVE header. +std::string pcm16_to_wav(const std::string& pcm, int sample_rate_hz, + int channels = 1); + +// Sample rate from a mime like "audio/L16;codec=pcm;rate=24000"; fallback +// when absent. +int sample_rate_from_mime(const std::string& mime, int fallback = 24000); + +} // namespace molo::google diff --git a/tests/example/google/status_test.cpp b/tests/example/google/status_test.cpp new file mode 100644 index 0000000..d50ea38 --- /dev/null +++ b/tests/example/google/status_test.cpp @@ -0,0 +1,32 @@ +// Route: GET /google/status (credential state and known Google services) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + { + GoogleRig rig; + Fixture f(rig.client); + auto res = f.client->get("/google/status"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "application/json"), "json"); + check(contains(res.body, "\"configured\":true"), "configured with a key"); + check(contains(res.body, "\"name\":\"texttospeech\"") && + contains(res.body, "\"name\":\"speech\"") && + contains(res.body, "\"name\":\"generativelanguage\""), + "lists the known services"); + check(contains(res.body, "https://texttospeech.googleapis.com/v1"), + "base urls shown"); + check(!contains(res.body, kTestKey), "credential never shown"); + check(rig.mock->send_count() == 0, "side effect: no upstream traffic"); + } + { + GoogleRig rig(false); + Fixture f(rig.client); + check(contains(f.client->get("/google/status").body, "\"configured\":false"), + "not configured without a key"); + } + Fixture f; + check(f.client->post("/google/status").status == 405, "post is 405"); + return finish("google/status"); +} diff --git a/tests/example/support/Assert.hpp b/tests/example/support/Assert.hpp index d79e2e4..1e57ac0 100644 --- a/tests/example/support/Assert.hpp +++ b/tests/example/support/Assert.hpp @@ -4,9 +4,11 @@ // test file owns one main() and uses these helpers for loud assertions. #include +#include #include #include "app/App.hpp" +#include "google/client.h" #include "fw/TestClient.hpp" namespace molo::example::test { @@ -40,7 +42,18 @@ struct Fixture { std::shared_ptr app; std::shared_ptr client; - Fixture() { + Fixture() : Fixture(nullptr) {} + + // A scripted Google client can be injected (see support/GoogleRig.hpp). + // Null builds the client from the environment like production. Scripted + // clients default to the classic cloud wire paths; pass a backend to test + // the Gemini ones. + explicit Fixture(std::shared_ptr google, + std::optional backend = + std::nullopt) { + if (google && !backend) backend = molo::google::VoiceBackend::kCloud; + services.voice_backend = backend; + services.google = std::move(google); app = molo::example::app::build_application(services); client = std::make_shared(app); } diff --git a/tests/example/support/GoogleRig.hpp b/tests/example/support/GoogleRig.hpp new file mode 100644 index 0000000..f7099d6 --- /dev/null +++ b/tests/example/support/GoogleRig.hpp @@ -0,0 +1,69 @@ +#pragma once + +// Scripted Google backend for route tests: a GoogleClient over MockTransport +// plus helpers that queue the three upstream replies a voice turn needs. +// The real client, auth, request building and error mapping all run. + +#include +#include + +#include "clients/transports/mock_transport.h" +#include "google/credentials.h" +#include "security/crypto/Base64.h" +#include "support/Assert.hpp" + +namespace molo::example::test { + +inline const char* kTestKey = "AIzaTESTKEY-DO-NOT-LEAK"; + +struct GoogleRig { + molo::clients::transports::MockTransport* mock = nullptr; + std::shared_ptr client; + + explicit GoogleRig(bool with_key = true) { + auto transport = std::make_unique(); + mock = transport.get(); + client = std::make_shared( + std::move(transport), + with_key ? molo::google::Credentials::api_key(kTestKey) : nullptr); + } + + static molo::clients::ClientResponse json(const std::string& body, + int status = 200) { + molo::clients::HeaderMap h; + h.set("Content-Type", "application/json"); + return molo::clients::ClientResponse(status, h, body); + } + + void tts(const std::string& audio) { + mock->expect_response( + molo::clients::Method::kPost, "/v1/text:synthesize", + json("{\"audioContent\":\"" + + molo::security::crypto::Base64::encode(audio) + "\"}")); + } + void stt(const std::string& text) { + mock->expect_response( + molo::clients::Method::kPost, "/v1/speech:recognize", + text.empty() ? json("{}") + : json("{\"results\":[{\"alternatives\":[{\"transcript\":\"" + + text + "\",\"confidence\":0.95}]}]}")); + } + void gemini(const std::string& text) { + mock->expect_response( + molo::clients::Method::kPost, "/v1beta/models/rig-model:generateContent", + json("{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"" + text + + "\"}]}}]}")); + } + void upstream_error(const std::string& path, int status, + const std::string& google_status, + const std::string& message) { + mock->expect_response( + molo::clients::Method::kPost, path, + json("{\"error\":{\"code\":" + std::to_string(status) + + ",\"message\":\"" + message + "\",\"status\":\"" + + google_status + "\"}}", + status)); + } +}; + +} // namespace molo::example::test diff --git a/tests/example/voice/hear_test.cpp b/tests/example/voice/hear_test.cpp new file mode 100644 index 0000000..92e8a35 --- /dev/null +++ b/tests/example/voice/hear_test.cpp @@ -0,0 +1,40 @@ +// Route: POST /voice/hear (audio bytes to transcript through Google STT) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + GoogleRig rig; + rig.stt("goedemorgen"); + Fixture f(rig.client); + auto res = f.client->post("/voice/hear?lang=nl-NL", "WEBM-AUDIO-BYTES", + "audio/webm;codecs=opus"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "application/json"), "json"); + check(contains(res.body, "\"text\":\"goedemorgen\""), "transcript returned"); + auto seen = rig.mock->seen(); + check(seen.size() == 1 && contains(seen[0].body, "WEBM_OPUS") && + contains(seen[0].body, "nl-NL"), + "side effect: encoding and language sent upstream"); + check(!contains(seen[0].body, "WEBM-AUDIO-BYTES"), "audio is base64, not raw"); + + check(f.client->post("/voice/hear", "xx", "video/mp4").status == 415, + "unknown media type is 415"); + check(f.client->post("/voice/hear", "", "audio/webm").status == 422, + "empty audio is 422"); + check(f.client->post("/voice/hear?lang=zz", "xx", "audio/webm").status == 422, + "bad language is 422"); + std::string big(1024 * 1024 + 8192, 'a'); + check(f.client->post("/voice/hear", big, "audio/webm").status == 413, + "body over the 1 MiB cap is 413"); + check(rig.mock->send_count() == 1, "rejected input never reached Google"); + check(f.client->get("/voice/hear").status == 405, "get is 405"); + + GoogleRig failing; + failing.upstream_error("/v1/speech:recognize", 429, "RESOURCE_EXHAUSTED", "quota"); + Fixture g(failing.client); + auto bad = g.client->post("/voice/hear", "xx", "audio/webm"); + check(bad.status == 502 && contains(bad.body, "RESOURCE_EXHAUSTED"), + "quota error surfaced as 502"); + return finish("voice/hear"); +} diff --git a/tests/example/voice/page_test.cpp b/tests/example/voice/page_test.cpp new file mode 100644 index 0000000..6cf0867 --- /dev/null +++ b/tests/example/voice/page_test.cpp @@ -0,0 +1,25 @@ +// Route: GET /voice (live voice conversation page, script embedded) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + GoogleRig rig; + Fixture f(rig.client); + auto page = f.client->get("/voice"); + check(page.status == 200, "status 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "html"); + check(contains(page.body, "id=\"v-toggle\""), "start/stop button"); + check(contains(page.body, "id=\"v-lang\""), "language select"); + check(contains(page.body, "id=\"v-log\""), "conversation log"); + check(contains(page.body, "/embedded/voice.js"), "script linked"); + check(contains(page.body, "width=device-width"), "responsive viewport"); + auto js = f.client->get("/embedded/voice.js"); + check(js.status == 200 && contains(js.body, "/voice/turn"), + "page script served from the bundle"); + check(contains(js.body, "getUserMedia") && contains(js.body, "MediaRecorder"), + "script records the microphone"); + check(f.client->post("/voice").status == 405, "post is 405"); + check(rig.mock->send_count() == 0, "side effect: page makes no Google call"); + return finish("voice/page"); +} diff --git a/tests/example/voice/reset_test.cpp b/tests/example/voice/reset_test.cpp new file mode 100644 index 0000000..e859d67 --- /dev/null +++ b/tests/example/voice/reset_test.cpp @@ -0,0 +1,25 @@ +// Route: POST /voice/reset (forget one conversation) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + setenv("GOOGLE_GENAI_MODEL", "rig-model", 1); + GoogleRig rig; + rig.gemini("Hoi!"); + rig.tts("A"); + Fixture f(rig.client); + f.client->post("/voice/turn?cid=keep1", "{\"text\":\"hallo\"}", "application/json"); + check(f.services.voice->sessions().history_size("keep1") == 2, "arranged: 2 messages"); + auto res = f.client->post("/voice/reset?cid=keep1"); + check(res.status == 200 && contains(res.body, "\"reset\":true"), "reset answered"); + check(f.services.voice->sessions().history_size("keep1") == 0, + "side effect: history gone"); + check(f.client->post("/voice/reset?cid=never-seen").status == 200, + "unknown conversation is a no-op"); + check(f.client->post("/voice/reset").status == 422, "missing cid is 422"); + check(f.client->post("/voice/reset?cid=a%2Fb").status == 422, "bad cid is 422"); + check(f.client->get("/voice/reset").status == 405, "get is 405"); + unsetenv("GOOGLE_GENAI_MODEL"); + return finish("voice/reset"); +} diff --git a/tests/example/voice/say_test.cpp b/tests/example/voice/say_test.cpp new file mode 100644 index 0000000..c169517 --- /dev/null +++ b/tests/example/voice/say_test.cpp @@ -0,0 +1,56 @@ +// Route: POST /voice/say (text to spoken mp3 through Google TTS) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + GoogleRig rig; + rig.tts("FAKE-MP3-BYTES"); + Fixture f(rig.client); + auto res = f.client->post("/voice/say", + "{\"text\":\"hallo wereld\",\"lang\":\"nl-NL\"}", + "application/json"); + check(res.status == 200, "status 200"); + check(res.headers.get("Content-Type") == "audio/mpeg", "audio/mpeg"); + check(res.body == "FAKE-MP3-BYTES", "decoded audio returned"); + auto seen = rig.mock->seen(); + check(seen.size() == 1 && contains(seen[0].body, "hallo wereld"), + "side effect: text sent upstream"); + check(seen[0].headers.get_or("x-goog-api-key", "") == kTestKey, + "key sent in header"); + check(seen[0].target.find(kTestKey) == std::string::npos, "key not in url"); + + check(f.client->post("/voice/say", "{\"text\":\"\"}", "application/json").status == 422, + "empty text is 422"); + check(f.client->post("/voice/say", "{\"text\":\"x\",\"lang\":\"dutch\"}", + "application/json").status == 422, + "bad language is 422"); + check(f.client->post("/voice/say", + "{\"text\":\"x\",\"voice\":\"a b;c\"}", + "application/json").status == 422, + "bad voice name is 422"); + std::string long_text(5001, 'a'); + check(f.client->post("/voice/say", "{\"text\":\"" + long_text + "\"}", + "application/json").status == 422, + "text over the 5000 char cap is 422"); + check(rig.mock->send_count() == 1, "rejected input never reached Google"); + check(f.client->get("/voice/say").status == 405, "get is 405"); + + GoogleRig failing; + failing.upstream_error("/v1/text:synthesize", 403, "PERMISSION_DENIED", + "API not enabled"); + Fixture g(failing.client); + auto bad = g.client->post("/voice/say", "{\"text\":\"x\"}", "application/json"); + check(bad.status == 502, "upstream failure is 502"); + check(contains(bad.body, "google_error") && contains(bad.body, "PERMISSION_DENIED"), + "google status surfaced"); + check(!contains(bad.body, kTestKey), "key never in an error"); + + GoogleRig none(false); + Fixture h(none.client); + auto nc = h.client->post("/voice/say", "{\"text\":\"x\"}", "application/json"); + check(nc.status == 503 && contains(nc.body, "google_not_configured"), + "no key is a loud 503"); + check(none.mock->send_count() == 0, "no network without a key"); + return finish("voice/say"); +} diff --git a/tests/example/voice/turn_test.cpp b/tests/example/voice/turn_test.cpp new file mode 100644 index 0000000..cb9831f --- /dev/null +++ b/tests/example/voice/turn_test.cpp @@ -0,0 +1,94 @@ +// Route: POST /voice/turn (one live-conversation turn: hear, think, speak) +#include "support/GoogleRig.hpp" + +#include "security/crypto/Base64.h" + +using namespace molo::example::test; + +int main() { + setenv("GOOGLE_GENAI_MODEL", "rig-model", 1); + GoogleRig rig; + rig.stt("wat is de hoofdstad van Nederland"); + rig.gemini("Amsterdam."); + rig.tts("SPOKEN-ANSWER"); + Fixture f(rig.client); + + auto res = f.client->post("/voice/turn?cid=abc123&lang=nl-NL", "WEBM-BYTES", + "audio/webm;codecs=opus"); + check(res.status == 200, "status 200"); + check(contains(res.headers.get("Content-Type"), "application/json"), "json"); + check(contains(res.body, "\"transcript\":\"wat is de hoofdstad van Nederland\""), + "transcript"); + check(contains(res.body, "\"reply\":\"Amsterdam.\""), "reply text"); + check(contains(res.body, "\"mime\":\"audio/mpeg\""), "mime"); + check(contains(res.body, "\"audio_b64\":\"" + + molo::security::crypto::Base64::encode("SPOKEN-ANSWER") + "\""), + "spoken answer as base64"); + check(rig.mock->send_count() == 3, "stt, gemini, tts called once each"); + check(f.services.voice->sessions().history_size("abc123") == 2, + "side effect: user and model turn remembered"); + check(contains(rig.mock->seen()[1].body, "hoofdstad"), + "transcript fed to the model"); + + // Second turn carries the first as history. + rig.stt("en van Duitsland"); + rig.gemini("Berlijn."); + rig.tts("SPOKEN-2"); + f.client->post("/voice/turn?cid=abc123", "WEBM", "audio/webm"); + auto gem2 = rig.mock->seen()[4].body; + check(contains(gem2, "Amsterdam.") && contains(gem2, "Duitsland"), + "history sent with the next turn"); + + // Typed turn. + rig.gemini("Hallo!"); + rig.tts("SPOKEN-3"); + auto typed = f.client->post("/voice/turn?cid=abc123", "{\"text\":\"hoi\"}", + "application/json"); + check(typed.status == 200 && contains(typed.body, "\"reply\":\"Hallo!\""), + "typed turn answered"); + + // Silence: one STT call, nothing else. + std::size_t before = rig.mock->send_count(); + rig.stt(""); + auto quiet = f.client->post("/voice/turn?cid=quiet1", "WEBM", "audio/webm"); + check(quiet.status == 200 && contains(quiet.body, "\"transcript\":\"\"") && + contains(quiet.body, "\"audio_b64\":\"\""), + "silence answers an empty turn"); + check(rig.mock->send_count() == before + 1, "silence costs one STT call"); + + // Validation and limits. + check(f.client->post("/voice/turn", "x", "audio/webm").status == 422, + "missing cid is 422"); + check(f.client->post("/voice/turn?cid=bad%20id", "x", "audio/webm").status == 422, + "bad cid is 422"); + check(f.client->post("/voice/turn?cid=a1", "x", "video/mp4").status == 415, + "unknown media type is 415"); + check(f.client->post("/voice/turn?cid=a1", "", "audio/webm").status == 422, + "empty audio is 422"); + check(f.client->post("/voice/turn?cid=a1", "{\"text\":\"\"}", + "application/json").status == 422, + "empty typed text is 422"); + check(f.client->post("/voice/turn?cid=a1", std::string(1024 * 1024 + 8192, 'a'), + "audio/webm").status == 413, + "body over the cap is 413"); + check(f.client->get("/voice/turn").status == 405, "get is 405"); + + // A failed model call leaves no half conversation. + GoogleRig failing; + failing.stt("hallo"); + failing.upstream_error("/v1beta/models/rig-model:generateContent", 503, + "UNAVAILABLE", "overloaded"); + Fixture g(failing.client); + auto bad = g.client->post("/voice/turn?cid=fail1", "x", "audio/webm"); + check(bad.status == 502 && contains(bad.body, "UNAVAILABLE"), "502 on model failure"); + check(g.services.voice->sessions().history_size("fail1") == 0, + "failed turn rolled back"); + + GoogleRig none(false); + Fixture h(none.client); + auto nc = h.client->post("/voice/turn?cid=n1", "x", "audio/webm"); + check(nc.status == 503 && contains(nc.body, "google_not_configured"), + "no key is a loud 503"); + unsetenv("GOOGLE_GENAI_MODEL"); + return finish("voice/turn"); +} diff --git a/tests/example/voice/vertex_test.cpp b/tests/example/voice/vertex_test.cpp new file mode 100644 index 0000000..abf3db8 --- /dev/null +++ b/tests/example/voice/vertex_test.cpp @@ -0,0 +1,44 @@ +// Route: POST /voice/turn (Gemini audio backend: Vertex express key) +#include "support/GoogleRig.hpp" + +using namespace molo::example::test; + +int main() { + setenv("GOOGLE_GENAI_MODEL", "chat-m", 1); + setenv("GOOGLE_STT_MODEL", "stt-m", 1); + setenv("GOOGLE_TTS_MODEL", "tts-m", 1); + GoogleRig rig; + const std::string p = "/v1/publishers/google/models/"; + auto text = [&](const std::string& model, const std::string& t) { + rig.mock->expect_response( + molo::clients::Method::kPost, p + model + ":generateContent", + GoogleRig::json("{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"" + + t + "\"}]}}]}")); + }; + text("stt-m", "hallo daar"); + text("chat-m", "Hoi!"); + rig.mock->expect_response( + molo::clients::Method::kPost, p + "tts-m:generateContent", + GoogleRig::json( + "{\"candidates\":[{\"content\":{\"parts\":[{\"inlineData\":{\"mimeType\":" + "\"audio/L16;codec=pcm;rate=24000\",\"data\":\"" + + molo::security::crypto::Base64::encode("PCM") + "\"}}]}}]}")); + Fixture f(rig.client, molo::google::VoiceBackend::kVertex); + + check(contains(f.client->get("/google/status").body, "\"backend\":\"vertex\""), + "status names the backend"); + auto res = f.client->post("/voice/turn?cid=v1", "WEBM", "audio/webm;codecs=opus"); + check(res.status == 200, "status 200"); + check(contains(res.body, "\"transcript\":\"hallo daar\""), "transcript"); + check(contains(res.body, "\"reply\":\"Hoi!\""), "reply"); + check(contains(res.body, "\"mime\":\"audio/wav\""), "wav for the browser"); + check(rig.mock->send_count() == 3, "three Gemini calls"); + check(rig.mock->seen()[0].target.rfind(p, 0) == 0, "vertex endpoint used"); + check(contains(rig.mock->seen()[0].body, "audio/webm"), "audio mime sent to Gemini"); + check(rig.mock->seen()[0].headers.get_or("x-goog-api-key", "") == kTestKey, + "key in header"); + unsetenv("GOOGLE_GENAI_MODEL"); + unsetenv("GOOGLE_STT_MODEL"); + unsetenv("GOOGLE_TTS_MODEL"); + return finish("voice/turn vertex"); +} diff --git a/tests/src/google/test_google.cpp b/tests/src/google/test_google.cpp new file mode 100644 index 0000000..16a27e2 --- /dev/null +++ b/tests/src/google/test_google.cpp @@ -0,0 +1,519 @@ +// Google Cloud part: request shape, credential placement, error mapping, +// caps, and the voice pipeline, all against a scripted transport. +#include + +#include "framework.h" +#include "google/caps.h" +#include "google/client.h" +#include "google/credentials.h" +#include "google/endpoints.h" +#include "google/gemini_voice.h" +#include "google/generative.h" +#include "google/voice_backend.h" +#include "google/wav.h" +#include "google/speech_to_text.h" +#include "google/text_to_speech.h" +#include "google/voice.h" +#include "clients/transports/mock_transport.h" +#include "security/crypto/Base64.h" + +using namespace molo::google; +using molo::clients::ClientResponse; +using molo::clients::HeaderMap; +using molo::clients::Method; +using molo::clients::transports::MockTransport; +using molo::clients::body::Json; +using molo::security::crypto::Base64; + +namespace { + +const char* kKey = "AIzaSECRETKEY123"; + +ClientResponse json_ok(const std::string& body, int status = 200) { + HeaderMap h; + h.set("Content-Type", "application/json"); + return ClientResponse(status, h, body); +} + +struct Rig { + MockTransport* mock; + std::shared_ptr client; + explicit Rig(bool with_key = true) { + auto transport = std::make_unique(); + mock = transport.get(); + client = std::make_shared( + std::move(transport), with_key ? Credentials::api_key(kKey) : nullptr); + } + void tts_ok(const std::string& mp3 = "MP3DATA") { + mock->expect_response(Method::kPost, "/v1/text:synthesize", + json_ok("{\"audioContent\":\"" + Base64::encode(mp3) + "\"}")); + } + void stt_ok(const std::string& text) { + mock->expect_response( + Method::kPost, "/v1/speech:recognize", + json_ok("{\"results\":[{\"alternatives\":[{\"transcript\":\"" + text + + "\",\"confidence\":0.9}]}]}")); + } + void gemini_ok(const std::string& text) { + mock->expect_response( + Method::kPost, "/v1beta/models/m1:generateContent", + json_ok("{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"" + text + + "\"}]}}]}")); + } +}; + +std::shared_ptr pipeline(Rig& rig, + std::shared_ptr s = nullptr) { + auto gen = std::make_shared(rig.client); + return std::make_shared( + std::make_shared(rig.client), + std::make_shared(gen), + std::make_shared(rig.client), + s ? s : std::make_shared()); +} + +} // namespace + +TEST(google, key_in_header_never_in_url) { + Rig rig; + rig.tts_ok(); + TextToSpeech tts(rig.client); + SynthesisRequest req; + req.text = "hallo"; + tts.synthesize(req); + auto seen = rig.mock->seen(); + ASSERT_EQ(seen.size(), 1u); + ASSERT_EQ(seen[0].headers.get_or("x-goog-api-key", ""), std::string(kKey)); + ASSERT_TRUE(seen[0].target.find(kKey) == std::string::npos); +} + +TEST(google, not_configured_fails_before_network) { + Rig rig(false); + TextToSpeech tts(rig.client); + SynthesisRequest req; + req.text = "hallo"; + try { + tts.synthesize(req); + FAIL("expected NotConfiguredError"); + } catch (const NotConfiguredError& e) { + ASSERT_EQ(e.http_status(), 503); + ASSERT_EQ(e.status(), "NOT_CONFIGURED"); + } + ASSERT_EQ(rig.mock->send_count(), 0u); + ASSERT_FALSE(tts.configured()); +} + +TEST(google, tts_request_shape_and_decode) { + Rig rig; + rig.tts_ok("AUDIO!"); + TextToSpeech tts(rig.client); + SynthesisRequest req; + req.text = "hoi"; + req.language = "nl-NL"; + req.voice = "nl-NL-Wavenet-A"; + Audio a = tts.synthesize(req); + ASSERT_EQ(a.bytes, "AUDIO!"); + ASSERT_EQ(a.mime, "audio/mpeg"); + Json body = Json::parse(rig.mock->seen()[0].body); + ASSERT_EQ(body.find("input")->find("text")->as_string(), "hoi"); + ASSERT_EQ(body.find("voice")->find("languageCode")->as_string(), "nl-NL"); + ASSERT_EQ(body.find("voice")->find("name")->as_string(), "nl-NL-Wavenet-A"); + ASSERT_EQ(body.find("audioConfig")->find("audioEncoding")->as_string(), "MP3"); +} + +TEST(google, tts_rejects_bad_input_without_network) { + Rig rig; + TextToSpeech tts(rig.client); + SynthesisRequest req; + ASSERT_THROW(tts.synthesize(req), GoogleError); // empty + req.text = std::string(Caps::kMaxTtsChars + 1, 'a'); + ASSERT_THROW(tts.synthesize(req), GoogleError); // over cap + req.text = "ok"; + req.speaking_rate = 9.0; + ASSERT_THROW(tts.synthesize(req), GoogleError); // rate + ASSERT_EQ(rig.mock->send_count(), 0u); +} + +TEST(google, tts_voices_list) { + Rig rig; + rig.mock->expect_response( + Method::kGet, "/v1/voices", + json_ok("{\"voices\":[{\"name\":\"nl-NL-Wavenet-A\",\"ssmlGender\":" + "\"FEMALE\",\"languageCodes\":[\"nl-NL\"]}]}")); + auto v = TextToSpeech(rig.client).voices("nl-NL"); + ASSERT_EQ(v.size(), 1u); + ASSERT_EQ(v[0].name, "nl-NL-Wavenet-A"); + ASSERT_EQ(v[0].languages[0], "nl-NL"); +} + +TEST(google, error_mapping_hides_key) { + Rig rig; + rig.mock->expect_response( + Method::kPost, "/v1/text:synthesize", + json_ok("{\"error\":{\"code\":403,\"message\":\"API not enabled\"," + "\"status\":\"PERMISSION_DENIED\"}}", + 403)); + SynthesisRequest req; + req.text = "x"; + try { + TextToSpeech(rig.client).synthesize(req); + FAIL("expected GoogleError"); + } catch (const GoogleError& e) { + ASSERT_EQ(e.http_status(), 403); + ASSERT_EQ(e.status(), "PERMISSION_DENIED"); + ASSERT_EQ(e.service(), "texttospeech"); + ASSERT_TRUE(std::string(e.what()).find("API not enabled") != std::string::npos); + ASSERT_TRUE(std::string(e.what()).find(kKey) == std::string::npos); + } +} + +TEST(google, non_json_error_body) { + Rig rig; + rig.mock->expect_status(Method::kPost, "/v1/text:synthesize", 400, ""); + SynthesisRequest req; + req.text = "x"; + try { + TextToSpeech(rig.client).synthesize(req); + FAIL("expected GoogleError"); + } catch (const GoogleError& e) { + ASSERT_EQ(e.http_status(), 400); + ASSERT_EQ(e.status(), "UNKNOWN"); + } +} + +TEST(google, transport_failure_becomes_unavailable) { + Rig rig; + rig.mock->expect_error( + Method::kPost, "/v1/text:synthesize", + std::make_exception_ptr(molo::clients::TransportError("boom"))); + SynthesisRequest req; + req.text = "x"; + try { + TextToSpeech(rig.client).synthesize(req); + FAIL("expected GoogleError"); + } catch (const GoogleError& e) { + ASSERT_EQ(e.status(), "UNAVAILABLE"); + } +} + +TEST(google, stt_request_shape_and_join) { + Rig rig; + rig.mock->expect_response( + Method::kPost, "/v1/speech:recognize", + json_ok("{\"results\":[{\"alternatives\":[{\"transcript\":\"hallo\"," + "\"confidence\":0.8}]},{\"alternatives\":[{\"transcript\":" + "\"wereld\"}]}]}")); + RecognitionRequest req; + req.audio = "RAWAUDIO"; + Transcript t = SpeechToText(rig.client).recognize(req); + ASSERT_EQ(t.text, "hallo wereld"); + ASSERT_TRUE(t.confidence > 0.79 && t.confidence < 0.81); + Json body = Json::parse(rig.mock->seen()[0].body); + ASSERT_EQ(body.find("config")->find("encoding")->as_string(), "WEBM_OPUS"); + ASSERT_EQ(body.find("config")->find("languageCode")->as_string(), "nl-NL"); + ASSERT_EQ(body.find("audio")->find("content")->as_string(), + Base64::encode("RAWAUDIO")); +} + +TEST(google, stt_silence_is_empty_transcript) { + Rig rig; + rig.mock->expect_response(Method::kPost, "/v1/speech:recognize", json_ok("{}")); + RecognitionRequest req; + req.audio = "x"; + ASSERT_EQ(SpeechToText(rig.client).recognize(req).text, ""); +} + +TEST(google, stt_rejects_bad_audio_without_network) { + Rig rig; + SpeechToText stt(rig.client); + RecognitionRequest req; + ASSERT_THROW(stt.recognize(req), GoogleError); + req.audio = std::string(Caps::kMaxSttBytes + 1, 'a'); + ASSERT_THROW(stt.recognize(req), GoogleError); + ASSERT_EQ(rig.mock->send_count(), 0u); +} + +TEST(google, stt_encoding_from_mime) { + SpeechEncoding e; + ASSERT_TRUE(speech_encoding_from_mime("audio/webm;codecs=opus", e)); + ASSERT_EQ(std::string(speech_encoding_name(e)), "WEBM_OPUS"); + ASSERT_TRUE(speech_encoding_from_mime("audio/ogg", e)); + ASSERT_EQ(std::string(speech_encoding_name(e)), "OGG_OPUS"); + ASSERT_FALSE(speech_encoding_from_mime("video/mp4", e)); +} + +TEST(google, generative_request_shape) { + Rig rig; + rig.gemini_ok("dag"); + GenerationRequest req; + req.model = "m1"; + req.system = "be brief"; + req.messages = {{"user", "hoi"}}; + ASSERT_EQ(Generative(rig.client).generate(req), "dag"); + Json body = Json::parse(rig.mock->seen()[0].body); + ASSERT_EQ(body.find("contents")->as_array()[0].find("role")->as_string(), "user"); + ASSERT_EQ(body.find("systemInstruction")->find("parts")->as_array()[0] + .find("text")->as_string(), "be brief"); + ASSERT_EQ(body.find("generationConfig")->find("maxOutputTokens")->as_int(), 300); + ASSERT_TRUE(body.find("generationConfig")->find("thinkingConfig") == nullptr); +} + +TEST(google, generative_validation_and_empty) { + Rig rig; + Generative gen(rig.client); + GenerationRequest req; + req.model = "m1"; + ASSERT_THROW(gen.generate(req), GoogleError); // no messages + req.messages = {{"robot", "x"}}; + ASSERT_THROW(gen.generate(req), GoogleError); // bad role + req.messages = {{"user", "x"}}; + req.max_output_tokens = Caps::kMaxOutputTokens + 1; + ASSERT_THROW(gen.generate(req), GoogleError); // token cap + ASSERT_EQ(rig.mock->send_count(), 0u); + req.max_output_tokens = 10; + rig.mock->expect_response(Method::kPost, "/v1beta/models/m1:generateContent", + json_ok("{\"candidates\":[]}")); + try { + gen.generate(req); + FAIL("expected EMPTY_RESPONSE"); + } catch (const GoogleError& e) { + ASSERT_EQ(e.status(), "EMPTY_RESPONSE"); + } +} + +TEST(google, voice_turn_full_pipeline) { + setenv("GOOGLE_GENAI_MODEL", "m1", 1); + Rig rig; + rig.stt_ok("wat is het weer"); + rig.gemini_ok("Zonnig."); + rig.tts_ok("SPOKEN"); + auto p = pipeline(rig); + RecognitionRequest heard; + heard.audio = "AUDIO"; + VoiceTurn t = p->turn("conv1", heard); + ASSERT_EQ(t.transcript, "wat is het weer"); + ASSERT_EQ(t.reply, "Zonnig."); + ASSERT_EQ(t.audio.bytes, "SPOKEN"); + ASSERT_EQ(Json::parse(rig.mock->seen()[1].body) + .find("generationConfig")->find("thinkingConfig") + ->find("thinkingBudget")->as_int(), 0); + ASSERT_EQ(p->sessions().history_size("conv1"), 2u); + rig.mock->verify(); + unsetenv("GOOGLE_GENAI_MODEL"); +} + +TEST(google, voice_silence_skips_reply_and_synthesis) { + Rig rig; + rig.mock->expect_response(Method::kPost, "/v1/speech:recognize", json_ok("{}")); + auto p = pipeline(rig); + RecognitionRequest heard; + heard.audio = "AUDIO"; + VoiceTurn t = p->turn("conv1", heard); + ASSERT_EQ(t.transcript, ""); + ASSERT_EQ(t.audio.bytes, ""); + ASSERT_EQ(rig.mock->send_count(), 1u); + ASSERT_EQ(p->sessions().size(), 0u); +} + +TEST(google, voice_failed_turn_rolls_back_history) { + setenv("GOOGLE_GENAI_MODEL", "m1", 1); + Rig rig; + rig.mock->expect_response( + Method::kPost, "/v1beta/models/m1:generateContent", + json_ok("{\"error\":{\"code\":429,\"message\":\"quota\",\"status\":" + "\"RESOURCE_EXHAUSTED\"}}", 400)); + auto p = pipeline(rig); + ASSERT_THROW(p->turn_text("c", "hoi", "nl-NL"), GoogleError); + ASSERT_EQ(p->sessions().history_size("c"), 0u); + unsetenv("GOOGLE_GENAI_MODEL"); +} + +TEST(google, voice_bad_conversation_id) { + Rig rig; + auto p = pipeline(rig); + ASSERT_THROW(p->turn_text("bad id!", "hoi", "nl-NL"), GoogleError); + ASSERT_THROW(p->turn_text("", "hoi", "nl-NL"), GoogleError); + ASSERT_EQ(rig.mock->send_count(), 0u); +} + +TEST(google, sessions_bounded_history_and_count) { + VoiceSessions s; + for (std::size_t i = 0; i < Caps::kMaxHistoryTurns * 2 + 10; ++i) + s.append("a", {i % 2 == 0 ? "user" : "model", "m"}); + ASSERT_TRUE(s.history_size("a") <= Caps::kMaxHistoryTurns * 2); + auto h = s.append("a", {"user", "x"}); + ASSERT_EQ(h.front().role, "user"); + for (std::size_t i = 0; i < Caps::kMaxConversations + 5; ++i) + s.append("c" + std::to_string(i), {"user", "m"}); + ASSERT_EQ(s.size(), Caps::kMaxConversations); + ASSERT_EQ(s.history_size("a"), 0u); // oldest evicted + s.reset("c10"); + ASSERT_EQ(s.history_size("c10"), 0u); +} + +TEST(google, catalog_lists_known_services) { + auto c = endpoints::catalog(); + ASSERT_EQ(c.size(), 4u); + ASSERT_EQ(c[0].name, "texttospeech"); + for (const auto& e : c) ASSERT_TRUE(e.base_url.rfind("https://", 0) == 0); +} + +// ---- Gemini audio backends (Vertex paths) ---------------------------------- + +namespace { +const char* kVertexPrefix = "/v1/publishers/google/models/"; + +void vertex_text(Rig& rig, const std::string& model, const std::string& text) { + rig.mock->expect_response( + Method::kPost, std::string(kVertexPrefix) + model + ":generateContent", + json_ok("{\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"" + text + + "\"}]}}]}")); +} +void vertex_pcm(Rig& rig, const std::string& model, const std::string& pcm) { + rig.mock->expect_response( + Method::kPost, std::string(kVertexPrefix) + model + ":generateContent", + json_ok("{\"candidates\":[{\"content\":{\"parts\":[{\"inlineData\":" + "{\"mimeType\":\"audio/L16;codec=pcm;rate=24000\",\"data\":\"" + + Base64::encode(pcm) + "\"}}]}}]}")); +} +} // namespace + +TEST(google, wav_wrap_and_rate) { + std::string wav = pcm16_to_wav(std::string(10, 'x'), 24000); + ASSERT_EQ(wav.size(), 54u); + ASSERT_EQ(wav.substr(0, 4), "RIFF"); + ASSERT_EQ(wav.substr(8, 4), "WAVE"); + ASSERT_EQ(static_cast(wav[24]), 0xC0); // 24000 = 0x5DC0 + ASSERT_EQ(static_cast(wav[25]), 0x5D); + ASSERT_EQ(wav.substr(44), std::string(10, 'x')); + ASSERT_EQ(sample_rate_from_mime("audio/L16;codec=pcm;rate=16000"), 16000); + ASSERT_EQ(sample_rate_from_mime("audio/L16"), 24000); + ASSERT_EQ(sample_rate_from_mime("audio/L16;rate=999999999"), 24000); +} + +TEST(google, gemini_transcribe_shape) { + Rig rig; + vertex_text(rig, "stt-m", " hallo wereld \n"); + Generative gen(rig.client, endpoints::vertex_generative()); + std::string text = gen.transcribe("stt-m", "RAW", "audio/webm", "nl-NL"); + ASSERT_EQ(text, "hallo wereld"); + Json body = Json::parse(rig.mock->seen()[0].body); + const Json& parts = *body.find("contents")->as_array()[0].find("parts"); + ASSERT_EQ(parts.as_array()[0].find("inlineData")->find("mimeType")->as_string(), + "audio/webm"); + ASSERT_EQ(parts.as_array()[0].find("inlineData")->find("data")->as_string(), + Base64::encode("RAW")); + ASSERT_TRUE(parts.as_array()[1].find("text")->as_string().find("nl-NL") != + std::string::npos); + ASSERT_THROW(gen.transcribe("stt-m", "", "audio/webm", "nl-NL"), GoogleError); + ASSERT_EQ(rig.mock->send_count(), 1u); +} + +TEST(google, gemini_speak_shape_and_wav) { + Rig rig; + vertex_pcm(rig, "tts-m", "PCMDATA!"); + Generative gen(rig.client, endpoints::vertex_generative()); + Audio a = gen.speak("tts-m", "hoi", ""); + ASSERT_EQ(a.mime, "audio/wav"); + ASSERT_EQ(a.bytes.substr(0, 4), "RIFF"); + ASSERT_EQ(a.bytes.substr(44), "PCMDATA!"); + Json body = Json::parse(rig.mock->seen()[0].body); + const Json& gc = *body.find("generationConfig"); + ASSERT_EQ(gc.find("responseModalities")->as_array()[0].as_string(), "AUDIO"); + ASSERT_EQ(gc.find("speechConfig")->find("voiceConfig")->find("prebuiltVoiceConfig") + ->find("voiceName")->as_string(), "Kore"); + vertex_text(rig, "tts-m", "no audio here"); + ASSERT_THROW(gen.speak("tts-m", "hoi", ""), GoogleError); // EMPTY_RESPONSE + ASSERT_THROW(gen.speak("tts-m", "", ""), GoogleError); +} + +TEST(google, backend_selection) { + unsetenv("GOOGLE_API_BACKEND"); + setenv("GOOGLE_API_KEY", "AQ.vertexstyle", 1); + ASSERT_EQ(std::string(voice_backend_name(voice_backend_from_env())), "vertex"); + setenv("GOOGLE_API_KEY", "AIzaclassic", 1); + ASSERT_EQ(std::string(voice_backend_name(voice_backend_from_env())), "cloud"); + setenv("GOOGLE_API_BACKEND", "gemini", 1); + ASSERT_EQ(std::string(voice_backend_name(voice_backend_from_env())), "gemini"); + unsetenv("GOOGLE_API_BACKEND"); + unsetenv("GOOGLE_API_KEY"); +} + +TEST(google, voice_turn_on_vertex_backend) { + setenv("GOOGLE_GENAI_MODEL", "chat-m", 1); + setenv("GOOGLE_STT_MODEL", "stt-m", 1); + setenv("GOOGLE_TTS_MODEL", "tts-m", 1); + Rig rig; + vertex_text(rig, "stt-m", "wat is het weer"); + vertex_text(rig, "chat-m", "Zonnig."); + vertex_pcm(rig, "tts-m", "SPEECH"); + VoiceParts parts = make_voice_parts(rig.client, VoiceBackend::kVertex); + VoicePipeline p(parts.recognizer, parts.responder, parts.synthesizer, + std::make_shared()); + RecognitionRequest heard; + heard.audio = "AUDIO"; + VoiceTurn t = p.turn("c1", heard); + ASSERT_EQ(t.transcript, "wat is het weer"); + ASSERT_EQ(t.reply, "Zonnig."); + ASSERT_EQ(t.audio.mime, "audio/wav"); + ASSERT_EQ(t.audio.bytes.substr(44), "SPEECH"); + rig.mock->verify(); + ASSERT_TRUE(rig.mock->seen()[0].target.rfind("/v1/publishers/google/", 0) == 0); + unsetenv("GOOGLE_GENAI_MODEL"); + unsetenv("GOOGLE_STT_MODEL"); + unsetenv("GOOGLE_TTS_MODEL"); +} + +// ---- live tests: real network, opt-in ------------------------------------- +// MOLO_LIVE_NET=1 enables the TLS check (no key needed). GOOGLE_API_KEY +// additionally enables a real synthesis. Without them the case prints SKIP +// and passes; the skip is visible in the output, never silent. + +#include +#include + +#include "clients/transports/http_transport.h" + +TEST(google_live, tls_handshake_with_google) { + if (!std::getenv("MOLO_LIVE_NET")) { + std::cout << "SKIP google_live.tls_handshake_with_google (set MOLO_LIVE_NET=1)\n"; + return; + } + // Regression: a wire-format ALPN list made Google answer a fatal alert. + try { + TextToSpeech(std::make_shared( + std::make_unique(), + Credentials::api_key("not-a-real-key"))) + .voices(); + FAIL("a bogus key must be rejected"); + } catch (const GoogleError& e) { + // Reaching Google and being refused proves TLS plus error mapping. + ASSERT_TRUE(e.http_status() == 400 || e.http_status() == 403); + ASSERT_TRUE(e.status() != "UNAVAILABLE"); + } +} + +TEST(google_live, real_speech_roundtrip) { + // Uses whatever backend the key implies (GOOGLE_API_BACKEND / key prefix). + // Speaks a sentence, then transcribes the audio it got back. + if (!std::getenv("MOLO_LIVE_NET") || Credentials::env().empty()) { + std::cout << "SKIP google_live.real_speech_roundtrip (set MOLO_LIVE_NET=1 and a Google key)\n"; + return; + } + VoiceParts parts = make_voice_parts(GoogleClient::from_env(), voice_backend_from_env()); + SynthesisRequest say; + say.text = "Hello, this is a test of the voice pipeline."; + say.language = "en-US"; + Audio audio = parts.synthesizer->synthesize(say); + ASSERT_TRUE(audio.bytes.size() > 5000); + RecognitionRequest heard; + heard.audio = audio.bytes; + SpeechEncoding enc; + ASSERT_TRUE(speech_encoding_from_mime(audio.mime, enc)); + heard.encoding = enc; + heard.language = "en-US"; + Transcript t = parts.recognizer->recognize(heard); + std::cout << " heard: " << t.text << "\n"; + ASSERT_TRUE(t.text.find("voice") != std::string::npos || + t.text.find("Voice") != std::string::npos); +}