// Opt-in real network checks of the Live API features beyond a plain text // turn: tool calls (toolCall / toolResponse round trip), clientContent turns, // input and output transcription, session resumption (handle + reconnect), // video input frames, goAway. MOLO_LIVE_NET=1 plus GOOGLE_CLOUD_PROJECT and // GOOGLE_API_KEY (express key, header) and/or GOOGLE_ACCESS_TOKEN (OAuth // bearer); each test runs once per credential that is set. Messages are tiny. // Credentials are never printed. Loud SKIP otherwise. The goAway check waits // for the server's connection lifetime and needs MOLO_LIVE_SLOW=1 as well. #include #include #include #include #include #include #include #include "framework.h" #include "google/errors.h" #include "google/live/live_session.h" #include "security/crypto/Base64.h" using namespace molo::google; using namespace molo::google::live; using molo::clients::body::Json; namespace { struct Auth { std::string label; LiveEndpoint endpoint; }; // Empty (with a loud SKIP) unless the opt-in env is complete. std::vector auths(const char* test) { std::vector out; const char* on = std::getenv("MOLO_LIVE_NET"); const char* project = std::getenv("GOOGLE_CLOUD_PROJECT"); if (!on || std::string(on) != "1" || !project) { std::printf(" SKIP %s: set MOLO_LIVE_NET=1, GOOGLE_CLOUD_PROJECT and GOOGLE_API_KEY / " "GOOGLE_ACCESS_TOKEN\n", test); return out; } CloudContext ctx = CloudContext::from_env(); if (const char* k = std::getenv("GOOGLE_API_KEY")) out.push_back({"express", LiveEndpoint::vertex_express(ctx, k)}); if (const char* t = std::getenv("GOOGLE_ACCESS_TOKEN")) out.push_back({"oauth", LiveEndpoint::vertex(ctx, t)}); if (out.empty()) std::printf(" SKIP %s: no credential in the environment\n", test); return out; } using Pred = std::function; // Reads events until `stop` accepts one (inclusive), the session ends or the // timeout passes. Returns everything seen. std::vector pump(LiveSession& s, const Pred& stop, int ms = 30000) { std::vector seen; auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); while (std::chrono::steady_clock::now() < deadline) { auto e = s.next_event(200); if (!e) { if (!s.open() && s.pending() == 0) break; continue; } seen.push_back(*e); if (stop(*e)) break; } return seen; } bool is(const LiveEvent& e, LiveEventKind k) { return e.kind == k; } bool turn_done(const LiveEvent& e) { return is(e, LiveEventKind::TurnComplete); } std::string transcript(const std::vector& evs, LiveEventKind kind) { std::string out; for (const auto& e : evs) if (e.kind == kind) out += e.text; return out; } std::string audio_of(const std::vector& evs) { std::string out; for (const auto& e : evs) if (e.kind == LiveEventKind::Audio) out += e.data; return out; } // 24 kHz 16-bit mono PCM (model output) to 16 kHz (model input), linear. std::string resample_24k_to_16k(const std::string& pcm) { std::size_t n = pcm.size() / 2; auto at = [&](std::size_t i) { return static_cast(static_cast(pcm[2 * i]) | (static_cast(pcm[2 * i + 1]) << 8)); }; std::string out; for (std::size_t j = 0; j * 3 / 2 + 1 < n; ++j) { double pos = j * 1.5; std::size_t i = static_cast(pos); double f = pos - static_cast(i); int16_t v = static_cast(at(i) * (1.0 - f) + at(i + 1) * f); out.push_back(static_cast(v & 0xFF)); out.push_back(static_cast((v >> 8) & 0xFF)); } return out; } std::string kinds(const std::vector& evs) { std::string out; for (const auto& e : evs) { if (!out.empty()) out += ","; out += live_event_name(e.kind); } return out; } void show(const std::string& label, const char* what, const std::string& text) { std::printf(" [%s] %s: %s\n", label.c_str(), what, text.substr(0, 120).c_str()); } LiveConfig base_config() { LiveConfig cfg; cfg.output_transcription = true; return cfg; } } // namespace TEST(live_net, tool_call_round_trip_and_client_content_turn) { for (auto& a : auths("live_net.tool_call_round_trip_and_client_content_turn")) { LiveConfig cfg = base_config(); cfg.system_instruction = "For any temperature question call get_temperature. After the tool result answer in one short sentence."; LiveFunctionDeclaration fn; fn.name = "get_temperature"; fn.description = "Current temperature in Celsius for a city"; fn.parameters = Json::parse( "{\"type\":\"OBJECT\",\"properties\":{\"city\":{\"type\":\"STRING\"}},\"required\":[\"city\"]}"); cfg.functions.push_back(fn); LiveSession s(a.endpoint, cfg); s.connect(); s.send_turn("What is the temperature in Utrecht right now?", true); // clientContent auto evs = pump(s, [](const LiveEvent& e) { return is(e, LiveEventKind::ToolCall) || turn_done(e); }); ASSERT_TRUE(!evs.empty() && evs.back().kind == LiveEventKind::ToolCall); const LiveEvent& call = evs.back(); ASSERT_EQ(call.calls.size(), 1u); ASSERT_EQ(call.calls[0].name, "get_temperature"); ASSERT_TRUE(call.calls[0].args.is_object() && call.calls[0].args.has("city")); show(a.label, "toolCall", call.calls[0].name + " id=" + (call.calls[0].id.empty() ? "(none)" : "set") + " args=" + call.calls[0].args.dump()); Json resp = Json::object(); resp.set("temp_c", Json(17)); s.send_tool_responses({{call.calls[0].id, call.calls[0].name, resp}}); // The turn that carried the toolCall ends first; the answer follows it. bool answered = false; auto after = pump(s, [&](const LiveEvent& e) { answered = answered || is(e, LiveEventKind::Audio) || (is(e, LiveEventKind::OutputTranscription) && !e.text.empty()); return answered && turn_done(e); }); show(a.label, "kinds after toolResponse", kinds(after).substr(0, 60)); ASSERT_TRUE(!after.empty() && after.back().kind == LiveEventKind::TurnComplete); std::string said = transcript(after, LiveEventKind::OutputTranscription); show(a.label, "answer after toolResponse", said); ASSERT_TRUE(!said.empty() || !audio_of(after).empty()); s.close(); } } TEST(live_net, input_transcription_from_audio_and_audio_stream_end) { for (auto& a : auths("live_net.input_transcription_from_audio_and_audio_stream_end")) { // Speech to feed: the model's own audio for a fixed sentence. std::string speech; { LiveSession tts(a.endpoint, base_config()); tts.connect(); tts.send_text("Say exactly these words and nothing else: the blue moon rises."); auto evs = pump(tts, turn_done); speech = resample_24k_to_16k(audio_of(evs)); tts.close(); } ASSERT_TRUE(speech.size() > 8000); LiveConfig cfg = base_config(); cfg.input_transcription = true; LiveSession s(a.endpoint, cfg); s.connect(); const std::size_t chunk = 8000; // 250 ms for (std::size_t off = 0; off < speech.size(); off += chunk) s.send_audio(speech.substr(off, chunk)); s.send_audio(std::string(32000, '\0')); // 1 s of silence so the server's VAD ends the utterance s.end_audio(); // audioStreamEnd auto evs = pump(s, turn_done); std::printf(" speech bytes=%zu received=%llu\n", speech.size(), (unsigned long long)s.stats().received); show(a.label, "kinds", kinds(evs)); std::string heard = transcript(evs, LiveEventKind::InputTranscription); show(a.label, "inputTranscription", heard); ASSERT_TRUE(!heard.empty()); s.close(); } } TEST(live_net, session_resumption_handle_and_reconnect) { for (auto& a : auths("live_net.session_resumption_handle_and_reconnect")) { std::string handle; { LiveConfig cfg = base_config(); cfg.session_resumption = true; LiveSession s(a.endpoint, cfg); s.connect(); s.send_turn("Remember this secret word: pineapple. Reply with just OK.", true); auto evs = pump(s, turn_done); ASSERT_TRUE(!evs.empty() && evs.back().kind == LiveEventKind::TurnComplete); // The handle arrives in sessionResumptionUpdate, around the turn end. for (int i = 0; i < 40 && s.resumption_handle().empty(); ++i) { auto e = s.next_event(250); (void)e; } handle = s.resumption_handle(); std::printf(" [%s] resumption handle %s (%zu chars)\n", a.label.c_str(), handle.empty() ? "NOT received" : "received", handle.size()); s.close(); } ASSERT_TRUE(!handle.empty()); LiveConfig cfg = base_config(); cfg.session_resumption = true; cfg.resumption_handle = handle; LiveSession r(a.endpoint, cfg); r.connect(); // setupComplete on the resumed session r.send_turn("What was the secret word? Answer with the word only.", true); auto evs = pump(r, turn_done); std::string said = transcript(evs, LiveEventKind::OutputTranscription); show(a.label, "resumed answer", said); ASSERT_TRUE(!evs.empty() && evs.back().kind == LiveEventKind::TurnComplete); r.close(); } } TEST(live_net, new_input_while_the_model_speaks_interrupts) { for (auto& a : auths("live_net.new_input_while_the_model_speaks_interrupts")) { LiveSession s(a.endpoint, base_config()); s.connect(); s.send_text("Count slowly from one to thirty, saying every number."); auto first = pump(s, [](const LiveEvent& e) { return is(e, LiveEventKind::Audio); }); ASSERT_TRUE(!first.empty() && first.back().kind == LiveEventKind::Audio); s.send_text("Stop. Say only: stopped."); bool interrupted = false; auto evs = pump(s, [&](const LiveEvent& e) { interrupted = interrupted || is(e, LiveEventKind::Interrupted); return interrupted && turn_done(e); }, 40000); show(a.label, "interrupted", interrupted ? "yes" : "no"); ASSERT_TRUE(interrupted); s.close(); } } TEST(live_net, resume_on_the_same_session_object) { for (auto& a : auths("live_net.resume_on_the_same_session_object")) { LiveConfig cfg = base_config(); cfg.session_resumption = true; LiveSession s(a.endpoint, cfg); s.connect(); s.send_turn("Remember this secret word: mango. Reply with just OK.", true); pump(s, turn_done); for (int i = 0; i < 40 && s.resumption_handle().empty(); ++i) s.next_event(250); ASSERT_TRUE(!s.resumption_handle().empty()); s.resume(); // new connection, same object, handle in setup ASSERT_TRUE(s.open()); s.send_turn("What was the secret word? Answer with the word only.", true); bool answered = false; auto evs = pump(s, [&](const LiveEvent& e) { answered = answered || is(e, LiveEventKind::Audio) || (is(e, LiveEventKind::OutputTranscription) && !e.text.empty()); return answered && turn_done(e); }); std::string said = transcript(evs, LiveEventKind::OutputTranscription); show(a.label, "answer after resume()", said); ASSERT_TRUE(answered); s.close(); } } TEST(live_net, setup_options_vad_compression_media_resolution_are_accepted) { for (auto& a : auths("live_net.setup_options_vad_compression_media_resolution_are_accepted")) { LiveConfig cfg = base_config(); LiveActivityDetection vad; vad.start_sensitivity = "START_SENSITIVITY_LOW"; vad.end_sensitivity = "END_SENSITIVITY_LOW"; vad.prefix_padding_ms = 100; vad.silence_duration_ms = 700; cfg.activity_detection = vad; cfg.activity_handling = "START_OF_ACTIVITY_INTERRUPTS"; cfg.media_resolution = "MEDIA_RESOLUTION_LOW"; cfg.context_compression = true; cfg.compression_trigger_tokens = 20000; cfg.compression_target_tokens = 10000; LiveSession s(a.endpoint, cfg); s.connect(); // a rejected option closes the socket before setupComplete s.send_turn("Say OK.", true); auto evs = pump(s, turn_done); show(a.label, "kinds", kinds(evs).substr(0, 60)); ASSERT_TRUE(!evs.empty() && evs.back().kind == LiveEventKind::TurnComplete); s.close(); } } TEST(live_net, video_input_frame) { static const char* kJpegB64 = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAEAAQADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDz+iiivmD9yCiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKACiiigAooooAKKKKAP/Z"; std::string jpeg; ASSERT_TRUE(molo::security::crypto::Base64::decode(kJpegB64, jpeg)); for (auto& a : auths("live_net.video_input_frame")) { LiveSession s(a.endpoint, base_config()); s.connect(); for (int i = 0; i < 4; ++i) { // a short stream of 256x256 frames (a 64x64 frame was not seen by the model) s.send_video(jpeg, "image/jpeg"); std::this_thread::sleep_for(std::chrono::milliseconds(700)); } s.send_text("What color is the video frame I am streaming? Answer with one word."); // The model sometimes closes the first turn empty when frames and the // question race; ask again once. Two attempts, then report. bool answered = false; std::vector evs; for (int attempt = 0; attempt < 2 && !answered; ++attempt) { if (attempt) s.send_text("Again: what color is the video frame? One word."); evs = pump(s, [&](const LiveEvent& e) { answered = answered || is(e, LiveEventKind::Audio) || (is(e, LiveEventKind::OutputTranscription) && !e.text.empty()); return answered && turn_done(e); }, 20000); } show(a.label, "answer about the frame", transcript(evs, LiveEventKind::OutputTranscription)); for (const auto& e : evs) { ASSERT_TRUE(e.kind != LiveEventKind::Error); ASSERT_TRUE(e.kind != LiveEventKind::Closed); // frames were accepted, no server close } ASSERT_TRUE(answered); s.close(); } } TEST(live_net, go_away_before_the_connection_lifetime_ends) { const char* slow = std::getenv("MOLO_LIVE_SLOW"); if (!slow || std::string(slow) != "1") { std::printf(" SKIP live_net.go_away_before_the_connection_lifetime_ends: set MOLO_LIVE_SLOW=1 " "(waits up to 14 minutes idle for the server's goAway)\n"); return; } auto list = auths("live_net.go_away_before_the_connection_lifetime_ends"); if (list.empty()) return; Auth& a = list.front(); LiveConfig cfg = base_config(); cfg.session_resumption = true; LiveOptions opt; opt.max_session_seconds = 15 * 60; // lift our own 30 min cap below the server's LiveSession s(a.endpoint, cfg, opt); auto t0 = std::chrono::steady_clock::now(); s.connect(); auto secs = [&] { return std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); }; bool go_away = false; long go_away_at = -1; double left = -1; long closed_at = -1; int close_code = 0; auto evs = pump(s, [&](const LiveEvent& e) { if (e.kind == LiveEventKind::GoAway) { go_away = true; go_away_at = secs(); left = e.seconds; } if (e.kind == LiveEventKind::Closed) { closed_at = secs(); close_code = e.code; return true; } return false; }, 14 * 60 * 1000); std::printf(" [%s] goAway=%d at %lds timeLeft=%.1fs; closed at %lds code=%d; events=%zu\n", a.label.c_str(), go_away, go_away_at, left, closed_at, close_code, evs.size()); s.close(); ASSERT_TRUE(go_away || closed_at >= 0); }