feat(clients): live: Gemini Live API over WebSocket, RFC 6455 client, session, relay, discovery check
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,251 @@
|
||||
#include "../AgentDemos.hpp"
|
||||
|
||||
#include <cstdlib>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "../../fw/Validate.hpp"
|
||||
#include "clients/body/json.h"
|
||||
#include "google/credentials.h"
|
||||
#include "google/errors.h"
|
||||
#include "google/live/live_relay.h"
|
||||
#include "google/resource.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// Agent Platform: live. Stub: the module agent fills this in.
|
||||
// Agent Platform: live. The Gemini Live API is a WebSocket protocol; the
|
||||
// example server has none, so the browser talks HTTP and the server holds
|
||||
// the WebSocket (src/google/live): POST create, POST send, GET events by
|
||||
// cursor, POST close. Every route that would reach Google is a POST (or the
|
||||
// events GET of an existing session) and answers 503 google_not_configured
|
||||
// without a credential.
|
||||
|
||||
namespace {
|
||||
|
||||
using clients::body::Json;
|
||||
namespace live = google::live;
|
||||
|
||||
const char kLivePage[] = R"LIVE({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Live API</h1>
|
||||
<p class="hint">Bidirectional text, audio and video with Gemini over a
|
||||
WebSocket held by the server. Create a session, send a message, and read the
|
||||
events as they arrive.</p>
|
||||
<p id="l-config" class="hint">Checking Google credentials...</p>
|
||||
<section class="live">
|
||||
<div class="voice-controls">
|
||||
<button id="l-create" type="button" class="primary" disabled>Start session</button>
|
||||
<button id="l-close" type="button" disabled>End session</button>
|
||||
<span id="l-state" class="pill">no session</span>
|
||||
</div>
|
||||
<p id="l-error" class="error" role="alert"></p>
|
||||
<ol id="l-log" class="voice-log"></ol>
|
||||
<form id="l-form" class="voice-form">
|
||||
<input id="l-text" type="text" maxlength="500" placeholder="Type a message" autocomplete="off" disabled>
|
||||
<button type="submit" disabled>Send</button>
|
||||
</form>
|
||||
</section>
|
||||
<script src="/embedded/agent_live.js"></script>
|
||||
{% endblock %}
|
||||
)LIVE";
|
||||
|
||||
std::string env_or(const char* name, const char* fallback) {
|
||||
std::string v = google::Credentials::env(name);
|
||||
return v.empty() ? std::string(fallback) : v;
|
||||
}
|
||||
|
||||
[[noreturn]] void rethrow(const google::GoogleError& e) {
|
||||
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "google_not_configured", e.message());
|
||||
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
if (e.status() == "INVALID_ARGUMENT")
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
if (e.status() == "NOT_FOUND" && e.service() == "live")
|
||||
throw fw::HttpError(404, "not_found", e.message());
|
||||
if (e.status() == "RESOURCE_EXHAUSTED" && e.service() == "live")
|
||||
throw fw::HttpError(429, "too_many_sessions", e.message());
|
||||
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
auto guarded(F&& f) -> decltype(f()) {
|
||||
try {
|
||||
return f();
|
||||
} catch (const google::GoogleError& e) {
|
||||
rethrow(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
// Which credential form this process would use, from the environment:
|
||||
// OAuth token (Vertex), express key "AQ." (Vertex, x-goog-api-key), any
|
||||
// other key (Gemini API, ?key=).
|
||||
struct LiveAuth {
|
||||
std::string mode = "none";
|
||||
live::LiveEndpoint endpoint;
|
||||
};
|
||||
|
||||
LiveAuth resolve_auth() {
|
||||
LiveAuth out;
|
||||
std::string token = google::Credentials::env("GOOGLE_ACCESS_TOKEN");
|
||||
std::string key = google::Credentials::env("GOOGLE_API_KEY");
|
||||
google::CloudContext ctx = google::CloudContext::from_env();
|
||||
if (!token.empty() && !ctx.project.empty()) {
|
||||
out.mode = "vertex_oauth";
|
||||
out.endpoint = live::LiveEndpoint::vertex(ctx, token);
|
||||
} else if (!key.empty() && key.rfind("AQ.", 0) == 0 && !ctx.project.empty()) {
|
||||
out.mode = "vertex_express";
|
||||
out.endpoint = live::LiveEndpoint::vertex_express(ctx, key);
|
||||
} else if (!key.empty()) {
|
||||
out.mode = "gemini_api_key";
|
||||
out.endpoint = live::LiveEndpoint::api_key(key);
|
||||
}
|
||||
// Tests and proxies: GOOGLE_LIVE_URL replaces the WebSocket URL.
|
||||
out.endpoint.url_override = google::Credentials::env("GOOGLE_LIVE_URL");
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string field(const std::map<std::string, std::string>& in, const char* k) {
|
||||
auto it = in.find(k);
|
||||
return it == in.end() ? std::string() : it->second;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_agent_live_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
if (!services.google) services.google = google::GoogleClient::from_env();
|
||||
auto gclient = services.google;
|
||||
auto relay = std::make_shared<live::LiveRelay>();
|
||||
fw::Environment* env = services.templates.get();
|
||||
env->add_template("agent_live.html", kLivePage);
|
||||
|
||||
services.add_feature("agent-live", "/agent/live");
|
||||
|
||||
// The client's configured() gates every Google-bound route, then the
|
||||
// process environment supplies the credential the socket needs.
|
||||
auto require = [gclient]() {
|
||||
if (!gclient->configured())
|
||||
throw fw::HttpError(503, "google_not_configured",
|
||||
"set GOOGLE_API_KEY (or GOOGLE_ACCESS_TOKEN and GOOGLE_CLOUD_PROJECT)");
|
||||
LiveAuth auth = resolve_auth();
|
||||
if (auth.mode == "none")
|
||||
throw fw::HttpError(503, "google_not_configured",
|
||||
"the live API needs GOOGLE_API_KEY in the environment");
|
||||
return auth;
|
||||
};
|
||||
|
||||
router.get("/agent/live", "agent-live-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Live API");
|
||||
return fw::Response(200, env->render("agent_live.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/agent/live/status", "agent-live-status", [gclient, relay](fw::Request req) {
|
||||
(void)req;
|
||||
LiveAuth auth = resolve_auth();
|
||||
Json caps = Json::object();
|
||||
caps.set("max_sessions", (double)live::LiveCaps::kMaxRelaySessions);
|
||||
caps.set("max_text_chars", (double)live::LiveCaps::kMaxTextChars);
|
||||
caps.set("max_audio_chunk_bytes", (double)live::LiveCaps::kMaxAudioChunkBytes);
|
||||
caps.set("max_video_chunk_bytes", (double)live::LiveCaps::kMaxVideoChunkBytes);
|
||||
caps.set("max_session_seconds", (double)live::LiveCaps::kMaxSessionSeconds);
|
||||
caps.set("max_page_events", (double)live::LiveCaps::kMaxRelayPage);
|
||||
Json o = Json::object();
|
||||
o.set("configured", gclient->configured() && auth.mode != "none");
|
||||
o.set("auth_mode", auth.mode);
|
||||
o.set("default_model", live::LiveConfig{}.model);
|
||||
o.set("sessions", (double)relay->size());
|
||||
o.set("caps", caps);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
// Body: {model?, system?, modality? AUDIO|TEXT, voice?, lang?, transcribe? "1"}.
|
||||
router.post("/agent/live/sessions", "agent-live-create", [relay, require](fw::Request req) {
|
||||
LiveAuth auth = require();
|
||||
auto in = fw::parse_json_object_flat(req.body);
|
||||
live::LiveConfig cfg;
|
||||
if (!field(in, "model").empty()) cfg.model = field(in, "model");
|
||||
cfg.system_instruction = field(in, "system");
|
||||
cfg.voice_name = field(in, "voice");
|
||||
cfg.language_code = field(in, "lang");
|
||||
if (!field(in, "modality").empty()) cfg.response_modalities = {field(in, "modality")};
|
||||
cfg.input_transcription = cfg.output_transcription = field(in, "transcribe") == "1";
|
||||
return guarded([&] {
|
||||
std::string id = relay->create(auth.endpoint, cfg);
|
||||
Json o = Json::object();
|
||||
o.set("id", id);
|
||||
o.set("auth_mode", auth.mode);
|
||||
return json_out(o, 201);
|
||||
});
|
||||
});
|
||||
|
||||
// Body: {kind: text|audio|video|audio_end, text?, data_b64?, mime?}.
|
||||
router.post("/agent/live/sessions/{id}/send", "agent-live-send", [relay, require](fw::Request req) {
|
||||
require();
|
||||
auto in = fw::parse_json_object_flat(req.body);
|
||||
std::string kind = field(in, "kind");
|
||||
if (kind != "text" && kind != "audio" && kind != "video" && kind != "audio_end")
|
||||
throw fw::HttpError(422, "validation_error", "kind must be text, audio, video or audio_end");
|
||||
std::string bytes;
|
||||
if (kind == "audio" || kind == "video") {
|
||||
if (!security::crypto::Base64::decode(field(in, "data_b64"), bytes))
|
||||
throw fw::HttpError(422, "validation_error", "data_b64 is not valid base64");
|
||||
}
|
||||
return guarded([&] {
|
||||
auto session = relay->get(req.param_or("id", ""));
|
||||
if (kind == "text") session->send_text(field(in, "text"));
|
||||
else if (kind == "audio_end") session->end_audio();
|
||||
else if (kind == "audio")
|
||||
field(in, "mime").empty() ? session->send_audio(bytes) : session->send_audio(bytes, field(in, "mime"));
|
||||
else
|
||||
field(in, "mime").empty() ? session->send_video(bytes) : session->send_video(bytes, field(in, "mime"));
|
||||
Json o = Json::object();
|
||||
o.set("sent", true);
|
||||
return json_out(o);
|
||||
});
|
||||
});
|
||||
|
||||
// Query: cursor (last seq seen, default 0), max (default 50).
|
||||
router.get("/agent/live/sessions/{id}/events", "agent-live-events", [relay](fw::Request req) {
|
||||
std::string c = req.query_or("cursor", "0"), m = req.query_or("max", "50");
|
||||
auto numeric = [](const std::string& s) {
|
||||
if (s.empty() || s.size() > 12) return false;
|
||||
for (char ch : s)
|
||||
if (ch < '0' || ch > '9') return false;
|
||||
return true;
|
||||
};
|
||||
if (!numeric(c) || !numeric(m))
|
||||
throw fw::HttpError(422, "validation_error", "cursor and max must be non-negative integers");
|
||||
return guarded([&] {
|
||||
live::RelayPage page = relay->poll(req.param_or("id", ""), std::stoull(c), std::stoull(m));
|
||||
Json events = Json::array();
|
||||
for (const auto& e : page.events) events.push(live::event_to_json(e));
|
||||
Json o = Json::object();
|
||||
o.set("open", page.open);
|
||||
o.set("next_cursor", (double)page.next_cursor);
|
||||
o.set("dropped", (double)page.dropped);
|
||||
o.set("events", events);
|
||||
return json_out(o);
|
||||
});
|
||||
});
|
||||
|
||||
router.post("/agent/live/sessions/{id}/close", "agent-live-close", [relay, require](fw::Request req) {
|
||||
require();
|
||||
return guarded([&] {
|
||||
relay->close(req.param_or("id", ""));
|
||||
Json o = Json::object();
|
||||
o.set("closed", true);
|
||||
return json_out(o);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
// Live API page: HTTP relay to a server-held WebSocket session.
|
||||
// POST /agent/live/sessions -> id; POST .../send; GET .../events?cursor=N;
|
||||
// POST .../close. Model audio (PCM16 mono, rate from the mime) is played as it arrives.
|
||||
(function () {
|
||||
"use strict";
|
||||
var POLL_MS = 400;
|
||||
var MAX_LOG = 200;
|
||||
|
||||
var $ = function (id) { return document.getElementById(id); };
|
||||
var createBtn = $("l-create"), closeBtn = $("l-close"), stateEl = $("l-state");
|
||||
var errorEl = $("l-error"), logEl = $("l-log"), form = $("l-form"), textEl = $("l-text");
|
||||
var sendBtn = form.querySelector("button");
|
||||
var sid = null, cursor = 0, timer = null, polling = false;
|
||||
var audioCtx = null, playAt = 0;
|
||||
|
||||
function showError(msg) { errorEl.textContent = msg || ""; }
|
||||
function setState(s) { stateEl.textContent = s; stateEl.className = "pill " + (sid ? "listening" : ""); }
|
||||
function setActive(on) {
|
||||
createBtn.disabled = on; closeBtn.disabled = !on;
|
||||
textEl.disabled = !on; sendBtn.disabled = !on;
|
||||
}
|
||||
function addLog(who, text) {
|
||||
var li = document.createElement("li");
|
||||
li.className = who;
|
||||
var b = document.createElement("b");
|
||||
b.textContent = who + ": ";
|
||||
li.appendChild(b);
|
||||
li.appendChild(document.createTextNode(text));
|
||||
logEl.appendChild(li);
|
||||
while (logEl.children.length > MAX_LOG) logEl.removeChild(logEl.firstChild);
|
||||
li.scrollIntoView({ block: "nearest" });
|
||||
}
|
||||
function api(method, url, body) {
|
||||
return fetch(url, {
|
||||
method: method,
|
||||
headers: body ? { "Content-Type": "application/json" } : {},
|
||||
body: body ? JSON.stringify(body) : undefined
|
||||
}).then(function (res) {
|
||||
return res.text().then(function (t) {
|
||||
var j = {};
|
||||
try { j = JSON.parse(t); } catch (e) { /* keep empty */ }
|
||||
if (!res.ok) throw new Error(j.message || j.error || ("HTTP " + res.status));
|
||||
return j;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function play(b64, mime) {
|
||||
var m = /rate=(\d+)/.exec(mime || "");
|
||||
var rate = m ? parseInt(m[1], 10) : 24000;
|
||||
var raw = atob(b64), n = raw.length >> 1;
|
||||
if (!n) return;
|
||||
audioCtx = audioCtx || new (window.AudioContext || window.webkitAudioContext)();
|
||||
var buf = audioCtx.createBuffer(1, n, rate), ch = buf.getChannelData(0);
|
||||
for (var i = 0; i < n; i++) {
|
||||
var v = raw.charCodeAt(2 * i) | (raw.charCodeAt(2 * i + 1) << 8);
|
||||
ch[i] = (v > 32767 ? v - 65536 : v) / 32768;
|
||||
}
|
||||
var src = audioCtx.createBufferSource();
|
||||
src.buffer = buf; src.connect(audioCtx.destination);
|
||||
playAt = Math.max(playAt, audioCtx.currentTime);
|
||||
src.start(playAt); playAt += buf.duration;
|
||||
}
|
||||
|
||||
function onEvent(e) {
|
||||
switch (e.type) {
|
||||
case "text": addLog("model", e.text); break;
|
||||
case "output_transcription": if (e.text) addLog("model", e.text); break;
|
||||
case "input_transcription": if (e.text) addLog("you", e.text); break;
|
||||
case "audio": try { play(e.data_b64, e.mime); } catch (x) { showError("audio: " + x.message); } break;
|
||||
case "interrupted": playAt = 0; addLog("event", "interrupted"); break;
|
||||
case "turn_complete": setState("ready"); break;
|
||||
case "go_away": addLog("event", "server will disconnect in " + e.seconds + "s"); break;
|
||||
case "error": showError(e.text); break;
|
||||
case "closed": addLog("event", "closed (" + e.code + ") " + (e.text || "")); break;
|
||||
}
|
||||
}
|
||||
|
||||
function poll() {
|
||||
if (!sid || polling) return;
|
||||
polling = true;
|
||||
api("GET", "/agent/live/sessions/" + sid + "/events?cursor=" + cursor + "&max=100").then(function (j) {
|
||||
j.events.forEach(onEvent);
|
||||
cursor = j.next_cursor;
|
||||
if (!j.open && j.events.length === 0) endSession("session ended");
|
||||
}).catch(function (e) { showError(e.message); endSession("lost"); })
|
||||
.then(function () { polling = false; });
|
||||
}
|
||||
|
||||
function endSession(label) {
|
||||
if (timer) clearInterval(timer);
|
||||
timer = null; sid = null; setActive(false); setState(label || "no session");
|
||||
}
|
||||
|
||||
createBtn.addEventListener("click", function () {
|
||||
showError(""); createBtn.disabled = true; setState("connecting");
|
||||
api("POST", "/agent/live/sessions", { modality: "AUDIO", transcribe: "1" }).then(function (j) {
|
||||
sid = j.id; cursor = 0; setActive(true); setState("ready");
|
||||
timer = setInterval(poll, POLL_MS);
|
||||
}).catch(function (e) { showError(e.message); setActive(false); setState("no session"); });
|
||||
});
|
||||
|
||||
closeBtn.addEventListener("click", function () {
|
||||
var id = sid;
|
||||
endSession("closing");
|
||||
if (id) api("POST", "/agent/live/sessions/" + id + "/close", {}).catch(function (e) { showError(e.message); })
|
||||
.then(function () { setState("no session"); });
|
||||
});
|
||||
|
||||
form.addEventListener("submit", function (ev) {
|
||||
ev.preventDefault();
|
||||
var t = textEl.value.trim();
|
||||
if (!t || !sid) return;
|
||||
textEl.value = "";
|
||||
addLog("you", t);
|
||||
setState("thinking");
|
||||
api("POST", "/agent/live/sessions/" + sid + "/send", { kind: "text", text: t })
|
||||
.catch(function (e) { showError(e.message); });
|
||||
});
|
||||
|
||||
api("GET", "/agent/live/status").then(function (s) {
|
||||
$("l-config").textContent = s.configured
|
||||
? "Credentials found (" + s.auth_mode + "). Default model: " + s.default_model + "."
|
||||
: "No Google credential is configured: starting a session answers 503 google_not_configured.";
|
||||
createBtn.disabled = false;
|
||||
}).catch(function (e) { showError(e.message); });
|
||||
})();
|
||||
Reference in New Issue
Block a user