Files

247 lines
9.6 KiB
C++
Raw Permalink Normal View History

#include "../AgentDemos.hpp"
#include <cstdlib>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "../../fw/Validate.hpp"
#include "clients/body/json.h"
#include "google/credentials.h"
#include "google/errors.h"
#include "google/live/live_relay.h"
#include "google/resource.h"
#include "security/crypto/Base64.h"
namespace molo::example::app {
// Agent Platform: live. The Gemini Live API is a WebSocket protocol; the
// example server has none, so the browser talks HTTP and the server holds
// the WebSocket (src/google/live): POST create, POST send, GET events by
// cursor, POST close. Every route that would reach Google is a POST (or the
// events GET of an existing session) and answers 503 google_not_configured
// without a credential.
namespace {
using clients::body::Json;
namespace live = google::live;
const char kLivePage[] = R"LIVE({% extends "base.html" %}
{% block content %}
<h1>Live API</h1>
<p class="hint">Bidirectional text, audio and video with Gemini over a
WebSocket held by the server. Create a session, send a message, and read the
events as they arrive.</p>
<p id="l-config" class="hint">Checking Google credentials...</p>
<section class="live">
<div class="voice-controls">
<button id="l-create" type="button" class="primary" disabled>Start session</button>
<button id="l-close" type="button" disabled>End session</button>
<span id="l-state" class="pill">no session</span>
</div>
<p id="l-error" class="error" role="alert"></p>
<ol id="l-log" class="voice-log"></ol>
<form id="l-form" class="voice-form">
<input id="l-text" type="text" maxlength="500" placeholder="Type a message" autocomplete="off" disabled>
<button type="submit" disabled>Send</button>
</form>
</section>
<script src="/embedded/agent_live.js"></script>
{% endblock %}
)LIVE";
[[noreturn]] void rethrow(const google::GoogleError& e) {
if (e.http_status() == 503 && e.status() == "NOT_CONFIGURED")
throw fw::HttpError(503, "google_not_configured", e.message());
if (e.service().empty() && e.status() == "INVALID_ARGUMENT")
throw fw::HttpError(422, "validation_error", e.message());
if (e.status() == "INVALID_ARGUMENT")
throw fw::HttpError(422, "validation_error", e.message());
if (e.status() == "NOT_FOUND" && e.service() == "live")
throw fw::HttpError(404, "not_found", e.message());
if (e.status() == "RESOURCE_EXHAUSTED" && e.service() == "live")
throw fw::HttpError(429, "too_many_sessions", e.message());
throw fw::HttpError(502, "google_error", e.status() + ": " + e.message());
}
template <typename F>
auto guarded(F&& f) -> decltype(f()) {
try {
return f();
} catch (const google::GoogleError& e) {
rethrow(e);
}
}
fw::Response json_out(const Json& j, int status = 200) {
fw::Response res = fw::JsonResponse(j.dump(), status);
res.headers.set("Cache-Control", "no-store");
return res;
}
// Which credential form this process would use, from the environment:
// OAuth token (Vertex), express key "AQ." (Vertex, x-goog-api-key), any
// other key (Gemini API, ?key=).
struct LiveAuth {
std::string mode = "none";
live::LiveEndpoint endpoint;
};
LiveAuth resolve_auth() {
LiveAuth out;
std::string token = google::Credentials::env("GOOGLE_ACCESS_TOKEN");
std::string key = google::Credentials::env("GOOGLE_API_KEY");
google::CloudContext ctx = google::CloudContext::from_env();
if (!token.empty() && !ctx.project.empty()) {
out.mode = "vertex_oauth";
out.endpoint = live::LiveEndpoint::vertex(ctx, token);
} else if (!key.empty() && key.rfind("AQ.", 0) == 0 && !ctx.project.empty()) {
out.mode = "vertex_express";
out.endpoint = live::LiveEndpoint::vertex_express(ctx, key);
} else if (!key.empty()) {
out.mode = "gemini_api_key";
out.endpoint = live::LiveEndpoint::api_key(key);
}
// Tests and proxies: GOOGLE_LIVE_URL replaces the WebSocket URL.
out.endpoint.url_override = google::Credentials::env("GOOGLE_LIVE_URL");
return out;
}
std::string field(const std::map<std::string, std::string>& in, const char* k) {
auto it = in.find(k);
return it == in.end() ? std::string() : it->second;
}
} // namespace
void register_agent_live_demo(fw::Router& router, Services& services) {
if (!services.google) services.google = google::GoogleClient::from_env();
auto gclient = services.google;
auto relay = std::make_shared<live::LiveRelay>();
fw::Environment* env = services.templates.get();
env->add_template("agent_live.html", kLivePage);
services.add_feature("agent-live", "/agent/live");
// The client's configured() gates every Google-bound route, then the
// process environment supplies the credential the socket needs.
auto require = [gclient]() {
if (!gclient->configured())
throw fw::HttpError(503, "google_not_configured",
"set GOOGLE_API_KEY (or GOOGLE_ACCESS_TOKEN and GOOGLE_CLOUD_PROJECT)");
LiveAuth auth = resolve_auth();
if (auth.mode == "none")
throw fw::HttpError(503, "google_not_configured",
"the live API needs GOOGLE_API_KEY in the environment");
return auth;
};
router.get("/agent/live", "agent-live-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Live API");
return fw::Response(200, env->render("agent_live.html", ctx), "text/html; charset=utf-8");
});
router.get("/agent/live/status", "agent-live-status", [gclient, relay](fw::Request req) {
(void)req;
LiveAuth auth = resolve_auth();
Json caps = Json::object();
caps.set("max_sessions", (double)live::LiveCaps::kMaxRelaySessions);
caps.set("max_text_chars", (double)live::LiveCaps::kMaxTextChars);
caps.set("max_audio_chunk_bytes", (double)live::LiveCaps::kMaxAudioChunkBytes);
caps.set("max_video_chunk_bytes", (double)live::LiveCaps::kMaxVideoChunkBytes);
caps.set("max_session_seconds", (double)live::LiveCaps::kMaxSessionSeconds);
caps.set("max_page_events", (double)live::LiveCaps::kMaxRelayPage);
Json o = Json::object();
o.set("configured", gclient->configured() && auth.mode != "none");
o.set("auth_mode", auth.mode);
o.set("default_model", live::LiveConfig{}.model);
o.set("sessions", (double)relay->size());
o.set("caps", caps);
return json_out(o);
});
// Body: {model?, system?, modality? AUDIO|TEXT, voice?, lang?, transcribe? "1"}.
router.post("/agent/live/sessions", "agent-live-create", [relay, require](fw::Request req) {
LiveAuth auth = require();
auto in = fw::parse_json_object_flat(req.body);
live::LiveConfig cfg;
if (!field(in, "model").empty()) cfg.model = field(in, "model");
cfg.system_instruction = field(in, "system");
cfg.voice_name = field(in, "voice");
cfg.language_code = field(in, "lang");
if (!field(in, "modality").empty()) cfg.response_modalities = {field(in, "modality")};
cfg.input_transcription = cfg.output_transcription = field(in, "transcribe") == "1";
return guarded([&] {
std::string id = relay->create(auth.endpoint, cfg);
Json o = Json::object();
o.set("id", id);
o.set("auth_mode", auth.mode);
return json_out(o, 201);
});
});
// Body: {kind: text|audio|video|audio_end, text?, data_b64?, mime?}.
router.post("/agent/live/sessions/{id}/send", "agent-live-send", [relay, require](fw::Request req) {
require();
auto in = fw::parse_json_object_flat(req.body);
std::string kind = field(in, "kind");
if (kind != "text" && kind != "audio" && kind != "video" && kind != "audio_end")
throw fw::HttpError(422, "validation_error", "kind must be text, audio, video or audio_end");
std::string bytes;
if (kind == "audio" || kind == "video") {
if (!security::crypto::Base64::decode(field(in, "data_b64"), bytes))
throw fw::HttpError(422, "validation_error", "data_b64 is not valid base64");
}
return guarded([&] {
auto session = relay->get(req.param_or("id", ""));
if (kind == "text") session->send_text(field(in, "text"));
else if (kind == "audio_end") session->end_audio();
else if (kind == "audio")
field(in, "mime").empty() ? session->send_audio(bytes) : session->send_audio(bytes, field(in, "mime"));
else
field(in, "mime").empty() ? session->send_video(bytes) : session->send_video(bytes, field(in, "mime"));
Json o = Json::object();
o.set("sent", true);
return json_out(o);
});
});
// Query: cursor (last seq seen, default 0), max (default 50).
router.get("/agent/live/sessions/{id}/events", "agent-live-events", [relay](fw::Request req) {
std::string c = req.query_or("cursor", "0"), m = req.query_or("max", "50");
auto numeric = [](const std::string& s) {
if (s.empty() || s.size() > 12) return false;
for (char ch : s)
if (ch < '0' || ch > '9') return false;
return true;
};
if (!numeric(c) || !numeric(m))
throw fw::HttpError(422, "validation_error", "cursor and max must be non-negative integers");
return guarded([&] {
live::RelayPage page = relay->poll(req.param_or("id", ""), std::stoull(c), std::stoull(m));
Json events = Json::array();
for (const auto& e : page.events) events.push(live::event_to_json(e));
Json o = Json::object();
o.set("open", page.open);
o.set("next_cursor", (double)page.next_cursor);
o.set("dropped", (double)page.dropped);
o.set("events", events);
return json_out(o);
});
});
router.post("/agent/live/sessions/{id}/close", "agent-live-close", [relay, require](fw::Request req) {
require();
return guarded([&] {
relay->close(req.param_or("id", ""));
Json o = Json::object();
o.set("closed", true);
return json_out(o);
});
});
}
} // namespace molo::example::app