Files
moloapp/src/xai/platform/realtime_session.h
T

104 lines
4.4 KiB
C++

#pragma once
// Realtime voice agent session (wss://api.x.ai/v1/realtime): connect, wait
// for session.created, configure with SessionConfig, stream audio in, read
// events out, run tool calls. One thread reads (next / run_response); any
// thread may send. The credential is an xAI API key (server side) or an
// ephemeral client secret from RealtimeRest::client_secret.
#include <deque>
#include <memory>
#include <string>
#include <vector>
#include "xai/platform/session_config.h"
#include "xai/platform/socket.h"
namespace molo::xai::platform {
struct RealtimeOptions {
std::string url; // "" = wss://api.x.ai/v1/realtime (tests point at loopback)
std::string model; // query: "" | grok-voice-latest | ...
std::string reasoning_effort; // query reasoning.effort: "" | high | none
std::string call_id; // SIP call (API key only; client secrets are refused)
std::string conversation_id; // session resumption (guide)
SocketAuth auth = SocketAuth::kBearer;
SocketOptions socket;
int setup_timeout_ms = PlatformCaps::kSetupTimeoutMs;
};
struct FunctionCall {
std::string call_id, name, arguments, item_id;
};
// What one assistant turn produced, folded from its events.
struct RealtimeResponse {
std::string response_id;
std::string status; // completed | cancelled | incomplete
std::string transcript; // response.output_audio_transcript.delta joined
std::string text; // response.(output_)text.delta joined
std::string audio; // decoded audio bytes (json or binary transport)
std::vector<FunctionCall> calls; // response.function_call_arguments.done
Json usage; // response.done.response.usage
std::vector<WsEvent> events; // every event, in order (bounded)
bool truncated = false; // event or audio cap hit
std::vector<std::string> errors; // error events (type: message)
};
class RealtimeSession {
public:
RealtimeSession(std::string token, RealtimeOptions options = {});
// "wss://api.x.ai/v1/realtime?model=..." as it will be opened (no credential).
std::string url() const;
// Opens the socket and waits for session.created (events before it are kept
// in setup_events). Throws llm::ApiError.
void connect();
void close();
bool is_open() const { return socket_.is_open(); }
const std::vector<WsEvent>& setup_events() const { return setup_events_; }
// conversation.created.conversation.id, "" before it arrived.
const std::string& conversation_id() const { return conversation_id_; }
// Client messages.
void update_session(const SessionConfig& config);
// PCM/codec bytes; sent as base64 JSON, or a binary frame when
// set_input_binary(true) (match audio.input.transport = "binary").
void append_audio(const std::string& bytes);
void set_input_binary(bool on) { input_binary_ = on; }
void commit_audio(); // manual turns only (turn_detection null)
void clear_audio();
void add_user_text(const std::string& text, const std::string& previous_item_id = "");
void seed_assistant_text(const std::string& text);
void add_function_output(const std::string& call_id, const std::string& output);
void add_function_call_history(const std::string& name, const std::string& arguments,
const std::string& call_id);
// Hard coded TTS line without the model (SpaceXSI extension).
void force_message(const std::string& text, bool interruptible = true);
void delete_item(const std::string& item_id);
void truncate_item(const std::string& item_id, int content_index, int audio_end_ms);
void create_response(const std::string& instructions = "");
void cancel_response(const std::string& response_id = "");
// Server events.
std::optional<WsEvent> next(int wait_ms);
// Reads until response.done (or error / close / wait elapsed) and folds
// the events. max_events bounds the loop.
RealtimeResponse run_response(int max_wait_ms,
std::size_t max_events = PlatformCaps::kMaxEventsPerWait);
const XaiSocket::Counters& counters() const { return socket_.counters(); }
private:
void send(Json message);
std::string token_;
RealtimeOptions options_;
XaiSocket socket_;
std::vector<WsEvent> setup_events_;
std::deque<WsEvent> pending_; // read ahead while settling the two setup events
std::string conversation_id_;
bool input_binary_ = false;
};
} // namespace molo::xai::platform