#pragma once // Realtime voice agent session (wss://api.x.ai/v1/realtime): connect, wait // for session.created, configure with SessionConfig, stream audio in, read // events out, run tool calls. One thread reads (next / run_response); any // thread may send. The credential is an xAI API key (server side) or an // ephemeral client secret from RealtimeRest::client_secret. #include #include #include #include #include "xai/platform/session_config.h" #include "xai/platform/socket.h" namespace molo::xai::platform { struct RealtimeOptions { std::string url; // "" = wss://api.x.ai/v1/realtime (tests point at loopback) std::string model; // query: "" | grok-voice-latest | ... std::string reasoning_effort; // query reasoning.effort: "" | high | none std::string call_id; // SIP call (API key only; client secrets are refused) std::string conversation_id; // session resumption (guide) SocketAuth auth = SocketAuth::kBearer; SocketOptions socket; int setup_timeout_ms = PlatformCaps::kSetupTimeoutMs; }; struct FunctionCall { std::string call_id, name, arguments, item_id; }; // What one assistant turn produced, folded from its events. struct RealtimeResponse { std::string response_id; std::string status; // completed | cancelled | incomplete std::string transcript; // response.output_audio_transcript.delta joined std::string text; // response.(output_)text.delta joined std::string audio; // decoded audio bytes (json or binary transport) std::vector calls; // response.function_call_arguments.done Json usage; // response.done.response.usage std::vector events; // every event, in order (bounded) bool truncated = false; // event or audio cap hit std::vector errors; // error events (type: message) }; class RealtimeSession { public: RealtimeSession(std::string token, RealtimeOptions options = {}); // "wss://api.x.ai/v1/realtime?model=..." as it will be opened (no credential). std::string url() const; // Opens the socket and waits for session.created (events before it are kept // in setup_events). Throws llm::ApiError. void connect(); void close(); bool is_open() const { return socket_.is_open(); } const std::vector& setup_events() const { return setup_events_; } // conversation.created.conversation.id, "" before it arrived. const std::string& conversation_id() const { return conversation_id_; } // Client messages. void update_session(const SessionConfig& config); // PCM/codec bytes; sent as base64 JSON, or a binary frame when // set_input_binary(true) (match audio.input.transport = "binary"). void append_audio(const std::string& bytes); void set_input_binary(bool on) { input_binary_ = on; } void commit_audio(); // manual turns only (turn_detection null) void clear_audio(); void add_user_text(const std::string& text, const std::string& previous_item_id = ""); void seed_assistant_text(const std::string& text); void add_function_output(const std::string& call_id, const std::string& output); void add_function_call_history(const std::string& name, const std::string& arguments, const std::string& call_id); // Hard coded TTS line without the model (SpaceXSI extension). void force_message(const std::string& text, bool interruptible = true); void delete_item(const std::string& item_id); void truncate_item(const std::string& item_id, int content_index, int audio_end_ms); void create_response(const std::string& instructions = ""); void cancel_response(const std::string& response_id = ""); // Server events. std::optional next(int wait_ms); // Reads until response.done (or error / close / wait elapsed) and folds // the events. max_events bounds the loop. RealtimeResponse run_response(int max_wait_ms, std::size_t max_events = PlatformCaps::kMaxEventsPerWait); const XaiSocket::Counters& counters() const { return socket_.counters(); } private: void send(Json message); std::string token_; RealtimeOptions options_; XaiSocket socket_; std::vector setup_events_; std::deque pending_; // read ahead while settling the two setup events std::string conversation_id_; bool input_binary_ = false; }; } // namespace molo::xai::platform