104 lines
4.4 KiB
C++
104 lines
4.4 KiB
C++
#pragma once
|
|
|
|
// Realtime voice agent session (wss://api.x.ai/v1/realtime): connect, wait
|
|
// for session.created, configure with SessionConfig, stream audio in, read
|
|
// events out, run tool calls. One thread reads (next / run_response); any
|
|
// thread may send. The credential is an xAI API key (server side) or an
|
|
// ephemeral client secret from RealtimeRest::client_secret.
|
|
|
|
#include <deque>
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "xai/platform/session_config.h"
|
|
#include "xai/platform/socket.h"
|
|
|
|
namespace molo::xai::platform {
|
|
|
|
struct RealtimeOptions {
|
|
std::string url; // "" = wss://api.x.ai/v1/realtime (tests point at loopback)
|
|
std::string model; // query: "" | grok-voice-latest | ...
|
|
std::string reasoning_effort; // query reasoning.effort: "" | high | none
|
|
std::string call_id; // SIP call (API key only; client secrets are refused)
|
|
std::string conversation_id; // session resumption (guide)
|
|
SocketAuth auth = SocketAuth::kBearer;
|
|
SocketOptions socket;
|
|
int setup_timeout_ms = PlatformCaps::kSetupTimeoutMs;
|
|
};
|
|
|
|
struct FunctionCall {
|
|
std::string call_id, name, arguments, item_id;
|
|
};
|
|
|
|
// What one assistant turn produced, folded from its events.
|
|
struct RealtimeResponse {
|
|
std::string response_id;
|
|
std::string status; // completed | cancelled | incomplete
|
|
std::string transcript; // response.output_audio_transcript.delta joined
|
|
std::string text; // response.(output_)text.delta joined
|
|
std::string audio; // decoded audio bytes (json or binary transport)
|
|
std::vector<FunctionCall> calls; // response.function_call_arguments.done
|
|
Json usage; // response.done.response.usage
|
|
std::vector<WsEvent> events; // every event, in order (bounded)
|
|
bool truncated = false; // event or audio cap hit
|
|
std::vector<std::string> errors; // error events (type: message)
|
|
};
|
|
|
|
class RealtimeSession {
|
|
public:
|
|
RealtimeSession(std::string token, RealtimeOptions options = {});
|
|
|
|
// "wss://api.x.ai/v1/realtime?model=..." as it will be opened (no credential).
|
|
std::string url() const;
|
|
// Opens the socket and waits for session.created (events before it are kept
|
|
// in setup_events). Throws llm::ApiError.
|
|
void connect();
|
|
void close();
|
|
bool is_open() const { return socket_.is_open(); }
|
|
const std::vector<WsEvent>& setup_events() const { return setup_events_; }
|
|
// conversation.created.conversation.id, "" before it arrived.
|
|
const std::string& conversation_id() const { return conversation_id_; }
|
|
|
|
// Client messages.
|
|
void update_session(const SessionConfig& config);
|
|
// PCM/codec bytes; sent as base64 JSON, or a binary frame when
|
|
// set_input_binary(true) (match audio.input.transport = "binary").
|
|
void append_audio(const std::string& bytes);
|
|
void set_input_binary(bool on) { input_binary_ = on; }
|
|
void commit_audio(); // manual turns only (turn_detection null)
|
|
void clear_audio();
|
|
void add_user_text(const std::string& text, const std::string& previous_item_id = "");
|
|
void seed_assistant_text(const std::string& text);
|
|
void add_function_output(const std::string& call_id, const std::string& output);
|
|
void add_function_call_history(const std::string& name, const std::string& arguments,
|
|
const std::string& call_id);
|
|
// Hard coded TTS line without the model (SpaceXSI extension).
|
|
void force_message(const std::string& text, bool interruptible = true);
|
|
void delete_item(const std::string& item_id);
|
|
void truncate_item(const std::string& item_id, int content_index, int audio_end_ms);
|
|
void create_response(const std::string& instructions = "");
|
|
void cancel_response(const std::string& response_id = "");
|
|
|
|
// Server events.
|
|
std::optional<WsEvent> next(int wait_ms);
|
|
// Reads until response.done (or error / close / wait elapsed) and folds
|
|
// the events. max_events bounds the loop.
|
|
RealtimeResponse run_response(int max_wait_ms,
|
|
std::size_t max_events = PlatformCaps::kMaxEventsPerWait);
|
|
|
|
const XaiSocket::Counters& counters() const { return socket_.counters(); }
|
|
|
|
private:
|
|
void send(Json message);
|
|
std::string token_;
|
|
RealtimeOptions options_;
|
|
XaiSocket socket_;
|
|
std::vector<WsEvent> setup_events_;
|
|
std::deque<WsEvent> pending_; // read ahead while settling the two setup events
|
|
std::string conversation_id_;
|
|
bool input_binary_ = false;
|
|
};
|
|
|
|
} // namespace molo::xai::platform
|