#include "meta/speech.h" #include #include "google/live/ws_error.h" #include "google/live/ws_url.h" #include "meta/calls.h" #include "meta/caps.h" #include "meta/check.h" #include "meta/client.h" namespace molo::meta { namespace { long le32(const std::string& s, std::size_t p) { return static_cast(static_cast(s[p])) | (static_cast(static_cast(s[p + 1])) << 8) | (static_cast(static_cast(s[p + 2])) << 16) | (static_cast(static_cast(s[p + 3])) << 24); } int le16(const std::string& s, std::size_t p) { return static_cast(s[p]) | (static_cast(s[p + 1]) << 8); } long now_ms() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()).count(); } } // namespace void SpeechOptions::validate() const { Check::model_id(model); if (!mode.empty()) Check::one_of("mode", mode, {"PUSH_TO_TALK", "ENDPOINTING", "DIARIZATION"}); if (!partial_mode.empty()) Check::one_of("partialMode", partial_mode, {"CUMULATIVE", "DELTA"}); if (keywords.size() > MetaCaps::kMaxKeywords) Check::fail("too many keywords"); for (const std::string& k : keywords) { Check::non_empty("keyword", k); Check::max_bytes("keyword", k, 256); } if (language_bias.size() > 25) Check::fail("languageBias takes at most 25 languages"); for (const std::string& l : language_bias) { Check::non_empty("language", l); Check::max_bytes("language", l, 64); } if (!session_id.empty()) Check::segment("sessionId", session_id); } Json SpeechOptions::settings_json(const std::string& audio_encoding) const { Json j = Json::object(); j.set("model", Json(model)); j.set("audioEncoding", Json(audio_encoding)); JsonX::set_if(j, "mode", mode); JsonX::set_if(j, "partialMode", partial_mode); if (!keywords.empty()) j.set("keywords", JsonX::strings(keywords)); if (!language_bias.empty()) j.set("languageBias", JsonX::strings(language_bias)); if (emit_audio_progress) j.set("emitAudioProgress", Json(true)); return j; } WavInfo WavInfo::parse(const std::string& w) { if (w.size() < 44 || w.compare(0, 4, "RIFF") != 0 || w.compare(8, 4, "WAVE") != 0) Check::fail("audio is not a RIFF/WAVE file"); if (w.size() > MetaCaps::kMaxTranscribeBytes) Check::fail("audio is over the 32 MB request limit"); WavInfo info; bool have_fmt = false, have_data = false; std::size_t pos = 12; while (pos + 8 <= w.size()) { std::string id = w.substr(pos, 4); long size = le32(w, pos + 4); std::size_t body = pos + 8; if (id == "fmt " && body + 16 <= w.size()) { int format = le16(w, body); info.channels = le16(w, body + 2); info.sample_rate = static_cast(le32(w, body + 4)); info.bits = le16(w, body + 14); if (format != 1) Check::fail("WAV must be integer PCM"); have_fmt = true; } else if (id == "data") { long avail = static_cast(w.size() - body); info.data_bytes = size > avail || size < 0 ? avail : size; have_data = true; break; } if (size < 0) break; pos = body + static_cast(size) + (static_cast(size) & 1u); } if (!have_fmt || !have_data) Check::fail("WAV is missing its fmt or data chunk"); if (info.channels != 1 || info.bits != 16) Check::fail("WAV must be mono 16-bit PCM"); if (info.sample_rate != 16000 && info.sample_rate != 24000) Check::fail("WAV must be 16 kHz or 24 kHz"); if (info.data_bytes <= 0) Check::fail("WAV holds no audio"); info.duration_ms = info.data_bytes * 1000L / (static_cast(info.sample_rate) * 2L); if (info.duration_ms > MetaCaps::kMaxTranscribeMs) Check::fail("audio is longer than 10 minutes"); return info; } TranscribeResult TranscribeResult::from_json(const Json& j) { TranscribeResult r; r.session_id = JsonX::str(j, "sessionId"); r.transcript = JsonX::str(j, "transcript"); r.audio_duration_ms = JsonX::num(j, "audioDurationMs"); if (const Json* t = JsonX::arr(j, "turns")) for (const Json& x : t->as_array()) { SpeechTurn turn; turn.turn_id = JsonX::num(x, "turnId"); turn.start_ms = JsonX::num(x, "startMs"); turn.end_ms = JsonX::num(x, "endMs"); turn.transcript = JsonX::str(x, "transcript"); turn.speaker = JsonX::str(x, "speaker"); r.turns.push_back(std::move(turn)); } return r; } const std::vector& SpeechEvent::known_types() { static const std::vector v = {"transcript", "speechStart", "speechEnd", "speechComplete", "speaker", "audioProgress", "error"}; return v; } SpeechEvent SpeechEvent::parse(const Json& j) { SpeechEvent e; e.type = JsonX::str(j, "type"); e.transcript = JsonX::str(j, "transcript"); e.final_flag = JsonX::flag(j, "final"); e.audio_processed_ms = JsonX::num(j, "audioProcessedMs"); e.turn_id = JsonX::num(j, "turnId", -1); e.speaker_label = JsonX::str(j, "label"); e.session_id = JsonX::str(j, "sessionId"); e.message = JsonX::str(j, "message"); e.error_type = JsonX::str(j, "errorType"); e.error_code = JsonX::str(j, "errorCode"); return e; } llm::RawResponse TranscribeService::post(const SpeechOptions& options, const std::string& wav, const char* accept) { options.validate(); WavInfo::parse(wav); // `request` is a form field with a JSON content type and no filename (the // docs' curl: -F 'request={...};type=application/json'); audio is the file. const std::string boundary = clients::body::MultipartBody().boundary(); std::string body; body += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"request\"\r\n"; body += "Content-Type: application/json\r\n\r\n" + options.settings_json("WAV").dump() + "\r\n"; body += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"audio\"; filename=\"audio.wav\"\r\n"; body += "Content-Type: audio/wav\r\n\r\n" + wav + "\r\n--" + boundary + "--\r\n"; std::string path = path_for(Call::kAsrTranscribe); if (!options.session_id.empty()) path += "?sessionId=" + options.session_id; return client_->request_raw(clients::Method::kPost, path, &body, "multipart/form-data; boundary=" + boundary, {{"Accept", accept}}); } TranscribeResult TranscribeService::transcribe(const SpeechOptions& options, const std::string& wav) { llm::RawResponse raw = post(options, wav, "application/json"); try { return TranscribeResult::from_json(Json::parse(raw.body)); } catch (const std::exception& e) { throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what()); } } std::vector TranscribeService::transcribe_events(const SpeechOptions& options, const std::string& wav) { llm::RawResponse raw = post(options, wav, "text/event-stream"); std::vector out; llm::SseDecoder decoder("meta", [&](const llm::SseEvent& e) { if (out.size() >= MetaCaps::kMaxRealtimeEvents) return false; SpeechEvent ev = SpeechEvent::parse(e.data); if (ev.type.empty()) ev.type = e.event; out.push_back(std::move(ev)); return true; }); decoder.feed(raw.body); decoder.finish(); return out; } std::string TranscribeService::transcribe_text(const SpeechOptions& options, const std::string& wav) { return post(options, wav, "text/plain").body; } void RealtimeConfig::validate() const { options.validate(); Check::one_of("audioEncoding", audio_encoding, {"PCM_16KHZ", "PCM_24KHZ"}); } RealtimeSession::RealtimeSession(RealtimeConfig config) : config_(std::move(config)) { if (config_.url.empty()) config_.url = MetaEndpoints::kRealtimeUrl; if (config_.key.empty()) config_.key = MetaClient::key_from_env(); } RealtimeSession::~RealtimeSession() { try { close(); } catch (...) { } } Json RealtimeSession::handshake_json(bool reveal) const { Json j = config_.options.settings_json(config_.audio_encoding); Json auth = Json::object(); auth.set("accessToken", Json("Bearer " + (reveal ? config_.key : std::string("***")))); j.set("authorization", auth); return j; } void RealtimeSession::connect() { config_.validate(); if (open_) Check::fail("session already connected"); if (config_.key.empty()) throw llm::NotConfiguredError("meta", MetaEndpoints::kKeyEnv); using namespace google::live; try { WsUrl url = WsUrl::parse(config_.url); if (!config_.options.session_id.empty()) url.target += (url.target.find('?') == std::string::npos ? "?" : "&") + std::string("sessionId=") + config_.options.session_id; ws_ = std::make_unique( open_stream(url, config_.connect_timeout_ms, MetaCaps::kRealtimeIdleMs, config_.tls)); ws_->handshake(url, {}, config_.handshake_timeout_ms); // no Authorization header ws_->send_text(handshake_json(true).dump()); ws_->set_read_timeout_ms(250); long deadline = now_ms() + config_.ack_timeout_ms; while (session_id_.empty()) { if (now_ms() > deadline) throw llm::ApiError("meta", 504, "TIMEOUT", "handshake acknowledgement not received"); auto m = ws_->receive(); if (!m) continue; if (m->kind == WsMessage::Kind::Close) { closed_ = true; close_code_ = m->code; throw llm::ApiError("meta", m->code == 1013 ? 429 : 400, "REALTIME_CLOSED", "closed before the acknowledgement (code " + std::to_string(m->code) + "): " + m->data.substr(0, 300)); } Json j = Json::parse(m->data); if (JsonX::str(j, "type") == "error") throw llm::ApiError("meta", 400, JsonX::str(j, "errorType", "error"), JsonX::str(j, "message")); session_id_ = JsonX::str(j, "sessionId"); if (session_id_.empty()) throw llm::ApiError("meta", 502, "BAD_RESPONSE", "acknowledgement has no sessionId"); } open_ = true; } catch (const WsError& e) { if (ws_) ws_->shutdown(); ws_.reset(); throw llm::ApiError("meta", e.http_status() ? e.http_status() : 502, "UNAVAILABLE", e.what()); } } void RealtimeSession::send_audio(const std::string& pcm) { if (!open_) Check::fail("session is not open"); if (pcm.empty() || pcm.size() > MetaCaps::kMaxRealtimeAudioFrame) Check::fail("audio frame must be 1.." + std::to_string(MetaCaps::kMaxRealtimeAudioFrame) + " bytes"); if (pcm.size() % 2 != 0) Check::fail("PCM frames are 16-bit samples"); try { ws_->send_binary(pcm); } catch (const google::live::WsError& e) { throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what()); } audio_bytes_ += pcm.size(); } void RealtimeSession::end_stream() { if (!open_) Check::fail("session is not open"); Json j = Json::object(); j.set("type", Json("endStream")); try { ws_->send_text(j.dump()); } catch (const google::live::WsError& e) { throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what()); } } std::optional RealtimeSession::next(int timeout_ms) { if (!ws_ || closed_) return std::nullopt; ws_->set_read_timeout_ms(timeout_ms < 1 ? 1 : timeout_ms); try { auto m = ws_->receive(); if (!m) return std::nullopt; if (m->kind == google::live::WsMessage::Kind::Close) { closed_ = true; close_code_ = m->code; open_ = false; SpeechEvent e; e.type = "close"; e.message = m->data; e.audio_processed_ms = m->code; return e; } if (++events_ > MetaCaps::kMaxRealtimeEvents) { close(); throw llm::ApiError("meta", 502, "TOO_MANY_EVENTS", "event cap reached"); } return SpeechEvent::parse(Json::parse(m->data)); } catch (const google::live::WsError& e) { throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what()); } catch (const clients::body::JsonError& e) { throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what()); } } void RealtimeSession::close() { if (ws_) { try { ws_->close(1000, ""); } catch (...) { } ws_->shutdown(); ws_.reset(); } open_ = false; closed_ = true; } } // namespace molo::meta