317 lines
12 KiB
C++
317 lines
12 KiB
C++
#include "meta/speech.h"
|
|
|
|
#include <chrono>
|
|
|
|
#include "google/live/ws_error.h"
|
|
#include "google/live/ws_url.h"
|
|
#include "meta/calls.h"
|
|
#include "meta/caps.h"
|
|
#include "meta/check.h"
|
|
#include "meta/client.h"
|
|
|
|
namespace molo::meta {
|
|
|
|
namespace {
|
|
long le32(const std::string& s, std::size_t p) {
|
|
return static_cast<long>(static_cast<unsigned char>(s[p])) |
|
|
(static_cast<long>(static_cast<unsigned char>(s[p + 1])) << 8) |
|
|
(static_cast<long>(static_cast<unsigned char>(s[p + 2])) << 16) |
|
|
(static_cast<long>(static_cast<unsigned char>(s[p + 3])) << 24);
|
|
}
|
|
int le16(const std::string& s, std::size_t p) {
|
|
return static_cast<unsigned char>(s[p]) | (static_cast<unsigned char>(s[p + 1]) << 8);
|
|
}
|
|
long now_ms() {
|
|
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now().time_since_epoch()).count();
|
|
}
|
|
} // namespace
|
|
|
|
void SpeechOptions::validate() const {
|
|
Check::model_id(model);
|
|
if (!mode.empty()) Check::one_of("mode", mode, {"PUSH_TO_TALK", "ENDPOINTING", "DIARIZATION"});
|
|
if (!partial_mode.empty()) Check::one_of("partialMode", partial_mode, {"CUMULATIVE", "DELTA"});
|
|
if (keywords.size() > MetaCaps::kMaxKeywords) Check::fail("too many keywords");
|
|
for (const std::string& k : keywords) {
|
|
Check::non_empty("keyword", k);
|
|
Check::max_bytes("keyword", k, 256);
|
|
}
|
|
if (language_bias.size() > 25) Check::fail("languageBias takes at most 25 languages");
|
|
for (const std::string& l : language_bias) {
|
|
Check::non_empty("language", l);
|
|
Check::max_bytes("language", l, 64);
|
|
}
|
|
if (!session_id.empty()) Check::segment("sessionId", session_id);
|
|
}
|
|
|
|
Json SpeechOptions::settings_json(const std::string& audio_encoding) const {
|
|
Json j = Json::object();
|
|
j.set("model", Json(model));
|
|
j.set("audioEncoding", Json(audio_encoding));
|
|
JsonX::set_if(j, "mode", mode);
|
|
JsonX::set_if(j, "partialMode", partial_mode);
|
|
if (!keywords.empty()) j.set("keywords", JsonX::strings(keywords));
|
|
if (!language_bias.empty()) j.set("languageBias", JsonX::strings(language_bias));
|
|
if (emit_audio_progress) j.set("emitAudioProgress", Json(true));
|
|
return j;
|
|
}
|
|
|
|
WavInfo WavInfo::parse(const std::string& w) {
|
|
if (w.size() < 44 || w.compare(0, 4, "RIFF") != 0 || w.compare(8, 4, "WAVE") != 0)
|
|
Check::fail("audio is not a RIFF/WAVE file");
|
|
if (w.size() > MetaCaps::kMaxTranscribeBytes) Check::fail("audio is over the 32 MB request limit");
|
|
WavInfo info;
|
|
bool have_fmt = false, have_data = false;
|
|
std::size_t pos = 12;
|
|
while (pos + 8 <= w.size()) {
|
|
std::string id = w.substr(pos, 4);
|
|
long size = le32(w, pos + 4);
|
|
std::size_t body = pos + 8;
|
|
if (id == "fmt " && body + 16 <= w.size()) {
|
|
int format = le16(w, body);
|
|
info.channels = le16(w, body + 2);
|
|
info.sample_rate = static_cast<int>(le32(w, body + 4));
|
|
info.bits = le16(w, body + 14);
|
|
if (format != 1) Check::fail("WAV must be integer PCM");
|
|
have_fmt = true;
|
|
} else if (id == "data") {
|
|
long avail = static_cast<long>(w.size() - body);
|
|
info.data_bytes = size > avail || size < 0 ? avail : size;
|
|
have_data = true;
|
|
break;
|
|
}
|
|
if (size < 0) break;
|
|
pos = body + static_cast<std::size_t>(size) + (static_cast<std::size_t>(size) & 1u);
|
|
}
|
|
if (!have_fmt || !have_data) Check::fail("WAV is missing its fmt or data chunk");
|
|
if (info.channels != 1 || info.bits != 16)
|
|
Check::fail("WAV must be mono 16-bit PCM");
|
|
if (info.sample_rate != 16000 && info.sample_rate != 24000)
|
|
Check::fail("WAV must be 16 kHz or 24 kHz");
|
|
if (info.data_bytes <= 0) Check::fail("WAV holds no audio");
|
|
info.duration_ms = info.data_bytes * 1000L / (static_cast<long>(info.sample_rate) * 2L);
|
|
if (info.duration_ms > MetaCaps::kMaxTranscribeMs) Check::fail("audio is longer than 10 minutes");
|
|
return info;
|
|
}
|
|
|
|
TranscribeResult TranscribeResult::from_json(const Json& j) {
|
|
TranscribeResult r;
|
|
r.session_id = JsonX::str(j, "sessionId");
|
|
r.transcript = JsonX::str(j, "transcript");
|
|
r.audio_duration_ms = JsonX::num(j, "audioDurationMs");
|
|
if (const Json* t = JsonX::arr(j, "turns"))
|
|
for (const Json& x : t->as_array()) {
|
|
SpeechTurn turn;
|
|
turn.turn_id = JsonX::num(x, "turnId");
|
|
turn.start_ms = JsonX::num(x, "startMs");
|
|
turn.end_ms = JsonX::num(x, "endMs");
|
|
turn.transcript = JsonX::str(x, "transcript");
|
|
turn.speaker = JsonX::str(x, "speaker");
|
|
r.turns.push_back(std::move(turn));
|
|
}
|
|
return r;
|
|
}
|
|
|
|
const std::vector<std::string>& SpeechEvent::known_types() {
|
|
static const std::vector<std::string> v = {"transcript", "speechStart", "speechEnd",
|
|
"speechComplete", "speaker", "audioProgress", "error"};
|
|
return v;
|
|
}
|
|
|
|
SpeechEvent SpeechEvent::parse(const Json& j) {
|
|
SpeechEvent e;
|
|
e.type = JsonX::str(j, "type");
|
|
e.transcript = JsonX::str(j, "transcript");
|
|
e.final_flag = JsonX::flag(j, "final");
|
|
e.audio_processed_ms = JsonX::num(j, "audioProcessedMs");
|
|
e.turn_id = JsonX::num(j, "turnId", -1);
|
|
e.speaker_label = JsonX::str(j, "label");
|
|
e.session_id = JsonX::str(j, "sessionId");
|
|
e.message = JsonX::str(j, "message");
|
|
e.error_type = JsonX::str(j, "errorType");
|
|
e.error_code = JsonX::str(j, "errorCode");
|
|
return e;
|
|
}
|
|
|
|
llm::RawResponse TranscribeService::post(const SpeechOptions& options, const std::string& wav,
|
|
const char* accept) {
|
|
options.validate();
|
|
WavInfo::parse(wav);
|
|
// `request` is a form field with a JSON content type and no filename (the
|
|
// docs' curl: -F 'request={...};type=application/json'); audio is the file.
|
|
const std::string boundary = clients::body::MultipartBody().boundary();
|
|
std::string body;
|
|
body += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"request\"\r\n";
|
|
body += "Content-Type: application/json\r\n\r\n" + options.settings_json("WAV").dump() + "\r\n";
|
|
body += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"audio\"; filename=\"audio.wav\"\r\n";
|
|
body += "Content-Type: audio/wav\r\n\r\n" + wav + "\r\n--" + boundary + "--\r\n";
|
|
std::string path = path_for(Call::kAsrTranscribe);
|
|
if (!options.session_id.empty()) path += "?sessionId=" + options.session_id;
|
|
return client_->request_raw(clients::Method::kPost, path, &body, "multipart/form-data; boundary=" + boundary,
|
|
{{"Accept", accept}});
|
|
}
|
|
|
|
TranscribeResult TranscribeService::transcribe(const SpeechOptions& options,
|
|
const std::string& wav) {
|
|
llm::RawResponse raw = post(options, wav, "application/json");
|
|
try {
|
|
return TranscribeResult::from_json(Json::parse(raw.body));
|
|
} catch (const std::exception& e) {
|
|
throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what());
|
|
}
|
|
}
|
|
|
|
std::vector<SpeechEvent> TranscribeService::transcribe_events(const SpeechOptions& options,
|
|
const std::string& wav) {
|
|
llm::RawResponse raw = post(options, wav, "text/event-stream");
|
|
std::vector<SpeechEvent> out;
|
|
llm::SseDecoder decoder("meta", [&](const llm::SseEvent& e) {
|
|
if (out.size() >= MetaCaps::kMaxRealtimeEvents) return false;
|
|
SpeechEvent ev = SpeechEvent::parse(e.data);
|
|
if (ev.type.empty()) ev.type = e.event;
|
|
out.push_back(std::move(ev));
|
|
return true;
|
|
});
|
|
decoder.feed(raw.body);
|
|
decoder.finish();
|
|
return out;
|
|
}
|
|
|
|
std::string TranscribeService::transcribe_text(const SpeechOptions& options,
|
|
const std::string& wav) {
|
|
return post(options, wav, "text/plain").body;
|
|
}
|
|
|
|
void RealtimeConfig::validate() const {
|
|
options.validate();
|
|
Check::one_of("audioEncoding", audio_encoding, {"PCM_16KHZ", "PCM_24KHZ"});
|
|
}
|
|
|
|
RealtimeSession::RealtimeSession(RealtimeConfig config) : config_(std::move(config)) {
|
|
if (config_.url.empty()) config_.url = MetaEndpoints::kRealtimeUrl;
|
|
if (config_.key.empty()) config_.key = MetaClient::key_from_env();
|
|
}
|
|
|
|
RealtimeSession::~RealtimeSession() {
|
|
try {
|
|
close();
|
|
} catch (...) {
|
|
}
|
|
}
|
|
|
|
Json RealtimeSession::handshake_json(bool reveal) const {
|
|
Json j = config_.options.settings_json(config_.audio_encoding);
|
|
Json auth = Json::object();
|
|
auth.set("accessToken", Json("Bearer " + (reveal ? config_.key : std::string("***"))));
|
|
j.set("authorization", auth);
|
|
return j;
|
|
}
|
|
|
|
void RealtimeSession::connect() {
|
|
config_.validate();
|
|
if (open_) Check::fail("session already connected");
|
|
if (config_.key.empty()) throw llm::NotConfiguredError("meta", MetaEndpoints::kKeyEnv);
|
|
using namespace google::live;
|
|
try {
|
|
WsUrl url = WsUrl::parse(config_.url);
|
|
if (!config_.options.session_id.empty())
|
|
url.target += (url.target.find('?') == std::string::npos ? "?" : "&") +
|
|
std::string("sessionId=") + config_.options.session_id;
|
|
ws_ = std::make_unique<WsClient>(
|
|
open_stream(url, config_.connect_timeout_ms, MetaCaps::kRealtimeIdleMs, config_.tls));
|
|
ws_->handshake(url, {}, config_.handshake_timeout_ms); // no Authorization header
|
|
ws_->send_text(handshake_json(true).dump());
|
|
ws_->set_read_timeout_ms(250);
|
|
long deadline = now_ms() + config_.ack_timeout_ms;
|
|
while (session_id_.empty()) {
|
|
if (now_ms() > deadline) throw llm::ApiError("meta", 504, "TIMEOUT", "handshake acknowledgement not received");
|
|
auto m = ws_->receive();
|
|
if (!m) continue;
|
|
if (m->kind == WsMessage::Kind::Close) {
|
|
closed_ = true;
|
|
close_code_ = m->code;
|
|
throw llm::ApiError("meta", m->code == 1013 ? 429 : 400, "REALTIME_CLOSED",
|
|
"closed before the acknowledgement (code " + std::to_string(m->code) +
|
|
"): " + m->data.substr(0, 300));
|
|
}
|
|
Json j = Json::parse(m->data);
|
|
if (JsonX::str(j, "type") == "error")
|
|
throw llm::ApiError("meta", 400, JsonX::str(j, "errorType", "error"), JsonX::str(j, "message"));
|
|
session_id_ = JsonX::str(j, "sessionId");
|
|
if (session_id_.empty()) throw llm::ApiError("meta", 502, "BAD_RESPONSE", "acknowledgement has no sessionId");
|
|
}
|
|
open_ = true;
|
|
} catch (const WsError& e) {
|
|
if (ws_) ws_->shutdown();
|
|
ws_.reset();
|
|
throw llm::ApiError("meta", e.http_status() ? e.http_status() : 502, "UNAVAILABLE", e.what());
|
|
}
|
|
}
|
|
|
|
void RealtimeSession::send_audio(const std::string& pcm) {
|
|
if (!open_) Check::fail("session is not open");
|
|
if (pcm.empty() || pcm.size() > MetaCaps::kMaxRealtimeAudioFrame)
|
|
Check::fail("audio frame must be 1.." + std::to_string(MetaCaps::kMaxRealtimeAudioFrame) + " bytes");
|
|
if (pcm.size() % 2 != 0) Check::fail("PCM frames are 16-bit samples");
|
|
try {
|
|
ws_->send_binary(pcm);
|
|
} catch (const google::live::WsError& e) {
|
|
throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what());
|
|
}
|
|
audio_bytes_ += pcm.size();
|
|
}
|
|
|
|
void RealtimeSession::end_stream() {
|
|
if (!open_) Check::fail("session is not open");
|
|
Json j = Json::object();
|
|
j.set("type", Json("endStream"));
|
|
try {
|
|
ws_->send_text(j.dump());
|
|
} catch (const google::live::WsError& e) {
|
|
throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what());
|
|
}
|
|
}
|
|
|
|
std::optional<SpeechEvent> RealtimeSession::next(int timeout_ms) {
|
|
if (!ws_ || closed_) return std::nullopt;
|
|
ws_->set_read_timeout_ms(timeout_ms < 1 ? 1 : timeout_ms);
|
|
try {
|
|
auto m = ws_->receive();
|
|
if (!m) return std::nullopt;
|
|
if (m->kind == google::live::WsMessage::Kind::Close) {
|
|
closed_ = true;
|
|
close_code_ = m->code;
|
|
open_ = false;
|
|
SpeechEvent e;
|
|
e.type = "close";
|
|
e.message = m->data;
|
|
e.audio_processed_ms = m->code;
|
|
return e;
|
|
}
|
|
if (++events_ > MetaCaps::kMaxRealtimeEvents) {
|
|
close();
|
|
throw llm::ApiError("meta", 502, "TOO_MANY_EVENTS", "event cap reached");
|
|
}
|
|
return SpeechEvent::parse(Json::parse(m->data));
|
|
} catch (const google::live::WsError& e) {
|
|
throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what());
|
|
} catch (const clients::body::JsonError& e) {
|
|
throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what());
|
|
}
|
|
}
|
|
|
|
void RealtimeSession::close() {
|
|
if (ws_) {
|
|
try {
|
|
ws_->close(1000, "");
|
|
} catch (...) {
|
|
}
|
|
ws_->shutdown();
|
|
ws_.reset();
|
|
}
|
|
open_ = false;
|
|
closed_ = true;
|
|
}
|
|
|
|
} // namespace molo::meta
|