Files
moloapp/src/vendors/audio/assemblyai/stream.h
T
retoorandClaude Sonnet 5.5 6d80dd0223 feat(clients): audio: ElevenLabs, AssemblyAI, Deepgram, Cartesia, Speechmatics, Hume
Table driven REST (509 rows verified against vendor specs by tools/check_spec.py),
WebSockets over the src/google/live client, neutral SpeechSynthesizer /
SpeechRecognizer seam, /ai/audio demo with dry runs, 110 package tests and 5
route tests. PlayHT has no public API; LeMUR is gone from AssemblyAI's spec.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-25 15:36:57 +01:00

82 lines
2.8 KiB
C++

#pragma once
// AssemblyAI streaming speech to text (v3): wss://streaming.assemblyai.com/v3/ws.
// Messages follow https://www.assemblyai.com/docs/api-reference/specs/streaming.yaml:
// client sends binary audio plus UpdateConfiguration / ForceEndpoint / Terminate /
// KeepAlive, server sends Begin / SpeechStarted / Turn / SpeakerRevision /
// Termination / Heartbeat / LLMGatewayResponse.
#include <memory>
#include <optional>
#include <string>
#include <vector>
#include "vendors/audio/assemblyai/client.h"
#include "vendors/audio/audio_service.h"
#include "vendors/audio/socket.h"
namespace molo::audio::assemblyai {
struct StreamParams {
// name -> value; names are checked against the channel's query parameters.
std::vector<std::pair<std::string, std::string>> pairs;
std::string token; // temporary token instead of the Authorization header (browser style)
StreamParams& add(const std::string& name, const std::string& value) {
pairs.emplace_back(name, value);
return *this;
}
};
struct StreamTurn {
std::string transcript;
bool end_of_turn = false;
bool formatted = false;
long turn_order = -1;
double end_of_turn_confidence = 0;
std::string speaker_label;
Json raw;
};
struct StreamResult {
std::string session_id;
std::vector<std::string> final_turns; // end_of_turn transcripts (formatted ones when format_turns is on)
std::string transcript; // joined by a space
std::string partial;
double audio_duration_seconds = -1; // from Termination
std::size_t speech_started = 0;
std::string error;
bool terminated = false;
bool closed = false;
std::size_t events = 0;
};
class TranscriptStream {
public:
TranscriptStream(std::string api_key, StreamParams params, std::string url = "", SocketOptions socket = {});
// The URL without a temporary token (safe to log).
std::string url() const { return public_url_; }
void connect() { socket_->connect(); }
void close() { socket_->close(); }
bool is_open() const { return socket_->is_open(); }
void send_audio(const std::string& audio) { socket_->send_binary(audio); }
void update_configuration(const Json& fields); // {"type":"UpdateConfiguration",...}
void force_endpoint(); // {"type":"ForceEndpoint"}
void keep_alive(); // {"type":"KeepAlive"}
void terminate(); // {"type":"Terminate"}
std::optional<SocketEvent> next(int wait_ms) { return socket_->next(wait_ms); }
static std::optional<StreamTurn> as_turn(const SocketEvent& ev);
// Reads until Termination, error, close, or silence of `max_wait_ms`.
StreamResult collect(int max_wait_ms);
static const std::vector<std::string>& query_params();
static Vocabulary vocabulary();
private:
std::string public_url_;
std::unique_ptr<AudioSocket> socket_;
};
} // namespace molo::audio::assemblyai