Table driven REST (509 rows verified against vendor specs by tools/check_spec.py), WebSockets over the src/google/live client, neutral SpeechSynthesizer / SpeechRecognizer seam, /ai/audio demo with dry runs, 110 package tests and 5 route tests. PlayHT has no public API; LeMUR is gone from AssemblyAI's spec. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
82 lines
2.8 KiB
C++
82 lines
2.8 KiB
C++
#pragma once
|
|
|
|
// AssemblyAI streaming speech to text (v3): wss://streaming.assemblyai.com/v3/ws.
|
|
// Messages follow https://www.assemblyai.com/docs/api-reference/specs/streaming.yaml:
|
|
// client sends binary audio plus UpdateConfiguration / ForceEndpoint / Terminate /
|
|
// KeepAlive, server sends Begin / SpeechStarted / Turn / SpeakerRevision /
|
|
// Termination / Heartbeat / LLMGatewayResponse.
|
|
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "vendors/audio/assemblyai/client.h"
|
|
#include "vendors/audio/audio_service.h"
|
|
#include "vendors/audio/socket.h"
|
|
|
|
namespace molo::audio::assemblyai {
|
|
|
|
struct StreamParams {
|
|
// name -> value; names are checked against the channel's query parameters.
|
|
std::vector<std::pair<std::string, std::string>> pairs;
|
|
std::string token; // temporary token instead of the Authorization header (browser style)
|
|
StreamParams& add(const std::string& name, const std::string& value) {
|
|
pairs.emplace_back(name, value);
|
|
return *this;
|
|
}
|
|
};
|
|
|
|
struct StreamTurn {
|
|
std::string transcript;
|
|
bool end_of_turn = false;
|
|
bool formatted = false;
|
|
long turn_order = -1;
|
|
double end_of_turn_confidence = 0;
|
|
std::string speaker_label;
|
|
Json raw;
|
|
};
|
|
|
|
struct StreamResult {
|
|
std::string session_id;
|
|
std::vector<std::string> final_turns; // end_of_turn transcripts (formatted ones when format_turns is on)
|
|
std::string transcript; // joined by a space
|
|
std::string partial;
|
|
double audio_duration_seconds = -1; // from Termination
|
|
std::size_t speech_started = 0;
|
|
std::string error;
|
|
bool terminated = false;
|
|
bool closed = false;
|
|
std::size_t events = 0;
|
|
};
|
|
|
|
class TranscriptStream {
|
|
public:
|
|
TranscriptStream(std::string api_key, StreamParams params, std::string url = "", SocketOptions socket = {});
|
|
|
|
// The URL without a temporary token (safe to log).
|
|
std::string url() const { return public_url_; }
|
|
void connect() { socket_->connect(); }
|
|
void close() { socket_->close(); }
|
|
bool is_open() const { return socket_->is_open(); }
|
|
|
|
void send_audio(const std::string& audio) { socket_->send_binary(audio); }
|
|
void update_configuration(const Json& fields); // {"type":"UpdateConfiguration",...}
|
|
void force_endpoint(); // {"type":"ForceEndpoint"}
|
|
void keep_alive(); // {"type":"KeepAlive"}
|
|
void terminate(); // {"type":"Terminate"}
|
|
std::optional<SocketEvent> next(int wait_ms) { return socket_->next(wait_ms); }
|
|
static std::optional<StreamTurn> as_turn(const SocketEvent& ev);
|
|
// Reads until Termination, error, close, or silence of `max_wait_ms`.
|
|
StreamResult collect(int max_wait_ms);
|
|
|
|
static const std::vector<std::string>& query_params();
|
|
static Vocabulary vocabulary();
|
|
|
|
private:
|
|
std::string public_url_;
|
|
std::unique_ptr<AudioSocket> socket_;
|
|
};
|
|
|
|
} // namespace molo::audio::assemblyai
|