154 lines
5.5 KiB
C++
154 lines
5.5 KiB
C++
#include "vendors/audio/hosts.h"
|
|||
|
|
|
||
|
|
#include "clients/transports/http_transport.h"
|
||
|
|
#include "llm/config.h"
|
||
|
|
#include "vendors/audio/caps.h"
|
||
|
|
|
||
|
|
namespace molo::audio {
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
clients::ClientConfig stream_client_config() {
|
||
|
|
clients::ClientConfig c;
|
||
|
|
c.persist_cookies = false;
|
||
|
|
return c;
|
||
|
|
}
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
ByteStreamer::ByteStreamer(std::unique_ptr<clients::Transport> transport,
|
||
|
|
llm::ApiClientConfig config, std::string key)
|
||
|
|
: config_(std::move(config)),
|
||
|
|
key_(std::move(key)),
|
||
|
|
client_(std::move(transport), stream_client_config()) {}
|
||
|
|
|
||
|
|
ByteStreamer::Result ByteStreamer::stream(clients::Method method, const std::string& path,
|
||
|
|
const Json* body, const ChunkSink& sink,
|
||
|
|
const llm::Headers& extra) {
|
||
|
|
if (!configured()) throw llm::NotConfiguredError(config_.vendor, config_.key_env);
|
||
|
|
if (path.empty() || path.front() != '/' || path.find("..") != std::string::npos)
|
||
|
|
throw llm::ApiError::invalid(config_.vendor, "bad request path");
|
||
|
|
Result result;
|
||
|
|
std::string probe; // leading bytes held back while they look like a JSON error
|
||
|
|
bool probing = true;
|
||
|
|
bool json_like = false;
|
||
|
|
bool stopped = false;
|
||
|
|
auto forward = [&](const std::string& chunk) -> bool {
|
||
|
|
if (chunk.empty()) return true;
|
||
|
|
if (result.bytes + chunk.size() > AudioCaps::kMaxStreamAudioBytes) {
|
||
|
|
result.truncated = true;
|
||
|
|
stopped = true;
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
result.bytes += chunk.size();
|
||
|
|
if (!sink(chunk)) {
|
||
|
|
stopped = true;
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
};
|
||
|
|
try {
|
||
|
|
clients::ClientRequest req(method, clients::Url::parse(config_.base_url + path));
|
||
|
|
if (config_.auth == llm::AuthStyle::kBearer) req.bearer(key_);
|
||
|
|
else req.header(config_.auth_header, key_);
|
||
|
|
for (const auto& [k, v] : config_.default_headers) req.header(k, v);
|
||
|
|
for (const auto& [k, v] : extra) req.header(k, v);
|
||
|
|
if (body) req.json(*body);
|
||
|
|
client_.send_stream_blocking(std::move(req), [&](const std::string& chunk) -> bool {
|
||
|
|
if (!probing) return forward(chunk);
|
||
|
|
probe += chunk;
|
||
|
|
if (!json_like) {
|
||
|
|
std::size_t at = probe.find_first_not_of(" \t\r\n");
|
||
|
|
if (at == std::string::npos) return true;
|
||
|
|
json_like = probe[at] == '{';
|
||
|
|
if (!json_like) {
|
||
|
|
probing = false;
|
||
|
|
std::string held;
|
||
|
|
held.swap(probe);
|
||
|
|
return forward(held);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (probe.size() > AudioCaps::kErrorProbeBytes) { // too big to be an error document
|
||
|
|
probing = false;
|
||
|
|
std::string held;
|
||
|
|
held.swap(probe);
|
||
|
|
return forward(held);
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
});
|
||
|
|
if (probing && !probe.empty()) {
|
||
|
|
// The whole body fit the probe window. A JSON object with an error shape is the vendor's error.
|
||
|
|
try {
|
||
|
|
Json doc = Json::parse(probe);
|
||
|
|
if (doc.is_object() && (doc.find("detail") || doc.find("error") || doc.find("message") ||
|
||
|
|
doc.find("err_code") || doc.find("err_msg"))) {
|
||
|
|
std::string msg = probe.substr(0, 300);
|
||
|
|
throw llm::ApiError(config_.vendor, 400, "stream_error", msg);
|
||
|
|
}
|
||
|
|
} catch (const clients::body::JsonError&) {
|
||
|
|
}
|
||
|
|
forward(probe);
|
||
|
|
}
|
||
|
|
} catch (const llm::ApiError&) {
|
||
|
|
throw;
|
||
|
|
} catch (const std::exception& e) {
|
||
|
|
if (!stopped) throw llm::ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
Hosts& Hosts::add(const std::string& name, AudioHost host) {
|
||
|
|
if (hosts_.empty()) first_ = name;
|
||
|
|
hosts_[name] = std::move(host);
|
||
|
|
return *this;
|
||
|
|
}
|
||
|
|
|
||
|
|
const AudioHost* Hosts::find(const std::string& name) const {
|
||
|
|
auto it = hosts_.find(name);
|
||
|
|
return it == hosts_.end() ? nullptr : &it->second;
|
||
|
|
}
|
||
|
|
|
||
|
|
const AudioHost& Hosts::get(const std::string& name, const std::string& vendor) const {
|
||
|
|
const AudioHost* h = find(name);
|
||
|
|
if (!h) throw llm::ApiError::invalid(vendor, "no such host: " + name);
|
||
|
|
return *h;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Hosts::any_configured() const {
|
||
|
|
for (const auto& [name, h] : hosts_)
|
||
|
|
if (h.configured()) return true;
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string allowlisted_base(const char* env_name, const std::string& fallback,
|
||
|
|
std::initializer_list<const char*> allowed) {
|
||
|
|
std::string v = llm::Config::get(env_name);
|
||
|
|
while (!v.empty() && v.back() == '/') v.pop_back();
|
||
|
|
for (const char* a : allowed)
|
||
|
|
if (v == a) return v;
|
||
|
|
return fallback;
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string to_ws_url(const std::string& http_url) {
|
||
|
|
if (http_url.rfind("https://", 0) == 0) return "wss://" + http_url.substr(8);
|
||
|
|
if (http_url.rfind("http://", 0) == 0) return "ws://" + http_url.substr(7);
|
||
|
|
return http_url;
|
||
|
|
}
|
||
|
|
|
||
|
|
AudioHost HostFactory::from_env(llm::ApiClientConfig config, const std::string& key_prefix) {
|
||
|
|
std::string key = llm::Config::get(config.key_env.c_str());
|
||
|
|
if (!key.empty()) key = key_prefix + key;
|
||
|
|
return with_transports(std::make_unique<clients::transports::HttpTransport>(),
|
||
|
|
std::make_unique<clients::transports::HttpTransport>(), std::move(config),
|
||
|
|
key);
|
||
|
|
}
|
||
|
|
|
||
|
|
AudioHost HostFactory::with_transports(std::unique_ptr<clients::Transport> api_transport,
|
||
|
|
std::unique_ptr<clients::Transport> stream_transport,
|
||
|
|
llm::ApiClientConfig config, const std::string& key) {
|
||
|
|
AudioHost host;
|
||
|
|
host.api = std::make_shared<llm::ApiClient>(std::move(api_transport), config, key);
|
||
|
|
host.streamer = std::make_shared<ByteStreamer>(std::move(stream_transport), config, key);
|
||
|
|
return host;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace molo::audio
|