feat(clients): llm vendor foundation: api client, SSE, errors, neutral chat types, ai.mk, example stubs

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 06:06:06 +02:00
co-authored by Claude Sonnet 5.5
parent 6f73535b82
commit 4c78017742
26 changed files with 1269 additions and 5 deletions
+176
View File
@@ -0,0 +1,176 @@
#include "llm/api_client.h"
#include "clients/transports/http_transport.h"
#include "llm/caps.h"
#include "llm/config.h"
namespace molo::llm {
using clients::body::Json;
namespace {
clients::ClientConfig make_config() {
clients::ClientConfig c;
c.persist_cookies = false;
return c;
}
} // namespace
ApiClient::ApiClient(std::unique_ptr<clients::Transport> transport,
ApiClientConfig config, std::string key)
: config_(std::move(config)),
key_(std::move(key)),
client_(std::move(transport), make_config()) {}
std::shared_ptr<ApiClient> ApiClient::from_env(ApiClientConfig config) {
std::string key = Config::get(config.key_env.c_str());
return std::make_shared<ApiClient>(
std::make_unique<clients::transports::HttpTransport>(), std::move(config),
std::move(key));
}
void ApiClient::guard() const {
if (!configured()) throw NotConfiguredError(config_.vendor, config_.key_env);
}
clients::ClientRequest ApiClient::build(clients::Method method, const std::string& path,
const Headers& extra) const {
if (path.empty() || path.front() != '/' || path.find("..") != std::string::npos)
throw ApiError::invalid(config_.vendor, "bad request path");
clients::ClientRequest req(method, clients::Url::parse(config_.base_url + path));
if (config_.auth == AuthStyle::kBearer) req.bearer(key_);
else req.header(config_.auth_header, key_);
for (const auto& [k, v] : config_.default_headers) req.header(k, v);
for (const auto& [k, v] : extra) req.header(k, v);
return req;
}
RawResponse ApiClient::finish(const clients::ClientResponse& response) const {
if (!response.is_success()) throw error_from_response(config_.vendor, response);
if (response.body().size() > Caps::kMaxResponseBytes)
throw ApiError(config_.vendor, 502, "BAD_RESPONSE", "response over the byte cap");
RawResponse out;
out.status = response.status();
out.body = response.body();
out.content_type = response.content_type();
out.headers = response.headers();
return out;
}
RawResponse ApiClient::request_raw(clients::Method method, const std::string& path,
const std::string* body,
const std::string& content_type,
const Headers& extra) {
guard();
try {
clients::ClientRequest req = build(method, path, extra);
if (body) {
if (body->size() > Caps::kMaxRequestBytes)
throw ApiError::invalid(config_.vendor, "request body over the byte cap");
req.bytes(*body, content_type);
}
return finish(client_.send_blocking(std::move(req)));
} catch (const ApiError&) {
throw;
} catch (const std::exception& e) {
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
}
}
Json ApiClient::request(clients::Method method, const std::string& path,
const Json* body, const Headers& extra) {
guard();
try {
clients::ClientRequest req = build(method, path, extra);
if (body) {
req.json(*body);
if (std::holds_alternative<std::string>(req.body()) &&
req.body_bytes().size() > Caps::kMaxRequestBytes)
throw ApiError::invalid(config_.vendor, "request body over the byte cap");
}
RawResponse raw = finish(client_.send_blocking(std::move(req)));
if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object();
try {
return Json::parse(raw.body);
} catch (const std::exception& e) {
throw ApiError(config_.vendor, 502, "BAD_RESPONSE", e.what());
}
} catch (const ApiError&) {
throw;
} catch (const std::exception& e) {
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
}
}
Json ApiClient::get(const std::string& path, const Headers& extra) {
return request(clients::Method::kGet, path, nullptr, extra);
}
Json ApiClient::post(const std::string& path, const Json& body, const Headers& extra) {
return request(clients::Method::kPost, path, &body, extra);
}
Json ApiClient::patch(const std::string& path, const Json& body, const Headers& extra) {
return request(clients::Method::kPatch, path, &body, extra);
}
Json ApiClient::del(const std::string& path, const Headers& extra) {
return request(clients::Method::kDelete, path, nullptr, extra);
}
Json ApiClient::post_multipart(const std::string& path,
const clients::body::MultipartBody& form,
const Headers& extra) {
guard();
try {
if (form.encoded_size() > Caps::kMaxUploadBytes)
throw ApiError::invalid(config_.vendor, "upload over the byte cap");
clients::ClientRequest req = build(clients::Method::kPost, path, extra);
req.multipart(form);
RawResponse raw = finish(client_.send_blocking(std::move(req)));
if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object();
return Json::parse(raw.body);
} catch (const ApiError&) {
throw;
} catch (const std::exception& e) {
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
}
}
void ApiClient::stream(const std::string& path, const Json& body,
const SseDecoder::Sink& sink, const Headers& extra) {
guard();
SseDecoder decoder(config_.vendor, sink);
try {
clients::ClientRequest req = build(clients::Method::kPost, path, extra);
req.json(body);
req.header("Accept", "text/event-stream");
client_.send_stream_blocking(std::move(req), [&](const std::string& chunk) {
return decoder.feed(chunk);
});
decoder.finish();
} catch (const ApiError&) {
throw;
} catch (const std::exception& e) {
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
}
}
bool Service::valid_id(const std::string& id) {
if (id.empty() || id.size() > 256 || id == "." || id == "..") return false;
for (char c : id)
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
c == '-' || c == '_' || c == '.' || c == ':'))
return false;
return true;
}
void Service::check_id(const std::string& what, const std::string& id) const {
if (!valid_id(id))
throw ApiError::invalid(client_->config().vendor, "bad " + what + " id");
}
std::string Service::string_at(const Json& doc, const std::string& key,
const std::string& fallback) {
const Json* v = doc.is_object() ? doc.find(key) : nullptr;
return v && v->is_string() ? v->as_string() : fallback;
}
} // namespace molo::llm
+111
View File
@@ -0,0 +1,111 @@
#pragma once
// ApiClient: the one place a vendor module talks HTTP. Applies the vendor's
// auth style, sends JSON / multipart / raw bodies, streams SSE, maps every
// failure to ApiError. Runs over any clients::Transport (HttpTransport in
// production).
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <string>
#include "clients/body/form.h"
#include "clients/body/json.h"
#include "clients/client.h"
#include "llm/error.h"
#include "llm/sse.h"
namespace molo::llm {
enum class AuthStyle { kBearer, kHeader };
struct ApiClientConfig {
std::string vendor; // "anthropic"
std::string base_url; // "https://api.anthropic.com", no trailing slash
std::string key_env; // "ANTHROPIC_API_KEY" (looked up via llm::Config)
AuthStyle auth = AuthStyle::kBearer;
std::string auth_header = "x-api-key"; // used for kHeader
std::map<std::string, std::string> default_headers; // e.g. anthropic-version
};
using Headers = std::map<std::string, std::string>;
struct RawResponse {
int status = 0;
std::string body;
std::string content_type;
clients::HeaderMap headers;
};
class ApiClient {
public:
// Empty key: not configured, every call throws NotConfiguredError before
// any network traffic.
ApiClient(std::unique_ptr<clients::Transport> transport, ApiClientConfig config,
std::string key);
// Real HTTPS transport, key from Config::get(config.key_env).
static std::shared_ptr<ApiClient> from_env(ApiClientConfig config);
bool configured() const { return !key_.empty(); }
const ApiClientConfig& config() const { return config_; }
clients::ClientStatsSnapshot stats() const { return client_.stats(); }
// JSON in, JSON out. body may be null (GET, DELETE). Empty 2xx body -> {}.
clients::body::Json request(clients::Method method, const std::string& path,
const clients::body::Json* body = nullptr,
const Headers& extra = {});
clients::body::Json get(const std::string& path, const Headers& extra = {});
clients::body::Json post(const std::string& path, const clients::body::Json& body,
const Headers& extra = {});
clients::body::Json patch(const std::string& path, const clients::body::Json& body,
const Headers& extra = {});
clients::body::Json del(const std::string& path, const Headers& extra = {});
// Bytes in/out for non-JSON payloads (file content, images, audio).
RawResponse request_raw(clients::Method method, const std::string& path,
const std::string* body = nullptr,
const std::string& content_type = "",
const Headers& extra = {});
clients::body::Json post_multipart(const std::string& path,
const clients::body::MultipartBody& form,
const Headers& extra = {});
// Streaming POST decoded as SSE; sink returns false to stop.
void stream(const std::string& path, const clients::body::Json& body,
const SseDecoder::Sink& sink, const Headers& extra = {});
private:
clients::ClientRequest build(clients::Method method, const std::string& path,
const Headers& extra) const;
void guard() const;
RawResponse finish(const clients::ClientResponse& response) const;
ApiClientConfig config_;
std::string key_;
clients::Client client_;
};
// Base for vendor service classes.
class Service {
public:
explicit Service(std::shared_ptr<ApiClient> client) : client_(std::move(client)) {}
virtual ~Service() = default;
bool configured() const { return client_->configured(); }
ApiClient& client() const { return *client_; }
// True for a single URL path segment: 1..256 of [A-Za-z0-9._:-] and not
// "." or "..". Use before putting any id in a path.
static bool valid_id(const std::string& id);
// Throws ApiError::invalid when the id is not a valid segment.
void check_id(const std::string& what, const std::string& id) const;
static std::string string_at(const clients::body::Json& doc, const std::string& key,
const std::string& fallback = "");
protected:
std::shared_ptr<ApiClient> client_;
};
} // namespace molo::llm
+21
View File
@@ -0,0 +1,21 @@
#pragma once
// Static ceilings shared by every LLM vendor module. Never autotuned.
#include <cstddef>
namespace molo::llm {
struct Caps {
static constexpr std::size_t kMaxRequestBytes = 32u * 1024u * 1024u;
static constexpr std::size_t kMaxResponseBytes = 64u * 1024u * 1024u;
static constexpr std::size_t kMaxStreamBytes = 64u * 1024u * 1024u;
static constexpr std::size_t kMaxSseEventBytes = 4u * 1024u * 1024u;
static constexpr std::size_t kMaxUploadBytes = 512u * 1024u * 1024u;
static constexpr std::size_t kMaxMessages = 4096;
static constexpr std::size_t kMaxTools = 256;
static constexpr std::size_t kMaxPages = 100;
static constexpr std::size_t kMaxConfigBytes = 64u * 1024u;
};
} // namespace molo::llm
+62
View File
@@ -0,0 +1,62 @@
#include "llm/chat_types.h"
namespace molo::llm {
const char* role_name(Role role) {
switch (role) {
case Role::kSystem: return "system";
case Role::kUser: return "user";
case Role::kAssistant: return "assistant";
case Role::kTool: return "tool";
}
return "user";
}
ContentPart ContentPart::of_text(std::string text) {
ContentPart p;
p.text = std::move(text);
return p;
}
ContentPart ContentPart::of_image(ImageData image) {
ContentPart p;
p.image = std::move(image);
return p;
}
ContentPart ContentPart::of_call(ToolCall call) {
ContentPart p;
p.tool_call = std::move(call);
return p;
}
ContentPart ContentPart::of_result(ToolResult result) {
ContentPart p;
p.tool_result = std::move(result);
return p;
}
namespace {
Message make(Role role, std::string text) {
Message m;
m.role = role;
m.parts.push_back(ContentPart::of_text(std::move(text)));
return m;
}
} // namespace
Message Message::system(std::string text) { return make(Role::kSystem, std::move(text)); }
Message Message::user(std::string text) { return make(Role::kUser, std::move(text)); }
Message Message::assistant(std::string text) { return make(Role::kAssistant, std::move(text)); }
std::string Message::text() const {
std::string out;
for (const ContentPart& p : parts) out += p.text;
return out;
}
std::vector<ToolCall> ChatResponse::tool_calls() const {
std::vector<ToolCall> out;
for (const ContentPart& p : message.parts)
if (p.tool_call) out.push_back(*p.tool_call);
return out;
}
} // namespace molo::llm
+127
View File
@@ -0,0 +1,127 @@
#pragma once
// Vendor-neutral chat model: messages, tools, requests, responses, stream
// chunks, and the ChatModel seam. Each vendor module implements ChatModel
// over its native wire format (and keeps its native types for everything
// beyond chat). Agents, evaluation and apps code against ChatModel.
#include <functional>
#include <optional>
#include <string>
#include <vector>
#include "clients/body/json.h"
namespace molo::llm {
enum class Role { kSystem, kUser, kAssistant, kTool };
const char* role_name(Role role);
struct ImageData {
std::string mime;
std::string bytes; // Raw bytes; empty when url is set.
std::string url;
};
struct ToolCall {
std::string id;
std::string name;
clients::body::Json arguments = clients::body::Json::object();
};
struct ToolResult {
std::string call_id;
std::string content;
bool is_error = false;
};
struct ContentPart {
std::string text;
std::string reasoning; // Model thinking text (never sent back unless the vendor requires it)
std::string reasoning_signature; // Opaque; replay it exactly (Anthropic thinking)
std::optional<ImageData> image;
std::optional<ToolCall> tool_call;
std::optional<ToolResult> tool_result;
static ContentPart of_text(std::string text);
static ContentPart of_image(ImageData image);
static ContentPart of_call(ToolCall call);
static ContentPart of_result(ToolResult result);
};
struct Message {
Role role = Role::kUser;
std::vector<ContentPart> parts;
std::string name; // Optional participant name.
static Message system(std::string text);
static Message user(std::string text);
static Message assistant(std::string text);
std::string text() const; // Concatenated text parts.
};
struct ToolSpec {
std::string name;
std::string description;
clients::body::Json parameters = clients::body::Json::object(); // JSON Schema
};
struct ChatRequest {
std::string model;
std::string system; // Convenience: prepended as a system message / field.
std::vector<Message> messages;
std::vector<ToolSpec> tools;
std::string tool_choice; // "", "auto", "none", "required", or a tool name
std::optional<int> max_tokens;
std::optional<double> temperature;
std::optional<double> top_p;
std::vector<std::string> stop;
bool json_mode = false; // Ask for a JSON object
clients::body::Json response_schema; // null: none; else strict JSON Schema
std::string reasoning_effort; // "", "low", "medium", "high", vendor values
clients::body::Json extra; // Vendor passthrough, merged into the body
};
struct Usage {
int input_tokens = 0;
int output_tokens = 0;
int total_tokens = 0;
int cached_tokens = 0;
int reasoning_tokens = 0;
};
struct ChatResponse {
std::string id;
std::string model;
Message message; // role = kAssistant
std::string finish_reason; // vendor-neutral where possible: stop, length, tool_calls, content_filter
Usage usage;
clients::body::Json raw; // The vendor document, for fields the neutral type omits
std::string text() const { return message.text(); }
std::vector<ToolCall> tool_calls() const;
};
struct ChatChunk {
std::string text_delta;
std::string reasoning_delta;
std::optional<ToolCall> tool_call; // Complete call when the vendor streams it whole
std::string tool_call_id; // Partial call fragments: id + name + args_delta
std::string tool_call_name;
std::string tool_args_delta;
std::string finish_reason;
std::optional<Usage> usage;
};
class ChatModel {
public:
virtual ~ChatModel() = default;
virtual std::string vendor() const = 0;
virtual ChatResponse generate(const ChatRequest& request) = 0;
// on_chunk returns false to stop early.
virtual void stream(const ChatRequest& request,
const std::function<bool(const ChatChunk&)>& on_chunk) = 0;
};
} // namespace molo::llm
+38
View File
@@ -0,0 +1,38 @@
#include "llm/config.h"
#include <cstdlib>
#include <fstream>
#include <sstream>
#include "clients/body/json.h"
#include "llm/caps.h"
namespace molo::llm {
namespace {
std::string from_json_file(const char* path, const char* name) {
std::ifstream in(path, std::ios::binary);
if (!in) return "";
std::ostringstream text;
text << in.rdbuf();
if (text.str().size() > Caps::kMaxConfigBytes) return "";
try {
clients::body::Json doc = clients::body::Json::parse(text.str());
const clients::body::Json* v = doc.is_object() ? doc.find(name) : nullptr;
return v && v->is_string() ? v->as_string() : "";
} catch (const std::exception&) {
return "";
}
}
} // namespace
std::string Config::get(const char* name) {
if (const char* v = std::getenv(name); v && *v) return v;
const char* files = std::getenv("MOLO_ENV_FILES");
if (files && std::string(files) == "off" && !std::getenv("MOLO_LIVE_NET")) return "";
std::string local = from_json_file(".env.json.local", name);
if (!local.empty()) return local;
return from_json_file(".env.json", name);
}
} // namespace molo::llm
+18
View File
@@ -0,0 +1,18 @@
#pragma once
// Configuration lookup shared by all vendors: process environment, then
// .env.json.local (git-ignored, for secrets), then .env.json, both in the
// working directory. Test runs set MOLO_ENV_FILES=off so a developer's local
// file never changes test results; MOLO_LIVE_NET=1 re-enables it.
#include <string>
namespace molo::llm {
class Config {
public:
// Value of `name`, or empty when unset everywhere.
static std::string get(const char* name);
};
} // namespace molo::llm
+70
View File
@@ -0,0 +1,70 @@
#include "llm/error.h"
#include "clients/body/json.h"
namespace molo::llm {
using clients::body::Json;
ApiError::ApiError(std::string vendor, int http_status, std::string type,
const std::string& message, std::string request_id)
: std::runtime_error("molo::llm::" + vendor + ": " + type + ": " + message),
vendor_(std::move(vendor)),
http_status_(http_status),
type_(std::move(type)),
message_(message),
request_id_(std::move(request_id)) {}
ApiError ApiError::invalid(const std::string& vendor, const std::string& message) {
return ApiError(vendor, 400, "INVALID_ARGUMENT", message);
}
NotConfiguredError::NotConfiguredError(const std::string& vendor,
const std::string& env_var)
: ApiError(vendor, 503, "NOT_CONFIGURED",
"no credential configured (set " + env_var + ")") {}
namespace {
std::string text_of(const Json* v) {
if (!v) return "";
if (v->is_string()) return v->as_string();
if (v->is_number()) return std::to_string(v->as_int());
return "";
}
} // namespace
ApiError error_from_response(const std::string& vendor,
const clients::ClientResponse& response) {
std::string type = "UNKNOWN";
std::string message = "HTTP " + std::to_string(response.status());
std::string request_id;
for (const char* h : {"request-id", "x-request-id", "x-should-retry-request-id"})
if (auto v = response.headers().get(h); v && !v->empty()) {
request_id = *v;
break;
}
try {
Json doc = response.json();
const Json* err = doc.is_object() ? doc.find("error") : nullptr;
if (err && err->is_object()) {
std::string t = text_of(err->find("type"));
if (t.empty()) t = text_of(err->find("code"));
if (!t.empty()) type = t;
std::string m = text_of(err->find("message"));
if (!m.empty()) message = m;
} else if (err && err->is_string()) {
message = err->as_string();
} else if (doc.is_object()) {
std::string m = text_of(doc.find("message"));
if (!m.empty()) message = m;
std::string t = text_of(doc.find("type"));
if (t.empty()) t = text_of(doc.find("code"));
if (!t.empty()) type = t;
}
} catch (const std::exception&) {
// Non-JSON body: keep the generic type and message.
}
return ApiError(vendor, response.status(), type, message, request_id);
}
} // namespace molo::llm
+47
View File
@@ -0,0 +1,47 @@
#pragma once
// One error type for every vendor call. Carries the HTTP status, the
// vendor's own error type/code and message, and the request id when the
// server sent one. Never the credential.
#include <stdexcept>
#include <string>
#include "clients/response.h"
namespace molo::llm {
class ApiError : public std::runtime_error {
public:
ApiError(std::string vendor, int http_status, std::string type,
const std::string& message, std::string request_id = "");
const std::string& vendor() const noexcept { return vendor_; }
int http_status() const noexcept { return http_status_; }
// Vendor error type or code ("authentication_error", "invalid_request_error",
// "rate_limit_exceeded", ...), or a framework code (NOT_CONFIGURED, ...).
const std::string& type() const noexcept { return type_; }
const std::string& message() const noexcept { return message_; }
const std::string& request_id() const noexcept { return request_id_; }
static ApiError invalid(const std::string& vendor, const std::string& message);
private:
std::string vendor_;
int http_status_;
std::string type_;
std::string message_;
std::string request_id_;
};
class NotConfiguredError : public ApiError {
public:
NotConfiguredError(const std::string& vendor, const std::string& env_var);
};
// Parses the common error envelopes: {"error":{"type"|"code","message"}},
// {"type":"error","error":{...}}, {"error":"text"}, {"message":"text"}.
ApiError error_from_response(const std::string& vendor,
const clients::ClientResponse& response);
} // namespace molo::llm
+108
View File
@@ -0,0 +1,108 @@
#include "llm/sse.h"
#include "llm/caps.h"
#include "llm/error.h"
namespace molo::llm {
using clients::body::Json;
SseDecoder::SseDecoder(std::string vendor, Sink sink)
: vendor_(std::move(vendor)), sink_(std::move(sink)) {}
bool SseDecoder::dispatch(const std::string& block) {
std::string event;
std::string data;
std::size_t pos = 0;
while (pos <= block.size()) {
std::size_t eol = block.find('\n', pos);
std::string line = block.substr(pos, eol == std::string::npos ? eol : eol - pos);
if (!line.empty() && line.back() == '\r') line.pop_back();
if (line.rfind("event:", 0) == 0) {
event = line.substr(6);
if (!event.empty() && event.front() == ' ') event.erase(0, 1);
} else if (line.rfind("data:", 0) == 0) {
std::string v = line.substr(5);
if (!v.empty() && v.front() == ' ') v.erase(0, 1);
if (!data.empty()) data += "\n";
data += v;
}
if (eol == std::string::npos) break;
pos = eol + 1;
}
if (data.empty()) return true;
if (data == "[DONE]") {
done_ = true;
return false;
}
if (data.size() > Caps::kMaxSseEventBytes)
throw ApiError(vendor_, 502, "BAD_RESPONSE", "SSE event over the byte cap");
SseEvent ev;
ev.event = event;
try {
ev.data = Json::parse(data);
} catch (const std::exception& e) {
throw ApiError(vendor_, 502, "BAD_RESPONSE", std::string("bad SSE data: ") + e.what());
}
if (!sink_(ev)) {
done_ = true;
return false;
}
return true;
}
bool SseDecoder::feed(const std::string& chunk) {
if (done_) return false;
total_ += chunk.size();
if (total_ > Caps::kMaxStreamBytes)
throw ApiError(vendor_, 502, "BAD_RESPONSE", "stream over the byte cap");
buffer_ += chunk;
if (!started_) {
std::size_t first = buffer_.find_first_not_of(" \t\r\n");
if (first == std::string::npos) return true;
started_ = true;
sse_ = buffer_[first] != '{' && buffer_[first] != '[';
}
if (!sse_) return true; // Plain JSON (an error body): handled in finish().
for (;;) {
std::size_t end = buffer_.find("\n\n");
std::size_t sep = 2;
std::size_t crlf = buffer_.find("\r\n\r\n");
if (crlf != std::string::npos && (end == std::string::npos || crlf < end)) {
end = crlf;
sep = 4;
}
if (end == std::string::npos) break;
std::string block = buffer_.substr(0, end);
buffer_.erase(0, end + sep);
if (!dispatch(block)) return false;
}
return true;
}
void SseDecoder::finish() {
if (done_ || !started_) return;
if (sse_) {
if (buffer_.find_first_not_of(" \t\r\n") != std::string::npos) dispatch(buffer_);
return;
}
// A JSON body where a stream was expected is the vendor's error document.
Json doc;
try {
doc = Json::parse(buffer_);
} catch (const std::exception& e) {
throw ApiError(vendor_, 502, "BAD_RESPONSE", e.what());
}
const Json* err = doc.is_object() ? doc.find("error") : nullptr;
std::string message = err && err->is_object() && err->find("message") &&
err->find("message")->is_string()
? err->find("message")->as_string()
: buffer_.substr(0, 300);
std::string type = err && err->is_object() && err->find("type") &&
err->find("type")->is_string()
? err->find("type")->as_string()
: "STREAM_ERROR";
throw ApiError(vendor_, 502, type, message);
}
} // namespace molo::llm
+41
View File
@@ -0,0 +1,41 @@
#pragma once
// Server-sent events decoder keeping the event name: `event: x` + `data: {...}`
// blocks (Anthropic), bare `data:` blocks (OpenAI-compatible vendors), the
// `data: [DONE]` sentinel (ends the stream, not delivered). Comment lines
// (`: ping`) are ignored. A plain JSON error body (no SSE framing) throws.
#include <functional>
#include <string>
#include "clients/body/json.h"
namespace molo::llm {
struct SseEvent {
std::string event; // Empty when the block had no `event:` line.
clients::body::Json data;
};
class SseDecoder {
public:
using Sink = std::function<bool(const SseEvent&)>; // false: stop
SseDecoder(std::string vendor, Sink sink);
// Returns false when the sink stopped the stream or [DONE] was seen.
bool feed(const std::string& chunk);
void finish();
private:
bool dispatch(const std::string& block);
std::string vendor_;
Sink sink_;
std::string buffer_;
std::size_t total_ = 0;
bool done_ = false;
bool started_ = false;
bool sse_ = true;
};
} // namespace molo::llm