Files
moloapp/src/vendors/bedrock/event_stream.cpp
T

193 lines
7.6 KiB
C++
Raw Normal View History

#include "vendors/bedrock/event_stream.h"
#include "clients/body/json.h"
#include "llm/error.h"
#include "vendors/bedrock/caps.h"
#include "vendors/bedrock/crc32.h"
namespace molo::bedrock {
namespace {
std::uint32_t be32(const std::string& s, std::size_t at) {
return (static_cast<std::uint32_t>(static_cast<unsigned char>(s[at])) << 24) |
(static_cast<std::uint32_t>(static_cast<unsigned char>(s[at + 1])) << 16) |
(static_cast<std::uint32_t>(static_cast<unsigned char>(s[at + 2])) << 8) |
static_cast<std::uint32_t>(static_cast<unsigned char>(s[at + 3]));
}
std::uint64_t be_n(const std::string& s, std::size_t at, int n) {
std::uint64_t v = 0;
for (int i = 0; i < n; ++i) v = (v << 8) | static_cast<unsigned char>(s[at + static_cast<std::size_t>(i)]);
return v;
}
void put32(std::string& out, std::uint32_t v) {
for (int shift = 24; shift >= 0; shift -= 8) out.push_back(static_cast<char>((v >> shift) & 0xFF));
}
[[noreturn]] void fail(const std::string& vendor, const std::string& type, const std::string& why) {
throw llm::ApiError(vendor, 502, type, why);
}
void parse_headers(const std::string& vendor, const std::string& block, EventMessage& out) {
std::size_t at = 0;
auto need = [&](std::size_t n) {
if (at + n > block.size()) fail(vendor, "EVENTSTREAM_HEADERS", "header block is truncated");
};
while (at < block.size()) {
need(1);
std::size_t name_len = static_cast<unsigned char>(block[at++]);
need(name_len + 1);
std::string name = block.substr(at, name_len);
at += name_len;
int type = static_cast<unsigned char>(block[at++]);
EventHeader h;
h.type = type;
switch (type) {
case 0: h.number = 1; break;
case 1: h.number = 0; break;
case 2: need(1); h.number = static_cast<std::int8_t>(block[at]); at += 1; break;
case 3: need(2); h.number = static_cast<std::int16_t>(be_n(block, at, 2)); at += 2; break;
case 4: need(4); h.number = static_cast<std::int32_t>(be_n(block, at, 4)); at += 4; break;
case 5:
case 8: need(8); h.number = static_cast<std::int64_t>(be_n(block, at, 8)); at += 8; break;
case 6:
case 7: {
need(2);
std::size_t len = static_cast<std::size_t>(be_n(block, at, 2));
at += 2;
need(len);
h.text = block.substr(at, len);
at += len;
break;
}
case 9: need(16); h.text = block.substr(at, 16); at += 16; break;
default: fail(vendor, "EVENTSTREAM_HEADERS", "unknown header value type " + std::to_string(type));
}
out.headers[name] = std::move(h);
}
}
} // namespace
std::string EventMessage::header(const std::string& name) const {
auto it = headers.find(name);
return it == headers.end() ? std::string() : it->second.text;
}
EventStreamDecoder::EventStreamDecoder(std::string vendor, Sink sink)
: vendor_(std::move(vendor)), sink_(std::move(sink)) {}
bool EventStreamDecoder::feed(const std::string& chunk) {
if (stopped_) return false;
total_ += chunk.size();
if (total_ > BedrockCaps::kMaxStreamBytes) fail(vendor_, "STREAM_TOO_LARGE", "stream over the byte cap");
buffer_ += chunk;
if (!started_ && !buffer_.empty()) {
started_ = true;
// A real frame starts with a 4 byte length; a JSON error body starts with '{'.
// 0x7B in the first length byte would mean a message over 2 GiB, far past the cap.
if (buffer_[0] == '{') json_error_ = true;
}
if (json_error_) {
if (buffer_.size() > 1024u * 1024u) fail(vendor_, "BAD_RESPONSE", "error body over 1 MiB");
return true;
}
return drain();
}
bool EventStreamDecoder::drain() {
while (buffer_.size() >= 12) {
std::uint32_t total = be32(buffer_, 0);
std::uint32_t headers_len = be32(buffer_, 4);
std::uint32_t prelude_crc = be32(buffer_, 8);
if (Crc32::of(buffer_.substr(0, 8)) != prelude_crc)
fail(vendor_, "EVENTSTREAM_CRC", "prelude CRC mismatch");
if (total < 16 + headers_len) fail(vendor_, "EVENTSTREAM_LENGTH", "message shorter than its headers");
if (total > BedrockCaps::kMaxEventMessageBytes) fail(vendor_, "EVENTSTREAM_LENGTH", "message over the size cap");
if (headers_len > BedrockCaps::kMaxEventHeaderBytes) fail(vendor_, "EVENTSTREAM_LENGTH", "headers over the size cap");
if (buffer_.size() < total) return true;
std::uint32_t message_crc = be32(buffer_, total - 4);
if (Crc32::of(buffer_.substr(0, total - 4)) != message_crc)
fail(vendor_, "EVENTSTREAM_CRC", "message CRC mismatch");
EventMessage msg;
parse_headers(vendor_, buffer_.substr(12, headers_len), msg);
msg.payload = buffer_.substr(12 + headers_len, total - 16 - headers_len);
buffer_.erase(0, total);
if (++events_ > BedrockCaps::kMaxStreamEvents) fail(vendor_, "STREAM_TOO_LONG", "too many events");
if (!sink_(msg)) {
stopped_ = true;
return false;
}
}
return true;
}
void EventStreamDecoder::finish() {
if (stopped_) return;
if (json_error_) {
std::string message = "the server answered with an error document instead of a stream";
std::string type = "STREAM_HTTP_ERROR";
try {
clients::body::Json doc = clients::body::Json::parse(buffer_);
for (const char* key : {"message", "Message"})
if (const auto* m = doc.find(key); m && m->is_string()) {
message = m->as_string();
break;
}
if (const auto* t = doc.find("__type"); t && t->is_string()) type = t->as_string();
} catch (const std::exception&) {
}
// The transport hides the HTTP status of a stream; 502 marks "not a stream".
throw llm::ApiError(vendor_, 502, type, message);
}
if (!buffer_.empty()) fail(vendor_, "EVENTSTREAM_TRUNCATED", "stream ended inside a message");
}
std::string EventStreamDecoder::encode(const std::map<std::string, std::string>& string_headers,
const std::string& payload) {
std::string headers;
for (const auto& [name, value] : string_headers) {
headers.push_back(static_cast<char>(name.size()));
headers += name;
headers.push_back(7);
headers.push_back(static_cast<char>((value.size() >> 8) & 0xFF));
headers.push_back(static_cast<char>(value.size() & 0xFF));
headers += value;
}
std::string out;
put32(out, static_cast<std::uint32_t>(16 + headers.size() + payload.size()));
put32(out, static_cast<std::uint32_t>(headers.size()));
put32(out, Crc32::of(out));
out += headers;
out += payload;
put32(out, Crc32::of(out));
return out;
}
void EventStreamDecoder::raise_if_failure(const std::string& vendor, const EventMessage& message) {
if (!message.is_failure()) return;
std::string type = message.message_type() == "exception" ? message.exception_type() : message.header(":error-code");
std::string text = message.header(":error-message");
try {
clients::body::Json doc = clients::body::Json::parse(message.payload);
if (const auto* m = doc.find("message"); m && m->is_string()) text = m->as_string();
} catch (const std::exception&) {
if (text.empty()) text = message.payload.substr(0, 512);
}
if (type.empty()) type = "EVENTSTREAM_ERROR";
int status = 502;
if (type == "throttlingException") status = 429;
else if (type == "validationException") status = 400;
else if (type == "accessDeniedException") status = 403;
else if (type == "resourceNotFoundException") status = 404;
else if (type == "serviceUnavailableException") status = 503;
else if (type == "internalServerException") status = 500;
else if (type == "modelTimeoutException") status = 408;
else if (type == "modelStreamErrorException") status = 424;
throw llm::ApiError(vendor, status, type, text);
}
} // namespace molo::bedrock