#include "vendors/bedrock/converse.h" #include #include "llm/error.h" #include "security/crypto/Base64.h" #include "vendors/bedrock/caps.h" namespace molo::bedrock { namespace { [[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("bedrock", why); } bool one_of(const std::string& v, std::initializer_list allowed) { for (const char* a : allowed) if (v == a) return true; return false; } bool tool_name_ok(const std::string& n) { if (n.empty() || n.size() > 64) return false; for (char c : n) if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-')) return false; return true; } std::string str(const Json* v) { return v && v->is_string() ? v->as_string() : std::string(); } } // namespace // ---- Blocks --------------------------------------------------------------- Json Blocks::text(const std::string& t) { Json b = Json::object(); b.set("text", Json(t)); return b; } Json Blocks::image(const std::string& format, const std::string& bytes) { if (!one_of(format, {"png", "jpeg", "gif", "webp"})) bad("image format must be png, jpeg, gif or webp"); if (bytes.empty()) bad("image bytes are empty"); Json source = Json::object(); source.set("bytes", Json(security::crypto::Base64::encode(bytes))); Json img = Json::object(); img.set("format", Json(format)); img.set("source", source); Json b = Json::object(); b.set("image", img); return b; } Json Blocks::document(const std::string& format, const std::string& name, const std::string& bytes) { if (!one_of(format, {"pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md"})) bad("document format must be pdf, csv, doc, docx, xls, xlsx, html, txt or md"); if (name.empty() || name.size() > 200) bad("document name must be 1..200 characters"); if (bytes.empty()) bad("document bytes are empty"); Json source = Json::object(); source.set("bytes", Json(security::crypto::Base64::encode(bytes))); Json doc = Json::object(); doc.set("format", Json(format)); doc.set("name", Json(name)); doc.set("source", source); Json b = Json::object(); b.set("document", doc); return b; } Json Blocks::tool_use(const std::string& id, const std::string& name, const Json& input) { if (id.empty() || id.size() > 64) bad("toolUseId must be 1..64 characters"); if (!tool_name_ok(name)) bad("tool name must be 1..64 of [A-Za-z0-9_-]"); Json t = Json::object(); t.set("toolUseId", Json(id)); t.set("name", Json(name)); t.set("input", input); Json b = Json::object(); b.set("toolUse", t); return b; } static Json tool_result_with(const std::string& id, const Json& content_block, bool is_error) { if (id.empty() || id.size() > 64) bad("toolUseId must be 1..64 characters"); Json content = Json::array(); content.push(content_block); Json r = Json::object(); r.set("toolUseId", Json(id)); r.set("content", content); r.set("status", Json(is_error ? "error" : "success")); Json b = Json::object(); b.set("toolResult", r); return b; } Json Blocks::tool_result(const std::string& id, const std::string& text_value, bool is_error) { return tool_result_with(id, text(text_value), is_error); } Json Blocks::tool_result_json(const std::string& id, const Json& value, bool is_error) { Json j = Json::object(); j.set("json", value); return tool_result_with(id, j, is_error); } Json Blocks::cache_point() { Json p = Json::object(); p.set("type", Json("default")); Json b = Json::object(); b.set("cachePoint", p); return b; } Json Blocks::message(const std::string& role, const std::vector& content) { if (role != "user" && role != "assistant") bad("message role must be user or assistant"); if (content.empty()) bad("a message needs at least one content block"); if (content.size() > BedrockCaps::kMaxContentBlocks) bad("too many content blocks in one message"); Json arr = Json::array(); for (const Json& c : content) arr.push(c); Json m = Json::object(); m.set("role", Json(role)); m.set("content", arr); return m; } Json Blocks::user(const std::string& t) { return message("user", {text(t)}); } Json Blocks::assistant(const std::string& t) { return message("assistant", {text(t)}); } Json Blocks::tool(const std::string& name, const std::string& description, const Json& json_schema, bool strict) { if (!tool_name_ok(name)) bad("tool name must be 1..64 of [A-Za-z0-9_-]"); if (!json_schema.is_object()) bad("tool input schema must be a JSON object"); Json schema = Json::object(); schema.set("json", json_schema); Json spec = Json::object(); spec.set("name", Json(name)); if (!description.empty()) spec.set("description", Json(description)); spec.set("inputSchema", schema); if (strict) spec.set("strict", Json(true)); Json b = Json::object(); b.set("toolSpec", spec); return b; } // ---- usage ---------------------------------------------------------------- ConverseUsage ConverseUsage::from_json(const Json& u) { ConverseUsage out; auto num = [&](const char* key) { const Json* v = u.is_object() ? u.find(key) : nullptr; return v && v->is_number() ? static_cast(v->as_int()) : 0; }; out.input_tokens = num("inputTokens"); out.output_tokens = num("outputTokens"); out.total_tokens = num("totalTokens"); out.cache_read_tokens = num("cacheReadInputTokens"); out.cache_write_tokens = num("cacheWriteInputTokens"); return out; } // ---- request -------------------------------------------------------------- ConverseRequest& ConverseRequest::add(Json message) { if (messages.size() >= BedrockCaps::kMaxMessages) bad("too many messages"); messages.push_back(std::move(message)); return *this; } ConverseRequest& ConverseRequest::add_user(const std::string& t) { return add(Blocks::user(t)); } ConverseRequest& ConverseRequest::add_assistant(const std::string& t) { return add(Blocks::assistant(t)); } ConverseRequest& ConverseRequest::add_system(const std::string& t) { if (t.empty()) bad("system text is empty"); if (system.size() >= BedrockCaps::kMaxSystemBlocks) bad("too many system blocks"); system.push_back(Blocks::text(t)); return *this; } void ConverseRequest::validate(bool for_stream) const { if (model_id.empty() || model_id.size() > BedrockCaps::kMaxModelIdChars) bad("modelId must be 1..2048 characters"); if (messages.empty()) bad("at least one message is required"); if (messages.size() > BedrockCaps::kMaxMessages) bad("too many messages"); for (const Json& m : messages) { if (!m.is_object()) bad("a message must be an object"); std::string role = str(m.find("role")); if (role != "user" && role != "assistant") bad("message role must be user or assistant"); const Json* c = m.find("content"); if (!c || !c->is_array() || c->as_array().empty()) bad("a message needs a non-empty content array"); if (c->as_array().size() > BedrockCaps::kMaxContentBlocks) bad("too many content blocks in one message"); } if (system.size() > BedrockCaps::kMaxSystemBlocks) bad("too many system blocks"); if (max_tokens && *max_tokens < 1) bad("maxTokens must be at least 1"); if (temperature && (*temperature < 0.0 || *temperature > 1.0)) bad("temperature must be between 0 and 1"); if (top_p && (*top_p < 0.0 || *top_p > 1.0)) bad("topP must be between 0 and 1"); if (stop_sequences.size() > BedrockCaps::kMaxStopSequences) bad("too many stop sequences"); for (const std::string& s : stop_sequences) if (s.empty()) bad("a stop sequence must not be empty"); if (tools.size() > BedrockCaps::kMaxTools) bad("too many tools"); if (!tool_choice.empty() && tools.empty()) bad("toolChoice needs tools"); if (!tool_choice.empty() && tool_choice != "auto" && tool_choice != "any" && !tool_name_ok(tool_choice)) bad("toolChoice must be auto, any or a tool name"); if (!guardrail_id.empty() && guardrail_version.empty()) bad("guardrailVersion is required with a guardrail"); if (!guardrail_trace.empty() && !one_of(guardrail_trace, {"enabled", "disabled", "enabled_full"})) bad("guardrail trace must be enabled, disabled or enabled_full"); if (!guardrail_stream_mode.empty()) { if (!for_stream) bad("streamProcessingMode applies to ConverseStream only"); if (!one_of(guardrail_stream_mode, {"sync", "async"})) bad("streamProcessingMode must be sync or async"); } if (!latency.empty() && !one_of(latency, {"standard", "optimized"})) bad("latency must be standard or optimized"); if (!service_tier.empty() && !one_of(service_tier, {"priority", "default", "flex", "reserved"})) bad("service tier must be priority, default, flex or reserved"); if (request_metadata.size() > 16) bad("requestMetadata holds at most 16 entries"); if (additional_response_paths.size() > 10) bad("additionalModelResponseFieldPaths holds at most 10 entries"); if (!output_schema.is_null() && !output_schema.is_object()) bad("output schema must be a JSON object"); if (effort.size() > 16) bad("effort is at most 16 characters"); if (!additional_fields.is_object()) bad("additionalModelRequestFields must be an object"); } Json ConverseRequest::to_json(bool for_stream) const { validate(for_stream); Json body = Json::object(); Json msgs = Json::array(); for (const Json& m : messages) msgs.push(m); body.set("messages", msgs); if (!system.empty()) { Json s = Json::array(); for (const Json& b : system) s.push(b); body.set("system", s); } Json inf = Json::object(); if (max_tokens) inf.set("maxTokens", Json(*max_tokens)); if (temperature) inf.set("temperature", Json(*temperature)); if (top_p) inf.set("topP", Json(*top_p)); if (!stop_sequences.empty()) { Json s = Json::array(); for (const std::string& x : stop_sequences) s.push(Json(x)); inf.set("stopSequences", s); } if (!inf.as_object().empty()) body.set("inferenceConfig", inf); if (!tools.empty()) { Json cfg = Json::object(); Json t = Json::array(); for (const Json& x : tools) t.push(x); cfg.set("tools", t); if (!tool_choice.empty()) { Json choice = Json::object(); if (tool_choice == "auto") choice.set("auto", Json::object()); else if (tool_choice == "any") choice.set("any", Json::object()); else { Json n = Json::object(); n.set("name", Json(tool_choice)); choice.set("tool", n); } cfg.set("toolChoice", choice); } body.set("toolConfig", cfg); } if (!guardrail_id.empty()) { Json g = Json::object(); g.set("guardrailIdentifier", Json(guardrail_id)); g.set("guardrailVersion", Json(guardrail_version)); if (!guardrail_trace.empty()) g.set("trace", Json(guardrail_trace)); if (for_stream && !guardrail_stream_mode.empty()) g.set("streamProcessingMode", Json(guardrail_stream_mode)); body.set("guardrailConfig", g); } if (!additional_fields.as_object().empty()) body.set("additionalModelRequestFields", additional_fields); if (!additional_response_paths.empty()) { Json p = Json::array(); for (const std::string& x : additional_response_paths) p.push(Json(x)); body.set("additionalModelResponseFieldPaths", p); } if (!request_metadata.empty()) { Json m = Json::object(); for (const auto& [k, v] : request_metadata) m.set(k, Json(v)); body.set("requestMetadata", m); } if (!latency.empty()) { Json p = Json::object(); p.set("latency", Json(latency)); body.set("performanceConfig", p); } if (!service_tier.empty()) { Json t = Json::object(); t.set("type", Json(service_tier)); body.set("serviceTier", t); } Json out = Json::object(); if (output_schema.is_object()) { Json def = Json::object(); def.set("schema", Json(output_schema.dump())); // the schema travels as a JSON string def.set("name", Json(output_schema_name)); Json structure = Json::object(); structure.set("jsonSchema", def); Json fmt = Json::object(); fmt.set("type", Json("json_schema")); fmt.set("structure", structure); out.set("textFormat", fmt); } if (!effort.empty()) out.set("effort", Json(effort)); if (!out.as_object().empty()) body.set("outputConfig", out); return body; } Json ConverseRequest::to_count_json() const { Json full = to_json(false); Json c = Json::object(); for (const char* k : {"messages", "system", "toolConfig", "additionalModelRequestFields"}) if (const Json* v = full.find(k)) c.set(k, *v); Json converse = Json::object(); converse.set("converse", c); return converse; } // ---- result --------------------------------------------------------------- ConverseResult::ConverseResult(Json raw) : raw_(std::move(raw)) { if (const Json* u = raw_.is_object() ? raw_.find("usage") : nullptr) usage_ = ConverseUsage::from_json(*u); } const Json* ConverseResult::content() const { const Json* out = raw_.is_object() ? raw_.find("output") : nullptr; const Json* msg = out && out->is_object() ? out->find("message") : nullptr; const Json* c = msg && msg->is_object() ? msg->find("content") : nullptr; return c && c->is_array() ? c : nullptr; } std::string ConverseResult::stop_reason() const { return str(raw_.find("stopReason")); } long ConverseResult::latency_ms() const { const Json* m = raw_.find("metrics"); const Json* l = m && m->is_object() ? m->find("latencyMs") : nullptr; return l && l->is_number() ? l->as_int() : 0; } std::string ConverseResult::text() const { std::string out; if (const Json* c = content()) for (const Json& b : c->as_array()) if (const Json* t = b.find("text"); t && t->is_string()) out += t->as_string(); return out; } std::string ConverseResult::reasoning() const { std::string out; if (const Json* c = content()) for (const Json& b : c->as_array()) if (const Json* r = b.find("reasoningContent")) if (const Json* rt = r->find("reasoningText")) out += str(rt->find("text")); return out; } std::string ConverseResult::reasoning_signature() const { if (const Json* c = content()) for (const Json& b : c->as_array()) if (const Json* r = b.find("reasoningContent")) if (const Json* rt = r->find("reasoningText")) if (std::string s = str(rt->find("signature")); !s.empty()) return s; return ""; } std::vector ConverseResult::tool_uses() const { std::vector out; if (const Json* c = content()) for (const Json& b : c->as_array()) if (const Json* t = b.find("toolUse")) { ConverseToolUse u; u.id = str(t->find("toolUseId")); u.name = str(t->find("name")); if (const Json* in = t->find("input")) u.input = *in; out.push_back(std::move(u)); } return out; } Json ConverseResult::assistant_message() const { if (const Json* out = raw_.find("output")) if (const Json* m = out->find("message")) return *m; return Json::object(); } Json ConverseResult::trace() const { const Json* t = raw_.find("trace"); return t ? *t : Json(); } // ---- stream --------------------------------------------------------------- StreamDelta ConverseStreamAccumulator::on_event(const std::string& event_type, const Json& p) { ++events_; StreamDelta d; d.event = event_type; const Json* idx = p.is_object() ? p.find("contentBlockIndex") : nullptr; if (idx && idx->is_number()) d.block_index = static_cast(idx->as_int()); if (event_type == "messageStart") { if (std::string r = str(p.find("role")); !r.empty()) role_ = r; } else if (event_type == "contentBlockStart") { const Json* start = p.find("start"); const Json* tu = start ? start->find("toolUse") : nullptr; if (tu && d.block_index >= 0) { Block& b = blocks_[d.block_index]; b.kind = "tool"; b.tool_id = d.tool_id = str(tu->find("toolUseId")); b.tool_name = d.tool_name = str(tu->find("name")); } } else if (event_type == "contentBlockDelta") { const Json* delta = p.find("delta"); if (delta && d.block_index >= 0) { Block& b = blocks_[d.block_index]; if (const Json* t = delta->find("text")) { if (b.kind.empty()) b.kind = "text"; b.text += d.text = str(t); } else if (const Json* tu = delta->find("toolUse")) { b.kind = "tool"; b.tool_args += d.tool_args = str(tu->find("input")); } else if (const Json* rc = delta->find("reasoningContent")) { b.kind = "reasoning"; if (const Json* rt = rc->find("text")) b.text += d.reasoning = str(rt); if (const Json* sg = rc->find("signature")) b.signature += d.reasoning_signature = str(sg); } } } else if (event_type == "messageStop") { stop_reason_ = d.stop_reason = str(p.find("stopReason")); } else if (event_type == "metadata") { if (const Json* u = p.find("usage")) { usage_ = *u; d.usage = ConverseUsage::from_json(*u); } if (const Json* m = p.find("metrics")) metrics_ = *m; if (const Json* t = p.find("trace")) trace_ = *t; } return d; } ConverseResult ConverseStreamAccumulator::result() const { Json content = Json::array(); for (const auto& [index, b] : blocks_) { (void)index; if (b.kind == "tool") { Json input = Json::object(); if (!b.tool_args.empty()) { try { input = Json::parse(b.tool_args); } catch (const std::exception&) { input = Json::object(); input.set("_invalid_json", Json(b.tool_args)); } } content.push(Blocks::tool_use(b.tool_id.empty() ? "tool" : b.tool_id, b.tool_name.empty() ? "tool" : b.tool_name, input)); } else if (b.kind == "reasoning") { Json rt = Json::object(); rt.set("text", Json(b.text)); if (!b.signature.empty()) rt.set("signature", Json(b.signature)); Json rc = Json::object(); rc.set("reasoningText", rt); Json blk = Json::object(); blk.set("reasoningContent", rc); content.push(blk); } else { content.push(Blocks::text(b.text)); } } Json msg = Json::object(); msg.set("role", Json(role_)); msg.set("content", content); Json out = Json::object(); out.set("message", msg); Json raw = Json::object(); raw.set("output", out); raw.set("stopReason", Json(stop_reason_)); raw.set("usage", usage_); raw.set("metrics", metrics_); if (!trace_.is_null()) raw.set("trace", trace_); return ConverseResult(raw); } } // namespace molo::bedrock