feat(clients): bedrock: SigV4 signer, Converse, event streams, guardrails, agents, KB, batch
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,649 @@
|
||||
#include "../AiDemos.hpp"
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "vendors/bedrock/agent.h"
|
||||
#include "vendors/bedrock/calls.h"
|
||||
#include "vendors/bedrock/caps.h"
|
||||
#include "vendors/bedrock/chat_model.h"
|
||||
#include "vendors/bedrock/client.h"
|
||||
#include "vendors/bedrock/control.h"
|
||||
#include "vendors/bedrock/credentials.h"
|
||||
#include "vendors/bedrock/runtime.h"
|
||||
#include "vendors/bedrock/sigv4.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// bedrock. Stub: the package agent fills this in.
|
||||
// Amazon Bedrock over the real src/vendors/bedrock package. GET routes need no
|
||||
// credential. Every route that calls AWS is a POST: 422 on bad input, 503
|
||||
// not_configured without AWS_BEARER_TOKEN_BEDROCK or AWS_ACCESS_KEY_ID +
|
||||
// AWS_SECRET_ACCESS_KEY. Routes that cost money or change a resource (inference,
|
||||
// guardrail checks, agents, retrieval, batch create / stop) are dry runs that
|
||||
// show the exact request unless the body has "confirm":true. /ai/bedrock/sign
|
||||
// is local: it signs with AWS's documentation credentials only.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace bk = bedrock;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
constexpr std::size_t kMaxBody = 8 * 1024 * 1024;
|
||||
constexpr std::size_t kMaxStreamChunks = 400;
|
||||
|
||||
const char kPage[] = R"BEDROCK({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Amazon Bedrock</h1>
|
||||
<p class="hint">Converse and ConverseStream, InvokeModel (and stream), count tokens, guardrails, agents,
|
||||
knowledge base retrieval, models, inference profiles, batch jobs, and a local SigV4 signer. Paid and
|
||||
changing calls are dry runs until the body says <code>"confirm":true</code>.</p>
|
||||
<p id="bk-config" class="hint">Checking credential...</p>
|
||||
<section class="bk">
|
||||
<label>Route
|
||||
<select id="bk-route"></select></label>
|
||||
<label>Body (JSON object) <textarea id="bk-body" rows="8"></textarea></label>
|
||||
<div class="bk-actions">
|
||||
<button id="bk-send" type="button" class="primary">Send</button>
|
||||
<button id="bk-confirm" type="button">Send with confirm</button>
|
||||
</div>
|
||||
<p id="bk-error" class="error" role="alert"></p>
|
||||
<pre id="bk-out" class="bk-out"></pre>
|
||||
</section>
|
||||
<style>
|
||||
.bk label{display:block;margin:.5rem 0}
|
||||
.bk select,.bk textarea{width:100%;box-sizing:border-box}
|
||||
.bk-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
||||
.bk-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
|
||||
</style>
|
||||
<script src="/embedded/ai_bedrock.js"></script>
|
||||
{% endblock %}
|
||||
)BEDROCK";
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
||||
if (req.body.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(req.body);
|
||||
if (!j.is_object()) bad("body must be a JSON object");
|
||||
return j;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
bad("body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
std::string text(const Json& b, const char* key, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return "";
|
||||
}
|
||||
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
||||
return v->as_string();
|
||||
}
|
||||
|
||||
std::optional<int> number(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return std::nullopt;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
return static_cast<int>(v->as_int());
|
||||
}
|
||||
|
||||
std::optional<double> real(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return std::nullopt;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
return v->as_number();
|
||||
}
|
||||
|
||||
bool flag(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return false;
|
||||
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
||||
return v->as_bool();
|
||||
}
|
||||
|
||||
std::vector<std::string> strings(const Json& b, const char* key) {
|
||||
std::vector<std::string> out;
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return out;
|
||||
if (!v->is_array()) bad(std::string(key) + " must be an array of strings");
|
||||
for (const Json& s : v->as_array()) {
|
||||
if (!s.is_string()) bad(std::string(key) + " must be an array of strings");
|
||||
out.push_back(s.as_string());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
Json object_at(const Json& b, const char* key, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return Json::object();
|
||||
}
|
||||
if (!v->is_object()) bad(std::string(key) + " must be an object");
|
||||
return *v;
|
||||
}
|
||||
|
||||
Json usage_json(const bk::ConverseUsage& u) {
|
||||
Json o = Json::object();
|
||||
o.set("input_tokens", Json(u.input_tokens));
|
||||
o.set("output_tokens", Json(u.output_tokens));
|
||||
o.set("total_tokens", Json(u.total_tokens));
|
||||
o.set("cache_read_tokens", Json(u.cache_read_tokens));
|
||||
o.set("cache_write_tokens", Json(u.cache_write_tokens));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json converse_json(const bk::ConverseResult& r) {
|
||||
Json o = Json::object();
|
||||
o.set("text", Json(r.text()));
|
||||
o.set("reasoning", Json(r.reasoning()));
|
||||
o.set("stop_reason", Json(r.stop_reason()));
|
||||
o.set("latency_ms", Json(static_cast<int>(r.latency_ms())));
|
||||
o.set("usage", usage_json(r.usage()));
|
||||
Json calls = Json::array();
|
||||
for (const bk::ConverseToolUse& t : r.tool_uses()) {
|
||||
Json x = Json::object();
|
||||
x.set("id", Json(t.id));
|
||||
x.set("name", Json(t.name));
|
||||
x.set("input", t.input);
|
||||
calls.push(x);
|
||||
}
|
||||
o.set("tool_uses", calls);
|
||||
return o;
|
||||
}
|
||||
|
||||
Json list_json(const bk::ListResult& l, const char* key) {
|
||||
Json o = Json::object();
|
||||
o.set(key, l.items);
|
||||
o.set("count", Json(static_cast<int>(l.items.as_array().size())));
|
||||
o.set("pages", Json(static_cast<int>(l.pages)));
|
||||
o.set("truncated", Json(l.truncated));
|
||||
if (!l.next_token.empty()) o.set("next_token", Json(l.next_token));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json dry_run(const Json& request, const std::string& method, const std::string& path) {
|
||||
Json o = Json::object();
|
||||
o.set("dry_run", Json(true));
|
||||
o.set("would_send_to", Json(method + " " + path));
|
||||
o.set("request", request);
|
||||
o.set("hint", Json("send confirm:true to run it"));
|
||||
return o;
|
||||
}
|
||||
|
||||
bk::ConverseRequest converse_request(const Json& b) {
|
||||
bk::ConverseRequest r;
|
||||
r.model_id = text(b, "model");
|
||||
if (r.model_id.empty()) r.model_id = bk::BedrockChatModel::kDefaultModel;
|
||||
std::string system = text(b, "system");
|
||||
if (!system.empty()) r.add_system(system);
|
||||
r.add_user(text(b, "prompt", true));
|
||||
r.max_tokens = number(b, "max_tokens");
|
||||
r.temperature = real(b, "temperature");
|
||||
r.top_p = real(b, "top_p");
|
||||
r.stop_sequences = strings(b, "stop");
|
||||
r.effort = text(b, "effort");
|
||||
r.guardrail_id = text(b, "guardrail_id");
|
||||
r.guardrail_version = text(b, "guardrail_version");
|
||||
r.service_tier = text(b, "service_tier");
|
||||
r.latency = text(b, "latency");
|
||||
r.additional_fields = object_at(b, "additional_fields");
|
||||
return r;
|
||||
}
|
||||
|
||||
// One route = validate into a Plan, then the generic runner decides dry run,
|
||||
// credential, call and error mapping.
|
||||
struct Plan {
|
||||
Json request; // what would be sent (shown in a dry run)
|
||||
std::string method = "POST";
|
||||
std::string path;
|
||||
bool needs_confirm = false;
|
||||
std::function<Json()> call;
|
||||
};
|
||||
|
||||
class BedrockRoutes {
|
||||
public:
|
||||
explicit BedrockRoutes(std::shared_ptr<bk::BedrockClient> client) : client_(std::move(client)) {}
|
||||
|
||||
const bk::BedrockClient& client() const { return *client_; }
|
||||
|
||||
using Planner = std::function<Plan(const Json&)>;
|
||||
|
||||
fw::Response run(const fw::Request& req, const Planner& planner) const {
|
||||
Json body = parse_body(req);
|
||||
Plan plan;
|
||||
try {
|
||||
plan = planner(body);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.method, plan.path));
|
||||
try {
|
||||
return json_out(plan.call());
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- planners -----------------------------------------------------------
|
||||
Plan converse(const Json& b) const {
|
||||
bk::ConverseRequest r = converse_request(b);
|
||||
Json wire = r.to_json(false);
|
||||
auto rt = std::make_shared<bk::Runtime>(client_);
|
||||
Plan pl{wire, "POST", bk::Calls::render(bk::Call::kConverse, {{"modelId", r.model_id}}), true, nullptr};
|
||||
pl.call = [rt, r] { return converse_json(rt->converse(r)); };
|
||||
return pl;
|
||||
}
|
||||
Plan converse_stream(const Json& b) const {
|
||||
bk::ConverseRequest r = converse_request(b);
|
||||
Json wire = r.to_json(true);
|
||||
auto rt = std::make_shared<bk::Runtime>(client_);
|
||||
Plan pl{wire, "POST", bk::Calls::render(bk::Call::kConverseStream, {{"modelId", r.model_id}}), true, nullptr};
|
||||
pl.call = [rt, r] {
|
||||
std::size_t events = 0;
|
||||
bk::ConverseResult res = rt->converse_stream(r, [&](const bk::StreamDelta&) {
|
||||
return ++events < bk::BedrockCaps::kMaxStreamEvents;
|
||||
});
|
||||
Json o = converse_json(res);
|
||||
o.set("events", Json(static_cast<int>(events)));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan invoke(const Json& b) const {
|
||||
std::string model = text(b, "model", true);
|
||||
Json body = object_at(b, "body", true);
|
||||
bk::InvokeOptions opt;
|
||||
if (std::string ct = text(b, "content_type"); !ct.empty()) opt.content_type = ct;
|
||||
opt.trace = text(b, "trace");
|
||||
opt.guardrail_id = text(b, "guardrail_id");
|
||||
opt.guardrail_version = text(b, "guardrail_version");
|
||||
opt.service_tier = text(b, "service_tier");
|
||||
opt.latency = text(b, "latency");
|
||||
opt.validate();
|
||||
if (!bk::Calls::valid_path_value(model)) bad("model has invalid characters");
|
||||
auto rt = std::make_shared<bk::Runtime>(client_);
|
||||
Plan pl{body, "POST", bk::Calls::render(bk::Call::kInvokeModel, {{"modelId", model}}), true, nullptr};
|
||||
pl.call = [rt, model, body, opt] {
|
||||
bk::InvokeResult r = rt->invoke_model(model, body, opt);
|
||||
Json o = Json::object();
|
||||
o.set("content_type", Json(r.content_type));
|
||||
try {
|
||||
o.set("body", r.json());
|
||||
} catch (const llm::ApiError&) {
|
||||
o.set("body_text", Json(r.body.substr(0, 20000)));
|
||||
}
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan invoke_stream(const Json& b) const {
|
||||
std::string model = text(b, "model", true);
|
||||
Json body = object_at(b, "body", true);
|
||||
if (!bk::Calls::valid_path_value(model)) bad("model has invalid characters");
|
||||
auto rt = std::make_shared<bk::Runtime>(client_);
|
||||
Plan pl{body, "POST", bk::Calls::render(bk::Call::kInvokeModelStream, {{"modelId", model}}), true, nullptr};
|
||||
pl.call = [rt, model, body] {
|
||||
Json chunks = Json::array();
|
||||
std::size_t n = rt->invoke_model_stream(model, body, [&](const std::string& c) {
|
||||
chunks.push(Json(c.substr(0, 4000)));
|
||||
return chunks.as_array().size() < kMaxStreamChunks;
|
||||
});
|
||||
Json o = Json::object();
|
||||
o.set("chunks", chunks);
|
||||
o.set("count", Json(static_cast<int>(n)));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan count_tokens(const Json& b) const {
|
||||
bk::ConverseRequest r = converse_request(b);
|
||||
Json wire = r.to_count_json();
|
||||
auto rt = std::make_shared<bk::Runtime>(client_);
|
||||
Plan pl{wire, "POST", bk::Calls::render(bk::Call::kCountTokens, {{"modelId", r.model_id}}), false, nullptr};
|
||||
pl.call = [rt, r] {
|
||||
Json o = Json::object();
|
||||
o.set("input_tokens", Json(static_cast<int>(rt->count_tokens(r))));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan guardrail(const Json& b) const {
|
||||
std::string id = text(b, "guardrail_id", true);
|
||||
std::string version = text(b, "version");
|
||||
if (version.empty()) version = "DRAFT";
|
||||
std::string source = text(b, "source");
|
||||
if (source.empty()) source = "INPUT";
|
||||
std::string scope = text(b, "scope");
|
||||
Json block = bk::Runtime::guardrail_text(text(b, "text", true), strings(b, "qualifiers"));
|
||||
if (!bk::Calls::valid_path_value(id) || !bk::Calls::valid_path_value(version)) bad("guardrail id or version has invalid characters");
|
||||
if (source != "INPUT" && source != "OUTPUT") bad("source must be INPUT or OUTPUT");
|
||||
if (!scope.empty() && scope != "INTERVENTIONS" && scope != "FULL") bad("scope must be INTERVENTIONS or FULL");
|
||||
Json req = Json::object();
|
||||
req.set("source", Json(source));
|
||||
Json content = Json::array();
|
||||
content.push(block);
|
||||
req.set("content", content);
|
||||
auto rt = std::make_shared<bk::Runtime>(client_);
|
||||
Plan pl{req, "POST", bk::Calls::render(bk::Call::kApplyGuardrail, {{"guardrailIdentifier", id}, {"guardrailVersion", version}}), true, nullptr};
|
||||
pl.call = [rt, id, version, source, block, scope] {
|
||||
bk::GuardrailResult r = rt->apply_guardrail(id, version, source, {block}, scope);
|
||||
Json o = Json::object();
|
||||
o.set("action", Json(r.action()));
|
||||
o.set("intervened", Json(r.intervened()));
|
||||
o.set("output_text", Json(r.output_text()));
|
||||
o.set("raw", r.raw);
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan agent(const Json& b) const {
|
||||
bk::AgentInvokeParams p;
|
||||
p.agent_id = text(b, "agent_id", true);
|
||||
p.alias_id = text(b, "alias_id", true);
|
||||
p.session_id = text(b, "session_id", true);
|
||||
p.input_text = text(b, "prompt", true);
|
||||
p.enable_trace = flag(b, "trace");
|
||||
p.end_session = flag(b, "end_session");
|
||||
p.memory_id = text(b, "memory_id");
|
||||
Json wire = p.to_json();
|
||||
auto ar = std::make_shared<bk::AgentRuntime>(client_);
|
||||
Plan pl{wire, "POST", bk::Calls::render(bk::Call::kInvokeAgent, {{"agentId", p.agent_id}, {"agentAliasId", p.alias_id}, {"sessionId", p.session_id}}), true, nullptr};
|
||||
pl.call = [ar, p] {
|
||||
bk::AgentResult r = ar->invoke_agent(p);
|
||||
Json o = Json::object();
|
||||
o.set("text", Json(r.text));
|
||||
o.set("session_id", Json(r.session_id));
|
||||
o.set("chunks", Json(static_cast<int>(r.chunks)));
|
||||
o.set("traces", Json(static_cast<int>(r.traces.as_array().size())));
|
||||
o.set("return_control", r.return_control);
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan retrieve(const Json& b) const {
|
||||
bk::RetrieveParams p;
|
||||
p.knowledge_base_id = text(b, "knowledge_base_id", true);
|
||||
p.query = text(b, "query", true);
|
||||
p.number_of_results = number(b, "number_of_results").value_or(0);
|
||||
p.search_type = text(b, "search_type");
|
||||
p.next_token = text(b, "next_token");
|
||||
if (const Json* f = b.find("filter"); f && f->is_object()) p.filter = *f;
|
||||
Json wire = p.to_json();
|
||||
auto ar = std::make_shared<bk::AgentRuntime>(client_);
|
||||
Plan pl{wire, "POST", bk::Calls::render(bk::Call::kRetrieve, {{"knowledgeBaseId", p.knowledge_base_id}}), true, nullptr};
|
||||
pl.call = [ar, p] {
|
||||
bk::RetrieveResult r = ar->retrieve(p);
|
||||
Json o = Json::object();
|
||||
o.set("results", r.results);
|
||||
o.set("count", Json(static_cast<int>(r.results.as_array().size())));
|
||||
if (!r.next_token.empty()) o.set("next_token", Json(r.next_token));
|
||||
if (!r.guardrail_action.empty()) o.set("guardrail_action", Json(r.guardrail_action));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
Plan models(const Json& b) const {
|
||||
std::string action = text(b, "action");
|
||||
if (action.empty()) action = "list";
|
||||
auto api = std::make_shared<bk::Models>(client_);
|
||||
if (action == "list") {
|
||||
bk::ModelFilter f;
|
||||
f.provider = text(b, "provider");
|
||||
f.output_modality = text(b, "output_modality");
|
||||
f.inference_type = text(b, "inference_type");
|
||||
f.customization_type = text(b, "customization_type");
|
||||
Json req = Json::object();
|
||||
req.set("provider", Json(f.provider));
|
||||
return Plan{req, "GET", "/foundation-models", false, [api, f] { return list_json(api->list(f), "models"); }};
|
||||
}
|
||||
if (action == "get") {
|
||||
std::string id = text(b, "id", true);
|
||||
if (!bk::Calls::valid_path_value(id)) bad("id has invalid characters");
|
||||
Json req = Json::object();
|
||||
req.set("id", Json(id));
|
||||
return Plan{req, "GET", bk::Calls::render(bk::Call::kGetFoundationModel, {{"modelIdentifier", id}}), false, [api, id] {
|
||||
Json o = Json::object();
|
||||
o.set("model", api->get(id));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
bad("action must be list or get");
|
||||
}
|
||||
Plan profiles(const Json& b) const {
|
||||
std::string action = text(b, "action");
|
||||
if (action.empty()) action = "list";
|
||||
auto api = std::make_shared<bk::Models>(client_);
|
||||
if (action == "list") {
|
||||
std::string type = text(b, "type");
|
||||
int max = number(b, "max_results").value_or(100);
|
||||
if (max < 1 || max > bk::BedrockCaps::kMaxResults) bad("max_results must be 1..1000");
|
||||
Json req = Json::object();
|
||||
req.set("type", Json(type));
|
||||
return Plan{req, "GET", "/inference-profiles", false, [api, type, max] { return list_json(api->list_profiles(type, max), "profiles"); }};
|
||||
}
|
||||
if (action == "get") {
|
||||
std::string id = text(b, "id", true);
|
||||
if (!bk::Calls::valid_path_value(id)) bad("id has invalid characters");
|
||||
Json req = Json::object();
|
||||
req.set("id", Json(id));
|
||||
return Plan{req, "GET", bk::Calls::render(bk::Call::kGetInferenceProfile, {{"inferenceProfileIdentifier", id}}), false, [api, id] {
|
||||
Json o = Json::object();
|
||||
o.set("profile", api->get_profile(id));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
bad("action must be list or get");
|
||||
}
|
||||
Plan guardrails(const Json& b) const {
|
||||
std::string action = text(b, "action");
|
||||
if (action.empty()) action = "list";
|
||||
auto api = std::make_shared<bk::Guardrails>(client_);
|
||||
if (action == "list") {
|
||||
std::string id = text(b, "id");
|
||||
if (!id.empty() && !bk::Calls::valid_path_value(id)) bad("id has invalid characters");
|
||||
Json req = Json::object();
|
||||
req.set("id", Json(id));
|
||||
return Plan{req, "GET", "/guardrails", false, [api, id] { return list_json(api->list(id), "guardrails"); }};
|
||||
}
|
||||
if (action == "get") {
|
||||
std::string id = text(b, "id", true);
|
||||
std::string version = text(b, "version");
|
||||
if (!bk::Calls::valid_path_value(id)) bad("id has invalid characters");
|
||||
Json req = Json::object();
|
||||
req.set("id", Json(id));
|
||||
return Plan{req, "GET", bk::Calls::render(bk::Call::kGetGuardrail, {{"guardrailIdentifier", id}}), false, [api, id, version] {
|
||||
Json o = Json::object();
|
||||
o.set("guardrail", api->get(id, version));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
bad("action must be list or get");
|
||||
}
|
||||
Plan batch(const Json& b) const {
|
||||
std::string action = text(b, "action", true);
|
||||
auto api = std::make_shared<bk::BatchJobs>(client_);
|
||||
if (action == "list") {
|
||||
std::string status = text(b, "status"), name = text(b, "name_contains");
|
||||
Json req = Json::object();
|
||||
req.set("status", Json(status));
|
||||
return Plan{req, "GET", "/model-invocation-jobs", false, [api, status, name] { return list_json(api->list(status, name), "jobs"); }};
|
||||
}
|
||||
if (action == "get" || action == "stop") {
|
||||
std::string id = text(b, "id", true);
|
||||
if (!bk::Calls::valid_path_value(id)) bad("id has invalid characters");
|
||||
Json req = Json::object();
|
||||
req.set("id", Json(id));
|
||||
if (action == "get")
|
||||
return Plan{req, "GET", bk::Calls::render(bk::Call::kGetBatchJob, {{"jobIdentifier", id}}), false, [api, id] {
|
||||
Json o = Json::object();
|
||||
o.set("job", api->get(id));
|
||||
return o;
|
||||
}};
|
||||
return Plan{req, "POST", bk::Calls::render(bk::Call::kStopBatchJob, {{"jobIdentifier", id}}), true, [api, id] {
|
||||
api->stop(id);
|
||||
Json o = Json::object();
|
||||
o.set("stopping", Json(id));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
if (action == "create") {
|
||||
bk::BatchJobParams p;
|
||||
p.job_name = text(b, "job_name", true);
|
||||
p.role_arn = text(b, "role_arn", true);
|
||||
p.model_id = text(b, "model", true);
|
||||
p.input_s3_uri = text(b, "input_s3_uri", true);
|
||||
p.output_s3_uri = text(b, "output_s3_uri", true);
|
||||
p.invocation_type = text(b, "invocation_type");
|
||||
p.timeout_hours = number(b, "timeout_hours").value_or(0);
|
||||
Json wire = p.to_json();
|
||||
return Plan{wire, "POST", "/model-invocation-job", true, [api, p] {
|
||||
Json o = Json::object();
|
||||
o.set("job_arn", Json(api->create(p)));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
bad("action must be list, get, create or stop");
|
||||
}
|
||||
// Local: signs with AWS's documentation credentials, never a real secret.
|
||||
Plan sign(const Json& b) const {
|
||||
bk::AwsCredentials creds{"AKIDEXAMPLE", "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", ""};
|
||||
std::string region = text(b, "region");
|
||||
if (region.empty()) region = "us-east-1";
|
||||
if (!bk::Endpoints::valid_region(region)) bad("bad region name");
|
||||
std::string service = text(b, "service");
|
||||
if (service.empty()) service = "bedrock";
|
||||
std::string method = text(b, "method");
|
||||
if (method.empty()) method = "GET";
|
||||
if (method != "GET" && method != "POST" && method != "PUT" && method != "DELETE" && method != "PATCH") bad("unsupported method");
|
||||
bk::SigV4Request r;
|
||||
r.method = method;
|
||||
r.path = text(b, "path");
|
||||
if (r.path.empty()) r.path = "/";
|
||||
if (r.path[0] != '/') bad("path must start with /");
|
||||
r.query = text(b, "query");
|
||||
r.body = text(b, "body");
|
||||
std::string host = text(b, "host");
|
||||
r.headers.emplace_back("Host", host.empty() ? "example.amazonaws.com" : host);
|
||||
std::string date = text(b, "date");
|
||||
if (date.empty()) date = "20150830T123600Z";
|
||||
Json req = Json::object();
|
||||
req.set("method", Json(method));
|
||||
req.set("path", Json(r.path));
|
||||
Plan pl{req, "LOCAL", "", false, nullptr};
|
||||
pl.call = [creds, region, service, r, date] {
|
||||
bk::SigV4Result s = bk::SigV4Signer(creds, region, service).sign(r, date);
|
||||
Json o = Json::object();
|
||||
o.set("canonical_request", Json(s.canonical_request));
|
||||
o.set("string_to_sign", Json(s.string_to_sign));
|
||||
o.set("signed_headers", Json(s.signed_headers));
|
||||
o.set("signature", Json(s.signature));
|
||||
o.set("authorization", Json(s.authorization));
|
||||
o.set("note", Json("signed with AWS documentation credentials, not yours"));
|
||||
return o;
|
||||
};
|
||||
return pl;
|
||||
}
|
||||
|
||||
private:
|
||||
[[noreturn]] static void map(const llm::ApiError& e) {
|
||||
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") throw fw::HttpError(503, "not_configured", e.message());
|
||||
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
throw fw::HttpError(502, "bedrock_error", std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
|
||||
}
|
||||
std::shared_ptr<bk::BedrockClient> client_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_ai_bedrock_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
auto routes = std::make_shared<BedrockRoutes>(bk::BedrockClient::from_env());
|
||||
fw::Environment* env = services.templates.get();
|
||||
services.templates->add_template("ai_bedrock.html", kPage);
|
||||
services.add_feature("ai-bedrock", "/ai/bedrock");
|
||||
|
||||
router.get("/ai/bedrock", "ai-bedrock-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Amazon Bedrock");
|
||||
return fw::Response(200, env->render("ai_bedrock.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/ai/bedrock/status", "ai-bedrock-status", [routes](fw::Request req) {
|
||||
(void)req;
|
||||
const bk::Credentials& c = routes->client().credentials();
|
||||
Json o = Json::object();
|
||||
o.set("configured", Json(routes->client().configured()));
|
||||
o.set("auth_mode", Json(c.mode_name()));
|
||||
o.set("region", Json(c.region));
|
||||
Json env = Json::array();
|
||||
for (const char* n : {bk::Credentials::kEnvBearer, bk::Credentials::kEnvAccessKey, bk::Credentials::kEnvSecretKey,
|
||||
bk::Credentials::kEnvSessionToken, bk::Credentials::kEnvRegion})
|
||||
env.push(Json(n));
|
||||
o.set("credential_env", env);
|
||||
Json hosts = Json::object();
|
||||
hosts.set("runtime", Json(bk::Endpoints::host(bk::AwsService::kRuntime, c.region)));
|
||||
hosts.set("control", Json(bk::Endpoints::host(bk::AwsService::kControl, c.region)));
|
||||
hosts.set("agent_runtime", Json(bk::Endpoints::host(bk::AwsService::kAgentRuntime, c.region)));
|
||||
o.set("hosts", hosts);
|
||||
o.set("default_model", Json(bk::BedrockChatModel::kDefaultModel));
|
||||
o.set("calls", Json(static_cast<int>(bk::Calls::count())));
|
||||
o.set("max_invoke_body_bytes", Json(static_cast<int>(bk::BedrockCaps::kMaxInvokeBodyBytes)));
|
||||
o.set("max_pages", Json(static_cast<int>(bk::BedrockCaps::kMaxPages)));
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
router.get("/ai/bedrock/calls", "ai-bedrock-calls", [](fw::Request req) {
|
||||
(void)req;
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < bk::Calls::count(); ++i) {
|
||||
const bk::CallSpec& s = bk::Calls::at(i);
|
||||
Json row = Json::object();
|
||||
row.set("operation", Json(s.op));
|
||||
row.set("http", Json(s.http_method));
|
||||
row.set("path", Json(s.uri));
|
||||
row.set("host", Json(bk::Endpoints::prefix(s.service)));
|
||||
row.set("bearer_ok", Json(s.bearer_ok));
|
||||
row.set("event_stream", Json(s.event_stream));
|
||||
arr.push(row);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("calls", arr);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
using Member = BedrockRoutes::Planner;
|
||||
auto add = [&](const char* path, const char* name, Member planner) {
|
||||
router.post(path, name, [routes, planner](fw::Request req) { return routes->run(req, planner); });
|
||||
};
|
||||
BedrockRoutes* r = routes.get();
|
||||
auto bind = [r](Plan (BedrockRoutes::*m)(const Json&) const) {
|
||||
return Member([r, m](const Json& b) { return (r->*m)(b); });
|
||||
};
|
||||
add("/ai/bedrock/converse", "ai-bedrock-converse", bind(&BedrockRoutes::converse));
|
||||
add("/ai/bedrock/converse-stream", "ai-bedrock-converse-stream", bind(&BedrockRoutes::converse_stream));
|
||||
add("/ai/bedrock/invoke", "ai-bedrock-invoke", bind(&BedrockRoutes::invoke));
|
||||
add("/ai/bedrock/invoke-stream", "ai-bedrock-invoke-stream", bind(&BedrockRoutes::invoke_stream));
|
||||
add("/ai/bedrock/count-tokens", "ai-bedrock-count-tokens", bind(&BedrockRoutes::count_tokens));
|
||||
add("/ai/bedrock/guardrail", "ai-bedrock-guardrail", bind(&BedrockRoutes::guardrail));
|
||||
add("/ai/bedrock/agent", "ai-bedrock-agent", bind(&BedrockRoutes::agent));
|
||||
add("/ai/bedrock/retrieve", "ai-bedrock-retrieve", bind(&BedrockRoutes::retrieve));
|
||||
add("/ai/bedrock/models", "ai-bedrock-models", bind(&BedrockRoutes::models));
|
||||
add("/ai/bedrock/profiles", "ai-bedrock-profiles", bind(&BedrockRoutes::profiles));
|
||||
add("/ai/bedrock/guardrails", "ai-bedrock-guardrails", bind(&BedrockRoutes::guardrails));
|
||||
add("/ai/bedrock/batch", "ai-bedrock-batch", bind(&BedrockRoutes::batch));
|
||||
add("/ai/bedrock/sign", "ai-bedrock-sign", bind(&BedrockRoutes::sign));
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
// Bedrock page: thin client over /ai/bedrock/* (all limits server side).
|
||||
(function () {
|
||||
"use strict";
|
||||
var $ = function (id) { return document.getElementById(id); };
|
||||
var out = $("bk-out"), err = $("bk-error");
|
||||
var templates = {
|
||||
"converse": { prompt: "Hello!", system: "Be brief.", max_tokens: 256, temperature: 0.5 },
|
||||
"converse-stream": { prompt: "Count to five.", max_tokens: 128 },
|
||||
"invoke": { model: "amazon.titan-embed-text-v2:0", body: { inputText: "hello" } },
|
||||
"invoke-stream": { model: "anthropic.claude-3-haiku-20240307-v1:0", body: { anthropic_version: "bedrock-2023-05-31", max_tokens: 64, messages: [{ role: "user", content: "Hi" }] } },
|
||||
"count-tokens": { prompt: "How many tokens is this?" },
|
||||
"guardrail": { guardrail_id: "", version: "DRAFT", source: "INPUT", text: "my card number is 4111 1111 1111 1111" },
|
||||
"agent": { agent_id: "", alias_id: "", session_id: "demo-session-1", prompt: "Hello" },
|
||||
"retrieve": { knowledge_base_id: "", query: "What is AWS?", number_of_results: 5 },
|
||||
"models": { action: "list", provider: "Anthropic", output_modality: "TEXT" },
|
||||
"profiles": { action: "list", type: "SYSTEM_DEFINED" },
|
||||
"guardrails": { action: "list" },
|
||||
"batch": { action: "list" },
|
||||
"sign": { method: "POST", path: "/model/anthropic.claude-3-sonnet-20240229-v1%3A0/converse", body: "{}", host: "bedrock-runtime.us-east-1.amazonaws.com" }
|
||||
};
|
||||
var sel = $("bk-route");
|
||||
Object.keys(templates).forEach(function (k) {
|
||||
var o = document.createElement("option");
|
||||
o.value = k; o.textContent = k;
|
||||
sel.appendChild(o);
|
||||
});
|
||||
function fill() { $("bk-body").value = JSON.stringify(templates[sel.value], null, 1); }
|
||||
sel.onchange = fill;
|
||||
fill();
|
||||
|
||||
function send(confirm) {
|
||||
var body;
|
||||
try { body = JSON.parse($("bk-body").value || "{}"); }
|
||||
catch (e) { err.textContent = "Body is not valid JSON"; return; }
|
||||
if (confirm) body.confirm = true;
|
||||
err.textContent = "";
|
||||
out.textContent = "...";
|
||||
fetch("/ai/bedrock/" + sel.value, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body)
|
||||
}).then(function (res) {
|
||||
return res.text().then(function (t) {
|
||||
var j;
|
||||
try { j = JSON.parse(t); } catch (e) { j = { message: t }; }
|
||||
if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; }
|
||||
else { out.textContent = JSON.stringify(j, null, 2); }
|
||||
});
|
||||
}).catch(function (e) { err.textContent = String(e); out.textContent = ""; });
|
||||
}
|
||||
$("bk-send").onclick = function () { send(false); };
|
||||
$("bk-confirm").onclick = function () { send(true); };
|
||||
|
||||
fetch("/ai/bedrock/status").then(function (r) { return r.json(); }).then(function (s) {
|
||||
$("bk-config").textContent = s.configured
|
||||
? "Credential found (" + s.auth_mode + ", " + s.region + "). Calls are real; paid ones need confirm."
|
||||
: "No AWS_BEARER_TOKEN_BEDROCK or AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: calls answer 503 until one is set; dry runs and /sign still work.";
|
||||
});
|
||||
})();
|
||||
Reference in New Issue
Block a user