feat(clients): media: image and video generation across 12 providers behind one job seam

Stability, BFL, Runway, Luma, Replicate, fal, Ideogram, Recraft, Leonardo,
MiniMax, Kling, Higgsfield: call tables checked against vendor specs/SDKs,
ImageGenerator/VideoGenerator seam with capped polling, /ai/media demo.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:19:43 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 5a0c18d0e0
78 changed files with 8727 additions and 3 deletions
+579 -3
View File
@@ -1,13 +1,589 @@
#include "../AiDemos.hpp"
#include <ctime>
#include <mutex>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "security/crypto/Base64.h"
#include "security/crypto/Sha256.h"
#include "vendors/media/common/caps.h"
#include "vendors/media/common/downloader.h"
#include "vendors/media/common/job_runner.h"
#include "vendors/media/common/registry.h"
#include "vendors/media/replicate/replicate.h"
namespace molo::example::app {
// media. Stub: the package agent fills this in.
// Package media over the real src/vendors/media: image and video generation
// across twelve providers behind one seam (submit, poll, fetch), plus a
// generic call route over every provider's call table. GET routes need no
// credential. Every route that calls a provider is POST or DELETE: 422 on bad
// input, 503 media_not_configured without the provider's credential. Anything
// that costs money or changes a resource is a dry run showing the exact
// request unless the body has "confirm":true.
namespace {
namespace md = media;
using Json = clients::body::Json;
constexpr std::size_t kMaxBody = 24 * 1024 * 1024; // uploads travel as base64
constexpr std::size_t kInlineAssetCap = 4 * 1024 * 1024; // base64 echoed back for inline results
constexpr std::size_t kTextPreviewCap = 64 * 1024;
constexpr int kDemoMaxWaitSeconds = 60; // below the package cap: a request thread is held
constexpr int kDemoMaxAttempts = 30;
const char kPage[] = R"MEDIA({% extends "base.html" %}
{% block content %}
<h1>Media generation</h1>
<p class="hint">Images and video from Stability, Black Forest Labs, Runway, Luma, Replicate, fal, Ideogram,
Recraft, Leonardo, MiniMax, Kling and Higgsfield behind one submit, poll, fetch seam. Every call that costs
money or changes a resource is a dry run until the body says <code>"confirm":true</code>.</p>
<p id="md-config" class="hint">Checking credentials...</p>
<section class="md">
<label>Action
<select id="md-route"></select></label>
<label>Provider
<select id="md-provider"></select></label>
<label>Body (JSON object) <textarea id="md-body" rows="9"></textarea></label>
<div class="md-actions">
<button id="md-send" type="button" class="primary">Send (dry run if paid)</button>
<button id="md-confirm" type="button">Send with confirm</button>
</div>
<p id="md-error" class="error" role="alert"></p>
<pre id="md-out" class="md-out"></pre>
<details><summary>Not available</summary><ul id="md-unsupported"></ul></details>
</section>
<style>
.md label{display:block;margin:.5rem 0}
.md select,.md textarea{width:100%;box-sizing:border-box}
.md-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
.md-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
</style>
<script src="/embedded/ai_media.js"></script>
{% endblock %}
)MEDIA";
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
fw::Response json_out(const Json& j, int status = 200) {
fw::Response res = fw::JsonResponse(j.dump(), status);
res.headers.set("Cache-Control", "no-store");
return res;
}
Json parse_body(const fw::Request& req) {
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
if (req.body.empty()) return Json::object();
try {
Json j = Json::parse(req.body);
if (!j.is_object()) bad("body must be a JSON object");
return j;
} catch (const clients::body::JsonError&) {
bad("body is not valid JSON");
}
}
std::string text(const Json& b, const char* key, bool required = false) {
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return "";
}
if (!v->is_string()) bad(std::string(key) + " must be a string");
return v->as_string();
}
int number(const Json& b, const char* key, int fallback = 0) {
const Json* v = b.find(key);
if (!v || v->is_null()) return fallback;
if (!v->is_number()) bad(std::string(key) + " must be a number");
return static_cast<int>(v->as_int());
}
long seed_of(const Json& b) {
const Json* v = b.find("seed");
if (!v || v->is_null()) return -1;
if (!v->is_number()) bad("seed must be a number");
return v->as_int();
}
bool flag(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return false;
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
return v->as_bool();
}
Json object_at(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return Json::object();
if (!v->is_object()) bad(std::string(key) + " must be a JSON object");
return *v;
}
std::map<std::string, std::string> strings_at(const Json& b, const char* key) {
std::map<std::string, std::string> out;
const Json* v = b.find(key);
if (!v || v->is_null()) return out;
if (!v->is_object()) bad(std::string(key) + " must be a JSON object of strings");
if (v->as_object().size() > md::MediaCaps::kMaxQueryArgs) bad(std::string(key) + " has too many entries");
for (const auto& [k, x] : v->as_object()) {
if (x.is_string()) out[k] = x.as_string();
else if (x.is_number() || x.is_bool()) out[k] = x.dump();
else bad(std::string(key) + "." + k + " must be a string");
}
return out;
}
Json dry_run(const Json& plan, const char* what) {
Json o = Json::object();
o.set("dry_run", Json(true));
o.set("what", Json(what));
o.set("request", plan);
o.set("hint", Json("send confirm:true to run it; this request costs money or changes a resource"));
return o;
}
Json asset_json(const md::MediaAsset& a) {
Json o = a.to_json();
if (!a.data_base64.empty()) {
if (a.data_base64.size() <= kInlineAssetCap) o.set("data_base64", Json(a.data_base64));
else o.set("note", Json("inline data over the echo cap; fetch it with the provider client"));
}
return o;
}
Json status_json(const md::JobStatus& s) {
Json o = s.to_json();
Json arr = Json::array();
for (const md::MediaAsset& a : s.assets) arr.push(asset_json(a));
o.set("assets", arr);
return o;
}
Json ticket_reply(const md::JobTicket& t) {
Json o = Json::object();
Json tj = t.to_json();
Json arr = Json::array();
for (const md::MediaAsset& a : t.assets) arr.push(asset_json(a));
tj.set("assets", arr);
o.set("ticket", tj);
o.set("next", Json(t.immediate ? "the result is in ticket.assets" : "POST /ai/media/poll or /ai/media/wait with the ticket"));
return o;
}
class MediaRoutes {
public:
// ---- read-only GET data ---------------------------------------------------
Json status() {
std::lock_guard<std::mutex> lock(mu_);
Json o = Json::object();
Json arr = Json::array();
for (const md::ProviderInfo& p : md::MediaRegistry::providers()) arr.push(provider_json(p));
o.set("providers", arr);
Json un = Json::array();
for (const md::UnsupportedProduct& u : md::MediaRegistry::unsupported()) {
Json x = Json::object();
x.set("name", Json(u.name));
x.set("reason", Json(u.reason));
un.push(x);
}
o.set("unsupported", un);
Json caps = Json::object();
caps.set("max_poll_attempts", Json(md::MediaCaps::kMaxPollAttempts));
caps.set("min_poll_interval_ms", Json(md::MediaCaps::kMinPollIntervalMs));
caps.set("max_wait_seconds", Json(md::MediaCaps::kMaxWaitSeconds));
caps.set("demo_max_wait_seconds", Json(kDemoMaxWaitSeconds));
caps.set("max_images_per_request", Json(md::MediaCaps::kMaxImagesPerRequest));
caps.set("max_video_seconds", Json(md::MediaCaps::kMaxVideoSeconds));
caps.set("max_file_bytes", Json(static_cast<double>(md::MediaCaps::kMaxFileBytes)));
caps.set("max_download_bytes", Json(static_cast<double>(md::MediaCaps::kMaxDownloadBytes)));
o.set("caps", caps);
return o;
}
Json calls(const std::string& provider) {
if (provider.empty()) {
Json o = Json::object();
Json arr = Json::array();
for (const md::ProviderInfo& p : md::MediaRegistry::providers()) {
Json x = Json::object();
x.set("provider", Json(p.id));
x.set("calls", Json(static_cast<double>(md::MediaRegistry::table(p.id).size())));
arr.push(x);
}
o.set("providers", arr);
return o;
}
if (!md::MediaRegistry::find(provider)) bad("unknown provider '" + provider + "'");
const md::CallTable& t = md::MediaRegistry::table(provider);
Json arr = Json::array();
for (std::size_t i = 0; i < t.size(); ++i) {
const md::CallRow& r = t.at(i);
Json x = Json::object();
x.set("key", Json(r.key));
x.set("http", Json(r.http_method));
x.set("path", Json(r.path));
x.set("effect", Json(r.effect));
x.set("body", Json(r.body_kind));
arr.push(x);
}
Json o = Json::object();
o.set("provider", Json(provider));
o.set("calls", arr);
return o;
}
// ---- generation seam ------------------------------------------------------
fw::Response image(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, true);
md::ImageRequest r;
r.prompt = text(b, "prompt", true);
r.negative_prompt = text(b, "negative_prompt");
r.model = text(b, "model");
r.width = number(b, "width");
r.height = number(b, "height");
r.aspect_ratio = text(b, "aspect_ratio");
r.count = number(b, "count", 1);
r.seed = seed_of(b);
r.source_url = text(b, "source_url");
r.extra = object_at(b, "extra");
std::shared_ptr<md::ImageGenerator> gen;
Json plan;
try {
std::lock_guard<std::mutex> lock(mu_);
gen = reg_.image(provider);
r.validate();
plan = gen->plan(r);
} catch (const llm::ApiError& e) {
map(e);
}
if (!flag(b, "confirm")) return json_out(dry_run(plan, "image generation"));
try {
return json_out(ticket_reply(gen->submit(r, true)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response video(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, false);
md::VideoRequest r;
r.prompt = text(b, "prompt");
r.negative_prompt = text(b, "negative_prompt");
r.model = text(b, "model");
r.duration_seconds = number(b, "duration_seconds");
r.aspect_ratio = text(b, "aspect_ratio");
r.resolution = text(b, "resolution");
r.seed = seed_of(b);
r.image_url = text(b, "image_url");
r.end_image_url = text(b, "end_image_url");
r.extra = object_at(b, "extra");
std::shared_ptr<md::VideoGenerator> gen;
Json plan;
try {
std::lock_guard<std::mutex> lock(mu_);
gen = reg_.video(provider);
r.validate();
plan = gen->plan(r);
} catch (const llm::ApiError& e) {
map(e);
}
if (!flag(b, "confirm")) return json_out(dry_run(plan, "video generation"));
try {
return json_out(ticket_reply(gen->submit(r, true)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response poll(const fw::Request& req) {
Json b = parse_body(req);
md::JobTicket t = ticket_of(b);
try {
require_configured(t.provider);
return json_out(status_json(generator_poll(t)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response wait(const fw::Request& req) {
Json b = parse_body(req);
md::JobTicket t = ticket_of(b);
md::WaitPolicy policy;
policy.max_attempts = number(b, "max_attempts", 15);
policy.max_wait_seconds = number(b, "max_wait_seconds", 45);
policy.first_interval_ms = number(b, "first_interval_ms", 0);
if (policy.max_attempts > kDemoMaxAttempts) bad("max_attempts is limited to " + std::to_string(kDemoMaxAttempts) + " in the demo");
if (policy.max_wait_seconds > kDemoMaxWaitSeconds)
bad("max_wait_seconds is limited to " + std::to_string(kDemoMaxWaitSeconds) + " in the demo");
try {
md::JobRunner runner(policy); // validates against the package caps
require_configured(t.provider);
std::shared_ptr<md::MediaGenerator> gen = generator_for(t);
return json_out(status_json(runner.wait(*gen, t)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response cancel(const fw::Request& req) {
Json b = parse_body(req);
md::JobTicket t = ticket_of(b);
Json plan = Json::object();
plan.set("provider", Json(t.provider));
plan.set("job", Json(t.id));
plan.set("action", Json("cancel (some providers delete the generation)"));
if (!flag(b, "confirm")) return json_out(dry_run(plan, "cancel a job"));
try {
require_configured(t.provider);
bool done = generator_for(t)->cancel(t, true);
Json o = Json::object();
o.set("cancelled", Json(done));
if (!done) o.set("note", Json(t.provider + " has no cancel endpoint; nothing was sent"));
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
// ---- generic table call ---------------------------------------------------
fw::Response call(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, false, false);
std::string key = text(b, "key", true);
md::CallArgs args;
args.path = strings_at(b, "path");
args.query = strings_at(b, "query");
args.headers = strings_at(b, "headers");
args.body = object_at(b, "body");
if (const Json* files = b.find("files"); files && !files->is_null()) {
if (!files->is_array() || files->as_array().size() > md::MediaCaps::kMaxFiles) bad("files must be an array of at most 16 objects");
for (const Json& f : files->as_array()) {
if (!f.is_object()) bad("each file must be an object");
md::FilePart part;
part.field = text(f, "field", true);
part.filename = text(f, "filename");
part.content_type = text(f, "content_type");
if (!security::crypto::Base64::decode(text(f, "data_base64", true), part.data)) bad("data_base64 is not valid base64");
args.files.push_back(part);
}
}
md::PreparedCall prepared;
std::shared_ptr<md::MediaService> service;
try {
std::lock_guard<std::mutex> lock(mu_);
service = reg_.service(provider, key);
prepared = service->prepare(key, args);
} catch (const llm::ApiError& e) {
map(e);
}
if (prepared.effect != md::Effect::kRead && !flag(b, "confirm")) return json_out(dry_run(prepared.summary(), "provider call"));
try {
llm::RawResponse raw = service->execute(prepared, true);
Json o = Json::object();
o.set("request", prepared.summary());
o.set("status", Json(raw.status));
o.set("content_type", Json(raw.content_type));
o.set("bytes", Json(static_cast<double>(raw.body.size())));
if (raw.content_type.find("json") != std::string::npos) {
try {
o.set("json", Json::parse(raw.body));
} catch (const std::exception&) {
o.set("text", Json(raw.body.substr(0, kTextPreviewCap)));
}
} else if (raw.content_type.rfind("text/", 0) == 0) {
o.set("text", Json(raw.body.substr(0, kTextPreviewCap)));
} else {
o.set("note", Json("binary reply; use the generation seam or the provider client to fetch it"));
}
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response account(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, false, false);
if (md::MediaRegistry::find(provider)->account_key.empty()) bad(provider + " has no free account read");
try {
std::lock_guard<std::mutex> lock(mu_);
require_configured_locked(provider);
Json o = Json::object();
o.set("provider", Json(provider));
o.set("read", Json(md::MediaRegistry::find(provider)->account_key));
o.set("result", reg_.account(provider));
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
// Downloads a result URL (public or pre-signed) and reports its size and
// digest; no provider credential is involved.
fw::Response fetch(const fw::Request& req) {
Json b = parse_body(req);
std::string url = text(b, "url", true);
try {
md::Downloader::validate_url(url);
md::Downloader dl;
std::string type;
std::string bytes = dl.fetch_url(url, &type);
Json o = Json::object();
o.set("bytes", Json(static_cast<double>(bytes.size())));
o.set("content_type", Json(type));
o.set("sha256", Json(security::crypto::Sha256::hex(security::crypto::Sha256::hash(bytes))));
if (bytes.size() <= 128 * 1024) o.set("data_base64", Json(security::crypto::Base64::encode(bytes)));
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response webhook(const fw::Request& req) {
Json b = parse_body(req);
std::string secret = text(b, "secret", true);
std::string id = text(b, "webhook_id", true);
std::string ts = text(b, "timestamp", true);
std::string sig = text(b, "signature", true);
std::string body = text(b, "payload", true);
if (body.size() > md::MediaCaps::kMaxWebhookBodyBytes) bad("payload is over the webhook body cap");
bool ok = md::ReplicateWebhook::verify(secret, id, ts, sig, body, static_cast<long>(std::time(nullptr)));
Json o = Json::object();
o.set("valid", Json(ok));
o.set("scheme", Json("Standard Webhooks (HMAC-SHA256 over id.timestamp.payload)"));
o.set("tolerance_seconds", Json(static_cast<double>(md::MediaCaps::kWebhookToleranceSeconds)));
return json_out(o);
}
private:
Json provider_json(const md::ProviderInfo& p) {
Json x = Json::object();
x.set("id", Json(p.id));
x.set("name", Json(p.name));
x.set("configured", Json(md::MediaRegistry::configured(p.id)));
Json env = Json::array();
for (const std::string& e : p.env) env.push(Json(e));
x.set("env", env);
x.set("image", Json(p.image));
x.set("video", Json(p.video));
x.set("evidence", Json(md::MediaRegistry::evidence_name(p.evidence)));
x.set("console", Json(p.console_url));
x.set("account_read", Json(p.account_key));
x.set("note", Json(p.note));
x.set("calls", Json(static_cast<double>(md::MediaRegistry::table(p.id).size())));
Json models = Json::array();
if (p.image)
for (const std::string& m : reg_.image(p.id)->models()) models.push(Json(m));
x.set("image_models", models);
Json vmodels = Json::array();
if (p.video)
for (const std::string& m : reg_.video(p.id)->models()) vmodels.push(Json(m));
x.set("video_models", vmodels);
return x;
}
std::string provider_of(const Json& b, bool image, bool generating = true) {
std::string id = text(b, "provider", true);
const md::ProviderInfo* p = md::MediaRegistry::find(id);
if (!p) bad("unknown provider '" + id + "'");
if (generating && image && !p->image) bad(id + " has no image generation");
if (generating && !image && !p->video) bad(id + " has no video generation");
return id;
}
md::JobTicket ticket_of(const Json& b) {
const Json* t = b.find("ticket");
if (!t || !t->is_object()) bad("ticket is required (the object returned by image or video)");
try {
md::JobTicket ticket = md::JobTicket::from_json(*t);
if (!md::MediaRegistry::find(ticket.provider)) bad("unknown provider '" + ticket.provider + "'");
return ticket;
} catch (const llm::ApiError& e) {
map(e);
}
}
std::shared_ptr<md::MediaGenerator> generator_for(const md::JobTicket& t) {
std::lock_guard<std::mutex> lock(mu_);
if (t.kind == md::MediaKind::kVideo) return reg_.video(t.provider);
return reg_.image(t.provider);
}
md::JobStatus generator_poll(const md::JobTicket& t) {
if (t.immediate) {
md::JobStatus s;
s.state = md::JobState::kSucceeded;
s.raw_state = "immediate";
s.assets = t.assets;
return s;
}
return generator_for(t)->poll(t);
}
void require_configured(const std::string& provider) {
std::lock_guard<std::mutex> lock(mu_);
require_configured_locked(provider);
}
void require_configured_locked(const std::string& provider) {
if (!md::MediaRegistry::configured(provider)) {
const md::ProviderInfo* p = md::MediaRegistry::find(provider);
throw llm::NotConfiguredError(provider, p->env[0]);
}
}
[[noreturn]] static void map(const llm::ApiError& e) {
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
throw fw::HttpError(503, "media_not_configured", e.message());
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
throw fw::HttpError(422, "validation_error", e.message());
if (e.type() == "WAIT_EXPIRED") throw fw::HttpError(504, "media_wait_expired", e.message());
throw fw::HttpError(502, "media_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
}
std::mutex mu_;
md::MediaRegistry reg_;
};
} // namespace
void register_ai_media_demo(fw::Router& router, Services& services) {
(void)router;
(void)services;
auto routes = std::make_shared<MediaRoutes>();
fw::Environment* env = services.templates.get();
services.templates->add_template("ai_media.html", kPage);
services.add_feature("ai-media", "/ai/media");
router.get("/ai/media", "ai-media-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Media generation");
return fw::Response(200, env->render("ai_media.html", ctx), "text/html; charset=utf-8");
});
router.get("/ai/media/status", "ai-media-status", [routes](fw::Request req) {
(void)req;
return json_out(routes->status());
});
router.get("/ai/media/calls", "ai-media-calls", [routes](fw::Request req) {
return json_out(routes->calls(req.query_or("provider", "")));
});
using Handler = fw::Response (MediaRoutes::*)(const fw::Request&);
auto bind = [routes](Handler h) { return [routes, h](fw::Request req) { return ((*routes).*h)(req); }; };
router.post("/ai/media/image", "ai-media-image", bind(&MediaRoutes::image));
router.post("/ai/media/video", "ai-media-video", bind(&MediaRoutes::video));
router.post("/ai/media/poll", "ai-media-poll", bind(&MediaRoutes::poll));
router.post("/ai/media/wait", "ai-media-wait", bind(&MediaRoutes::wait));
router.post("/ai/media/call", "ai-media-call", bind(&MediaRoutes::call));
router.post("/ai/media/account", "ai-media-account", bind(&MediaRoutes::account));
router.post("/ai/media/fetch", "ai-media-fetch", bind(&MediaRoutes::fetch));
router.post("/ai/media/webhook", "ai-media-webhook", bind(&MediaRoutes::webhook));
router.del("/ai/media/job", "ai-media-cancel", bind(&MediaRoutes::cancel));
}
} // namespace molo::example::app