feat(clients): media: image and video generation across 12 providers behind one job seam

Stability, BFL, Runway, Luma, Replicate, fal, Ideogram, Recraft, Leonardo,
MiniMax, Kling, Higgsfield: call tables checked against vendor specs/SDKs,
ImageGenerator/VideoGenerator seam with capped polling, /ai/media demo.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:19:43 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 5a0c18d0e0
78 changed files with 8727 additions and 3 deletions
+579 -3
View File
@@ -1,13 +1,589 @@
#include "../AiDemos.hpp"
#include <ctime>
#include <mutex>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "security/crypto/Base64.h"
#include "security/crypto/Sha256.h"
#include "vendors/media/common/caps.h"
#include "vendors/media/common/downloader.h"
#include "vendors/media/common/job_runner.h"
#include "vendors/media/common/registry.h"
#include "vendors/media/replicate/replicate.h"
namespace molo::example::app {
// media. Stub: the package agent fills this in.
// Package media over the real src/vendors/media: image and video generation
// across twelve providers behind one seam (submit, poll, fetch), plus a
// generic call route over every provider's call table. GET routes need no
// credential. Every route that calls a provider is POST or DELETE: 422 on bad
// input, 503 media_not_configured without the provider's credential. Anything
// that costs money or changes a resource is a dry run showing the exact
// request unless the body has "confirm":true.
namespace {
namespace md = media;
using Json = clients::body::Json;
constexpr std::size_t kMaxBody = 24 * 1024 * 1024; // uploads travel as base64
constexpr std::size_t kInlineAssetCap = 4 * 1024 * 1024; // base64 echoed back for inline results
constexpr std::size_t kTextPreviewCap = 64 * 1024;
constexpr int kDemoMaxWaitSeconds = 60; // below the package cap: a request thread is held
constexpr int kDemoMaxAttempts = 30;
const char kPage[] = R"MEDIA({% extends "base.html" %}
{% block content %}
<h1>Media generation</h1>
<p class="hint">Images and video from Stability, Black Forest Labs, Runway, Luma, Replicate, fal, Ideogram,
Recraft, Leonardo, MiniMax, Kling and Higgsfield behind one submit, poll, fetch seam. Every call that costs
money or changes a resource is a dry run until the body says <code>"confirm":true</code>.</p>
<p id="md-config" class="hint">Checking credentials...</p>
<section class="md">
<label>Action
<select id="md-route"></select></label>
<label>Provider
<select id="md-provider"></select></label>
<label>Body (JSON object) <textarea id="md-body" rows="9"></textarea></label>
<div class="md-actions">
<button id="md-send" type="button" class="primary">Send (dry run if paid)</button>
<button id="md-confirm" type="button">Send with confirm</button>
</div>
<p id="md-error" class="error" role="alert"></p>
<pre id="md-out" class="md-out"></pre>
<details><summary>Not available</summary><ul id="md-unsupported"></ul></details>
</section>
<style>
.md label{display:block;margin:.5rem 0}
.md select,.md textarea{width:100%;box-sizing:border-box}
.md-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
.md-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
</style>
<script src="/embedded/ai_media.js"></script>
{% endblock %}
)MEDIA";
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
fw::Response json_out(const Json& j, int status = 200) {
fw::Response res = fw::JsonResponse(j.dump(), status);
res.headers.set("Cache-Control", "no-store");
return res;
}
Json parse_body(const fw::Request& req) {
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
if (req.body.empty()) return Json::object();
try {
Json j = Json::parse(req.body);
if (!j.is_object()) bad("body must be a JSON object");
return j;
} catch (const clients::body::JsonError&) {
bad("body is not valid JSON");
}
}
std::string text(const Json& b, const char* key, bool required = false) {
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return "";
}
if (!v->is_string()) bad(std::string(key) + " must be a string");
return v->as_string();
}
int number(const Json& b, const char* key, int fallback = 0) {
const Json* v = b.find(key);
if (!v || v->is_null()) return fallback;
if (!v->is_number()) bad(std::string(key) + " must be a number");
return static_cast<int>(v->as_int());
}
long seed_of(const Json& b) {
const Json* v = b.find("seed");
if (!v || v->is_null()) return -1;
if (!v->is_number()) bad("seed must be a number");
return v->as_int();
}
bool flag(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return false;
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
return v->as_bool();
}
Json object_at(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return Json::object();
if (!v->is_object()) bad(std::string(key) + " must be a JSON object");
return *v;
}
std::map<std::string, std::string> strings_at(const Json& b, const char* key) {
std::map<std::string, std::string> out;
const Json* v = b.find(key);
if (!v || v->is_null()) return out;
if (!v->is_object()) bad(std::string(key) + " must be a JSON object of strings");
if (v->as_object().size() > md::MediaCaps::kMaxQueryArgs) bad(std::string(key) + " has too many entries");
for (const auto& [k, x] : v->as_object()) {
if (x.is_string()) out[k] = x.as_string();
else if (x.is_number() || x.is_bool()) out[k] = x.dump();
else bad(std::string(key) + "." + k + " must be a string");
}
return out;
}
Json dry_run(const Json& plan, const char* what) {
Json o = Json::object();
o.set("dry_run", Json(true));
o.set("what", Json(what));
o.set("request", plan);
o.set("hint", Json("send confirm:true to run it; this request costs money or changes a resource"));
return o;
}
Json asset_json(const md::MediaAsset& a) {
Json o = a.to_json();
if (!a.data_base64.empty()) {
if (a.data_base64.size() <= kInlineAssetCap) o.set("data_base64", Json(a.data_base64));
else o.set("note", Json("inline data over the echo cap; fetch it with the provider client"));
}
return o;
}
Json status_json(const md::JobStatus& s) {
Json o = s.to_json();
Json arr = Json::array();
for (const md::MediaAsset& a : s.assets) arr.push(asset_json(a));
o.set("assets", arr);
return o;
}
Json ticket_reply(const md::JobTicket& t) {
Json o = Json::object();
Json tj = t.to_json();
Json arr = Json::array();
for (const md::MediaAsset& a : t.assets) arr.push(asset_json(a));
tj.set("assets", arr);
o.set("ticket", tj);
o.set("next", Json(t.immediate ? "the result is in ticket.assets" : "POST /ai/media/poll or /ai/media/wait with the ticket"));
return o;
}
class MediaRoutes {
public:
// ---- read-only GET data ---------------------------------------------------
Json status() {
std::lock_guard<std::mutex> lock(mu_);
Json o = Json::object();
Json arr = Json::array();
for (const md::ProviderInfo& p : md::MediaRegistry::providers()) arr.push(provider_json(p));
o.set("providers", arr);
Json un = Json::array();
for (const md::UnsupportedProduct& u : md::MediaRegistry::unsupported()) {
Json x = Json::object();
x.set("name", Json(u.name));
x.set("reason", Json(u.reason));
un.push(x);
}
o.set("unsupported", un);
Json caps = Json::object();
caps.set("max_poll_attempts", Json(md::MediaCaps::kMaxPollAttempts));
caps.set("min_poll_interval_ms", Json(md::MediaCaps::kMinPollIntervalMs));
caps.set("max_wait_seconds", Json(md::MediaCaps::kMaxWaitSeconds));
caps.set("demo_max_wait_seconds", Json(kDemoMaxWaitSeconds));
caps.set("max_images_per_request", Json(md::MediaCaps::kMaxImagesPerRequest));
caps.set("max_video_seconds", Json(md::MediaCaps::kMaxVideoSeconds));
caps.set("max_file_bytes", Json(static_cast<double>(md::MediaCaps::kMaxFileBytes)));
caps.set("max_download_bytes", Json(static_cast<double>(md::MediaCaps::kMaxDownloadBytes)));
o.set("caps", caps);
return o;
}
Json calls(const std::string& provider) {
if (provider.empty()) {
Json o = Json::object();
Json arr = Json::array();
for (const md::ProviderInfo& p : md::MediaRegistry::providers()) {
Json x = Json::object();
x.set("provider", Json(p.id));
x.set("calls", Json(static_cast<double>(md::MediaRegistry::table(p.id).size())));
arr.push(x);
}
o.set("providers", arr);
return o;
}
if (!md::MediaRegistry::find(provider)) bad("unknown provider '" + provider + "'");
const md::CallTable& t = md::MediaRegistry::table(provider);
Json arr = Json::array();
for (std::size_t i = 0; i < t.size(); ++i) {
const md::CallRow& r = t.at(i);
Json x = Json::object();
x.set("key", Json(r.key));
x.set("http", Json(r.http_method));
x.set("path", Json(r.path));
x.set("effect", Json(r.effect));
x.set("body", Json(r.body_kind));
arr.push(x);
}
Json o = Json::object();
o.set("provider", Json(provider));
o.set("calls", arr);
return o;
}
// ---- generation seam ------------------------------------------------------
fw::Response image(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, true);
md::ImageRequest r;
r.prompt = text(b, "prompt", true);
r.negative_prompt = text(b, "negative_prompt");
r.model = text(b, "model");
r.width = number(b, "width");
r.height = number(b, "height");
r.aspect_ratio = text(b, "aspect_ratio");
r.count = number(b, "count", 1);
r.seed = seed_of(b);
r.source_url = text(b, "source_url");
r.extra = object_at(b, "extra");
std::shared_ptr<md::ImageGenerator> gen;
Json plan;
try {
std::lock_guard<std::mutex> lock(mu_);
gen = reg_.image(provider);
r.validate();
plan = gen->plan(r);
} catch (const llm::ApiError& e) {
map(e);
}
if (!flag(b, "confirm")) return json_out(dry_run(plan, "image generation"));
try {
return json_out(ticket_reply(gen->submit(r, true)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response video(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, false);
md::VideoRequest r;
r.prompt = text(b, "prompt");
r.negative_prompt = text(b, "negative_prompt");
r.model = text(b, "model");
r.duration_seconds = number(b, "duration_seconds");
r.aspect_ratio = text(b, "aspect_ratio");
r.resolution = text(b, "resolution");
r.seed = seed_of(b);
r.image_url = text(b, "image_url");
r.end_image_url = text(b, "end_image_url");
r.extra = object_at(b, "extra");
std::shared_ptr<md::VideoGenerator> gen;
Json plan;
try {
std::lock_guard<std::mutex> lock(mu_);
gen = reg_.video(provider);
r.validate();
plan = gen->plan(r);
} catch (const llm::ApiError& e) {
map(e);
}
if (!flag(b, "confirm")) return json_out(dry_run(plan, "video generation"));
try {
return json_out(ticket_reply(gen->submit(r, true)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response poll(const fw::Request& req) {
Json b = parse_body(req);
md::JobTicket t = ticket_of(b);
try {
require_configured(t.provider);
return json_out(status_json(generator_poll(t)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response wait(const fw::Request& req) {
Json b = parse_body(req);
md::JobTicket t = ticket_of(b);
md::WaitPolicy policy;
policy.max_attempts = number(b, "max_attempts", 15);
policy.max_wait_seconds = number(b, "max_wait_seconds", 45);
policy.first_interval_ms = number(b, "first_interval_ms", 0);
if (policy.max_attempts > kDemoMaxAttempts) bad("max_attempts is limited to " + std::to_string(kDemoMaxAttempts) + " in the demo");
if (policy.max_wait_seconds > kDemoMaxWaitSeconds)
bad("max_wait_seconds is limited to " + std::to_string(kDemoMaxWaitSeconds) + " in the demo");
try {
md::JobRunner runner(policy); // validates against the package caps
require_configured(t.provider);
std::shared_ptr<md::MediaGenerator> gen = generator_for(t);
return json_out(status_json(runner.wait(*gen, t)));
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response cancel(const fw::Request& req) {
Json b = parse_body(req);
md::JobTicket t = ticket_of(b);
Json plan = Json::object();
plan.set("provider", Json(t.provider));
plan.set("job", Json(t.id));
plan.set("action", Json("cancel (some providers delete the generation)"));
if (!flag(b, "confirm")) return json_out(dry_run(plan, "cancel a job"));
try {
require_configured(t.provider);
bool done = generator_for(t)->cancel(t, true);
Json o = Json::object();
o.set("cancelled", Json(done));
if (!done) o.set("note", Json(t.provider + " has no cancel endpoint; nothing was sent"));
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
// ---- generic table call ---------------------------------------------------
fw::Response call(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, false, false);
std::string key = text(b, "key", true);
md::CallArgs args;
args.path = strings_at(b, "path");
args.query = strings_at(b, "query");
args.headers = strings_at(b, "headers");
args.body = object_at(b, "body");
if (const Json* files = b.find("files"); files && !files->is_null()) {
if (!files->is_array() || files->as_array().size() > md::MediaCaps::kMaxFiles) bad("files must be an array of at most 16 objects");
for (const Json& f : files->as_array()) {
if (!f.is_object()) bad("each file must be an object");
md::FilePart part;
part.field = text(f, "field", true);
part.filename = text(f, "filename");
part.content_type = text(f, "content_type");
if (!security::crypto::Base64::decode(text(f, "data_base64", true), part.data)) bad("data_base64 is not valid base64");
args.files.push_back(part);
}
}
md::PreparedCall prepared;
std::shared_ptr<md::MediaService> service;
try {
std::lock_guard<std::mutex> lock(mu_);
service = reg_.service(provider, key);
prepared = service->prepare(key, args);
} catch (const llm::ApiError& e) {
map(e);
}
if (prepared.effect != md::Effect::kRead && !flag(b, "confirm")) return json_out(dry_run(prepared.summary(), "provider call"));
try {
llm::RawResponse raw = service->execute(prepared, true);
Json o = Json::object();
o.set("request", prepared.summary());
o.set("status", Json(raw.status));
o.set("content_type", Json(raw.content_type));
o.set("bytes", Json(static_cast<double>(raw.body.size())));
if (raw.content_type.find("json") != std::string::npos) {
try {
o.set("json", Json::parse(raw.body));
} catch (const std::exception&) {
o.set("text", Json(raw.body.substr(0, kTextPreviewCap)));
}
} else if (raw.content_type.rfind("text/", 0) == 0) {
o.set("text", Json(raw.body.substr(0, kTextPreviewCap)));
} else {
o.set("note", Json("binary reply; use the generation seam or the provider client to fetch it"));
}
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response account(const fw::Request& req) {
Json b = parse_body(req);
std::string provider = provider_of(b, false, false);
if (md::MediaRegistry::find(provider)->account_key.empty()) bad(provider + " has no free account read");
try {
std::lock_guard<std::mutex> lock(mu_);
require_configured_locked(provider);
Json o = Json::object();
o.set("provider", Json(provider));
o.set("read", Json(md::MediaRegistry::find(provider)->account_key));
o.set("result", reg_.account(provider));
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
// Downloads a result URL (public or pre-signed) and reports its size and
// digest; no provider credential is involved.
fw::Response fetch(const fw::Request& req) {
Json b = parse_body(req);
std::string url = text(b, "url", true);
try {
md::Downloader::validate_url(url);
md::Downloader dl;
std::string type;
std::string bytes = dl.fetch_url(url, &type);
Json o = Json::object();
o.set("bytes", Json(static_cast<double>(bytes.size())));
o.set("content_type", Json(type));
o.set("sha256", Json(security::crypto::Sha256::hex(security::crypto::Sha256::hash(bytes))));
if (bytes.size() <= 128 * 1024) o.set("data_base64", Json(security::crypto::Base64::encode(bytes)));
return json_out(o);
} catch (const llm::ApiError& e) {
map(e);
}
}
fw::Response webhook(const fw::Request& req) {
Json b = parse_body(req);
std::string secret = text(b, "secret", true);
std::string id = text(b, "webhook_id", true);
std::string ts = text(b, "timestamp", true);
std::string sig = text(b, "signature", true);
std::string body = text(b, "payload", true);
if (body.size() > md::MediaCaps::kMaxWebhookBodyBytes) bad("payload is over the webhook body cap");
bool ok = md::ReplicateWebhook::verify(secret, id, ts, sig, body, static_cast<long>(std::time(nullptr)));
Json o = Json::object();
o.set("valid", Json(ok));
o.set("scheme", Json("Standard Webhooks (HMAC-SHA256 over id.timestamp.payload)"));
o.set("tolerance_seconds", Json(static_cast<double>(md::MediaCaps::kWebhookToleranceSeconds)));
return json_out(o);
}
private:
Json provider_json(const md::ProviderInfo& p) {
Json x = Json::object();
x.set("id", Json(p.id));
x.set("name", Json(p.name));
x.set("configured", Json(md::MediaRegistry::configured(p.id)));
Json env = Json::array();
for (const std::string& e : p.env) env.push(Json(e));
x.set("env", env);
x.set("image", Json(p.image));
x.set("video", Json(p.video));
x.set("evidence", Json(md::MediaRegistry::evidence_name(p.evidence)));
x.set("console", Json(p.console_url));
x.set("account_read", Json(p.account_key));
x.set("note", Json(p.note));
x.set("calls", Json(static_cast<double>(md::MediaRegistry::table(p.id).size())));
Json models = Json::array();
if (p.image)
for (const std::string& m : reg_.image(p.id)->models()) models.push(Json(m));
x.set("image_models", models);
Json vmodels = Json::array();
if (p.video)
for (const std::string& m : reg_.video(p.id)->models()) vmodels.push(Json(m));
x.set("video_models", vmodels);
return x;
}
std::string provider_of(const Json& b, bool image, bool generating = true) {
std::string id = text(b, "provider", true);
const md::ProviderInfo* p = md::MediaRegistry::find(id);
if (!p) bad("unknown provider '" + id + "'");
if (generating && image && !p->image) bad(id + " has no image generation");
if (generating && !image && !p->video) bad(id + " has no video generation");
return id;
}
md::JobTicket ticket_of(const Json& b) {
const Json* t = b.find("ticket");
if (!t || !t->is_object()) bad("ticket is required (the object returned by image or video)");
try {
md::JobTicket ticket = md::JobTicket::from_json(*t);
if (!md::MediaRegistry::find(ticket.provider)) bad("unknown provider '" + ticket.provider + "'");
return ticket;
} catch (const llm::ApiError& e) {
map(e);
}
}
std::shared_ptr<md::MediaGenerator> generator_for(const md::JobTicket& t) {
std::lock_guard<std::mutex> lock(mu_);
if (t.kind == md::MediaKind::kVideo) return reg_.video(t.provider);
return reg_.image(t.provider);
}
md::JobStatus generator_poll(const md::JobTicket& t) {
if (t.immediate) {
md::JobStatus s;
s.state = md::JobState::kSucceeded;
s.raw_state = "immediate";
s.assets = t.assets;
return s;
}
return generator_for(t)->poll(t);
}
void require_configured(const std::string& provider) {
std::lock_guard<std::mutex> lock(mu_);
require_configured_locked(provider);
}
void require_configured_locked(const std::string& provider) {
if (!md::MediaRegistry::configured(provider)) {
const md::ProviderInfo* p = md::MediaRegistry::find(provider);
throw llm::NotConfiguredError(provider, p->env[0]);
}
}
[[noreturn]] static void map(const llm::ApiError& e) {
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
throw fw::HttpError(503, "media_not_configured", e.message());
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
throw fw::HttpError(422, "validation_error", e.message());
if (e.type() == "WAIT_EXPIRED") throw fw::HttpError(504, "media_wait_expired", e.message());
throw fw::HttpError(502, "media_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
}
std::mutex mu_;
md::MediaRegistry reg_;
};
} // namespace
void register_ai_media_demo(fw::Router& router, Services& services) {
(void)router;
(void)services;
auto routes = std::make_shared<MediaRoutes>();
fw::Environment* env = services.templates.get();
services.templates->add_template("ai_media.html", kPage);
services.add_feature("ai-media", "/ai/media");
router.get("/ai/media", "ai-media-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Media generation");
return fw::Response(200, env->render("ai_media.html", ctx), "text/html; charset=utf-8");
});
router.get("/ai/media/status", "ai-media-status", [routes](fw::Request req) {
(void)req;
return json_out(routes->status());
});
router.get("/ai/media/calls", "ai-media-calls", [routes](fw::Request req) {
return json_out(routes->calls(req.query_or("provider", "")));
});
using Handler = fw::Response (MediaRoutes::*)(const fw::Request&);
auto bind = [routes](Handler h) { return [routes, h](fw::Request req) { return ((*routes).*h)(req); }; };
router.post("/ai/media/image", "ai-media-image", bind(&MediaRoutes::image));
router.post("/ai/media/video", "ai-media-video", bind(&MediaRoutes::video));
router.post("/ai/media/poll", "ai-media-poll", bind(&MediaRoutes::poll));
router.post("/ai/media/wait", "ai-media-wait", bind(&MediaRoutes::wait));
router.post("/ai/media/call", "ai-media-call", bind(&MediaRoutes::call));
router.post("/ai/media/account", "ai-media-account", bind(&MediaRoutes::account));
router.post("/ai/media/fetch", "ai-media-fetch", bind(&MediaRoutes::fetch));
router.post("/ai/media/webhook", "ai-media-webhook", bind(&MediaRoutes::webhook));
router.del("/ai/media/job", "ai-media-cancel", bind(&MediaRoutes::cancel));
}
} // namespace molo::example::app
+102
View File
@@ -0,0 +1,102 @@
// Media page: thin client over /ai/media/* (all limits are enforced server side).
(function () {
"use strict";
var $ = function (id) { return document.getElementById(id); };
var out = $("md-out"), err = $("md-error");
var providers = [];
var last = null; // last ticket returned by image / video
var routes = {
"image": function (p) { return { provider: p, prompt: "A lighthouse at dusk, oil painting", model: "", aspect_ratio: "", count: 1 }; },
"video": function (p) { return { provider: p, prompt: "Waves rolling into a harbour", model: "", duration_seconds: 5 }; },
"poll": function () { return { ticket: last || {} }; },
"wait": function () { return { ticket: last || {}, max_attempts: 10, max_wait_seconds: 40 }; },
"job": function () { return { ticket: last || {} }; },
"account": function (p) { return { provider: p }; },
"call": function (p) { return { provider: p, key: "", path: {}, query: {}, body: {} }; },
"fetch": function () { return { url: "https://example.com/result.png" }; },
"webhook": function () { return { secret: "whsec_...", webhook_id: "msg_...", timestamp: "0", signature: "v1,...", payload: "{}" }; }
};
var labels = {
"image": "image: submit a generation",
"video": "video: submit a generation",
"poll": "poll: one status check",
"wait": "wait: poll under caps",
"job": "job: cancel (DELETE)",
"account": "account: free credential check",
"call": "call: any table row",
"fetch": "fetch: download a result URL",
"webhook": "webhook: verify a Replicate signature"
};
var route = $("md-route"), prov = $("md-provider");
Object.keys(routes).forEach(function (k) {
var o = document.createElement("option");
o.value = k; o.textContent = labels[k];
route.appendChild(o);
});
function fill() {
var p = prov.value || "fal";
$("md-body").value = JSON.stringify(routes[route.value](p), null, 1);
}
route.onchange = fill;
prov.onchange = fill;
function send(confirm) {
var body;
try { body = JSON.parse($("md-body").value || "{}"); }
catch (e) { err.textContent = "Body is not valid JSON"; return; }
if (confirm) body.confirm = true;
err.textContent = "";
out.textContent = "...";
var key = route.value;
var url = "/ai/media/" + (key === "job" ? "job" : key);
fetch(url, {
method: key === "job" ? "DELETE" : "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body)
}).then(function (res) {
return res.text().then(function (t) {
var j;
try { j = JSON.parse(t); } catch (e) { j = { message: t }; }
if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; return; }
if (j.ticket) last = j.ticket;
out.textContent = JSON.stringify(j, null, 2);
var imgs = (j.ticket && j.ticket.assets) || j.assets || [];
imgs.forEach(function (a) {
if (a.data_base64 && /^image\//.test(a.mime || "")) {
var im = document.createElement("img");
im.src = "data:" + a.mime + ";base64," + a.data_base64;
im.style.maxWidth = "100%";
out.appendChild(document.createElement("br"));
out.appendChild(im);
}
});
});
}).catch(function (e) { err.textContent = String(e); out.textContent = ""; });
}
$("md-send").onclick = function () { send(false); };
$("md-confirm").onclick = function () { send(true); };
fetch("/ai/media/status").then(function (r) { return r.json(); }).then(function (s) {
providers = s.providers || [];
providers.forEach(function (p) {
var o = document.createElement("option");
o.value = p.id;
o.textContent = p.name + (p.configured ? "" : " (no key)");
prov.appendChild(o);
});
var ready = providers.filter(function (p) { return p.configured; }).length;
$("md-config").textContent = ready
? ready + " of " + providers.length + " providers have a credential. Paid calls need confirm."
: "No provider credential found: calls answer 503 until one is configured; dry runs still work.";
var ul = $("md-unsupported");
(s.unsupported || []).forEach(function (u) {
var li = document.createElement("li");
li.textContent = u.name + ": " + u.reason;
ul.appendChild(li);
});
fill();
});
})();
+254
View File
@@ -0,0 +1,254 @@
# Package media - Specification
Owned directory: `src/vendors/media/`. Namespace `molo::media`. One subdirectory
per provider, one shared `common/`. Depends on `molo::llm` (`ApiClient`,
`Service`, `ApiError`, `Config`), `molo::clients` and `security/crypto`
(Base64, Sha256, Hmac). Plan: `plans/18-breadth.md` section 2, method of
`plans/17-llm-vendor-platforms.md`. Demo: `example/app/ai/MediaDemo.cpp`,
routes `/ai/media`. Nothing in `src/llm` was edited.
## 1. Scope and module map
Twelve providers, one call table each (`<p>/calls.cpp`, checked by
`tools/check_spec.py`), one client class per host, generator adapters on a
neutral seam.
| Directory | Classes | Covers |
|---|---|---|
| `common/` | `CallTable`, `MediaService`, `PreparedCall`, `ImageGenerator`, `VideoGenerator`, `JobTicket`, `JobStatus`, `JobRunner`, `Downloader`, `OutputScanner`, `MediaRegistry`, `ClientFactory`, `AdapterSupport`, `MediaCaps` | the seam, request building, polling, downloads, provider list |
| `stability/` | `StabilityClient`, `StabilityImageGenerator` | v2beta Stable Image (generate ultra / core / sd3, edit x7, upscale x3, control x4), 3D (2), Stable Audio (6 + results), v1 engines list, account, balance, v1 text-to-image / image-to-image / masking |
| `bfl/` | `BflClient`, `BflImageGenerator`, `BflVideoGenerator` | all 25 FLUX model and tool endpoints, get_result polling, credits, fine-tune list / details / delete |
| `runway/` | `RunwayClient`, `RunwayImageGenerator`, `RunwayVideoGenerator` | every SDK route: generation (18), recipes (7), tasks, uploads, organization, avatars, voices, documents, routers, workflows, realtime sessions |
| `luma/` | `LumaClient`, `LumaVideoGenerator`, `LumaImageGenerator` | Ray video, Photon image, reframe, modify, upscale, audio, list, delete, concepts, credits |
| `replicate/` | `ReplicateClient`, `ReplicateWebhook`, `ReplicateJobs`, `ReplicateImageGenerator`, `ReplicateVideoGenerator` | all 36 operations: predictions (model, version, deployment), models, versions, deployments, files, trainings, hardware, collections, search, account, webhook secret; Standard Webhooks verification |
| `fal/` | `FalQueue`, `FalPlatform`, `FalImageGenerator`, `FalVideoGenerator` | queue API (submit, submit with path, status, result, cancel) and the model-related Platform API (search, pricing, estimate, usage, analytics, billing events, request listing, payload deletion, meta) |
| `ideogram/` | `IdeogramClient`, `IdeogramImageGenerator`, `IdeogramVideoGenerator` | all 70 v2 operations (image, SVG, video, ad tools, generations, account); free price quote through `dry_run=true` |
| `recraft/` | `RecraftClient`, `RecraftImageGenerator` | all 22 operations: generation (3 routes), image tools, styles, prompt enhancement, user |
| `leonardo/` | `LeonardoClient`, `LeonardoImageGenerator`, `LeonardoVideoGenerator` | v2 async and sync generations, models; v1 generation get and `me` (prose only) |
| `minimax/` | `MinimaxClient`, `MinimaxVideoGenerator` | Hailuo video v1 (create, query, file retrieve) and v2 (create, query, list, delete) |
| `kling/` | `KlingJwt`, `KlingClient`, `KlingVideoGenerator` | text2video, image2video, image generation, task queries; JWT auth |
| `higgsfield/` | `HiggsfieldClient`, `HiggsfieldJobs`, `HiggsfieldImageGenerator`, `HiggsfieldVideoGenerator` | the shared lifecycle (submit by model path, status, cancel, delete) for every model |
Products without a public API (not implemented, shown in
`MediaRegistry::unsupported()` and on the demo page):
- **Midjourney**: no public API. Its docs describe the web and Discord products
only; third-party relays are unofficial and break its terms.
- **Pika**: `pika.art/api` advertises a membership API that resells other
vendors' models, but no endpoint, schema or auth document exists
(2026-10-25). Pika models are reachable through fal.
## 2. The seam (`common/job.h`)
```
ImageRequest / VideoRequest neutral request, validate() with caps
ImageGenerator::plan(req) exact request as JSON, no network, no credential
ImageGenerator::submit(req, confirm) JobTicket (confirm has no default)
MediaGenerator::poll(ticket) JobStatus {queued|running|succeeded|failed|cancelled}
MediaGenerator::cancel(ticket, confirm) false when the API has no cancel
JobRunner::wait(generator, ticket) polls under hard caps
Downloader::fetch(asset) bytes of a URL or an inline base64 output
```
Synchronous providers (Stability, Recraft, Ideogram by default) return an
`immediate` ticket that already carries its assets; `JobRunner` answers those
without a poll. A failed job is a `JobStatus`, a transport or auth failure is an
`ApiError`.
## 3. Static caps (see `common/caps.h`)
| Knob | Value |
|---|---|
| Prompt | 32000 chars |
| Body fields / one text field | 128 / 1 MiB |
| Uploaded file / files per call | 16 MiB / 16 |
| Query args / value | 16 / 1024 chars |
| Path argument / segments of a multi-segment id | 256 chars / 6 |
| Extra headers | 8, 512 chars each |
| Images per request / video seconds / dimension | 8 / 60 / 8192 |
| Poll interval floor / ceiling | 500 ms / 30000 ms |
| Poll attempts / total wait | 240 / 1800 s (demo: 30 / 60 s) |
| Assets per job / download / URL | 16 / 128 MiB / 4096 chars |
| Scan depth / nodes | 8 / 4096 |
| Page walk / items returned | 20 / 1000 |
| Webhook tolerance / body | 300 s / 1 MiB |
| Call table | 256 rows |
`src/llm` caps still apply (32 MiB request, 64 MiB response).
## 4. Contracts
- Every call lives in the provider's table: key, method, path, spec operation
id, body kind (`none|json|multipart|either`), accepted body fields, accepted
query parameters, effect (`read|paid|write|delete`).
- `MediaService::prepare` is pure. It rejects unknown keys, missing or extra
path arguments, ids outside `[A-Za-z0-9._:=~-]` (and `.` / `..`), unknown query
parameters, unknown body fields, files on a JSON row, empty or oversized
files, bad header names. A rejected request never reaches the network.
- `MediaService::execute` refuses any non-read effect unless `confirmed` is
true. Typed methods that delete or cancel take `confirm` with no default.
The only two lowered effects are named and documented: Ideogram `quote`
(`dry_run=true`, defined by the spec as not billed) and Runway
`organization_usage` (a POST that only queries).
- No key: `configured()` is false and every call throws `NotConfiguredError`
(503) before any traffic. Keys travel in headers only.
- Lists report truncation (`"truncated": true`), never silently.
- Poll caps: the runner never goes below the interval floor, throws
`WAIT_EXPIRED` (504) naming the job when attempts or total wait run out, and
never cancels on its own.
- Errors stay `ApiError` with the vendor's status. Envelopes that hide an error
in a 200 are converted: MiniMax `base_resp.status_code`, Kling `code`.
## 5. Credentials
| Provider | Variables | Page |
|---|---|---|
| Stability AI | `STABILITY_API_KEY` | https://platform.stability.ai/account/keys (read: redirects to sign-in) |
| Black Forest Labs | `BFL_API_KEY` | https://dashboard.bfl.ai/ (read: redirects to sign-in) |
| Runway | `RUNWAY_API_KEY` (also `RUNWAYML_API_SECRET`, the SDK name) | https://dev.runwayml.com/ (read: 200) |
| Luma | `LUMA_API_KEY` | https://lumalabs.ai/api/keys (read: 200) |
| Replicate | `REPLICATE_API_TOKEN` | https://replicate.com/account/api-tokens (read: redirects to sign-in) |
| fal | `FAL_KEY` | https://fal.ai/dashboard/keys (read: redirects to sign-in) |
| Ideogram | `IDEOGRAM_API_KEY` | https://ideogram.ai/manage-api (unverified: Cloudflare challenge blocked the fetch) |
| Recraft | `RECRAFT_API_TOKEN` | https://www.recraft.ai/profile/api (read: redirects to sign-in with that callback) |
| Leonardo | `LEONARDO_API_KEY` | https://app.leonardo.ai/api-access (docs; fetch answered 429) |
| MiniMax | `MINIMAX_API_KEY` | https://platform.minimax.io/user-center/basic-information/interface-key (read: redirects to /console/access) |
| Kling | `KLING_ACCESS_KEY` and `KLING_SECRET_KEY` | https://app.klingai.com/global/dev/api-key (read: redirect) |
| Higgsfield | `HF_API_KEY_ID` and `HF_API_KEY_SECRET` | https://cloud.higgsfield.ai/ (docs; read: redirect) |
Add these rows to `src/llm/SPEC.md` section 5 at integration.
## 6. Sources
Per provider, the machine-readable source `tools/check_spec.py` reads.
| Provider | Source |
|---|---|
| Stability | https://docs.stability.ai/openapi.json (v2beta) and https://docs.stability.ai/openapi-v1.json; prose index https://docs.stability.ai/llms.txt |
| BFL | https://api.bfl.ai/openapi.json (version 0.0.1) |
| Replicate | https://api.replicate.com/openapi.json (server `https://api.replicate.com/v1`) |
| fal | queue: https://fal.ai/docs/api-reference/inference-api/openapi/queue.json; platform: https://api.fal.ai/v1/openapi.json; index https://docs.fal.ai/llms.txt |
| Ideogram | https://developer.ideogram.ai/v2/openapi.json |
| Recraft | https://www.recraft.ai/docs/api-reference/openapi.json |
| Higgsfield | https://docs.higgsfield.ai/docs/openapi.json (declared supplementary by the vendor) plus https://docs.higgsfield.ai/docs/llms.txt |
| Leonardo | OpenAPI embedded in https://docs.leonardo.ai/reference/creategeneration.md, `createsyncgeneration.md`, `getmodels.md`; quick start `docs/getting-started.md`, FAQ `docs/api-faq.md` |
| MiniMax | OpenAPI embedded in the `platform.minimax.io/docs/api-reference/video-generation-*.md` pages and `file-management-retrieve.md`; index `llms.txt` |
| Runway | npm `@runwayml/sdk` 4.24.0: `resources/**/*.js` (paths), `*.d.ts` (fields), `client.js` (base URL, `X-Runway-Version` default 2024-11-06) |
| Luma | npm `lumaai` 1.19.1: `resources/**/*.js`, `*.d.ts`, `index.js` (base URL) |
| Kling | none machine-readable; community sources (a third-party OpenAPI listing, a skill pack) plus live probes |
Cross-check: Stability, BFL, Replicate, Ideogram and Recraft are also covered by
live unauthenticated probes of the same paths (section 8). Standard Webhooks
vector for the Replicate verifier comes from the Standard Webhooks project.
## 7. Verified against spec
`python3 -I src/vendors/media/tools/check_spec.py`, run 2026-10-25 (twelve
providers, all green; a deliberately corrupted BFL row failed it with both a row
error and a coverage error, then was restored):
| Provider | Rows | Source operations | Result |
|---|---|---|---|
| stability | 34 | 37 (3 alpha operations excluded) | ok |
| bfl | 30 | 30 | ok |
| replicate | 36 | 36 | ok |
| fal | 16 | every queue and model-related platform operation (1 SSE stream route excluded; 78 platform operations outside model inference out of scope) | ok |
| ideogram | 70 | 70 | ok |
| recraft | 22 | 22 | ok |
| leonardo | 6 | 3 (3 v1 rows prose only, listed by the checker) | ok |
| runway | 66 | 66 SDK calls | ok |
| luma | 13 | 13 SDK calls | ok |
| minimax | 7 | 7 (5 reference pages listed as not implemented) | ok |
| higgsfield | 10 | 9 (1 generic row by design) | ok |
| kling | 6 | probe only | UNVERIFIED |
The checker verifies: each row's method and path exist, operation ids match,
body kind matches the spec content types, every body field and query parameter
we send exists, each response schema field we read is reachable, and every
source operation is in the table or in the explicit exclusion list.
## 8. Verified live
Unauthenticated probes, 2026-10-25, recorded by `probe` runs and by
`tests/src/vendors/media/test_live.cpp` (`MOLO_LIVE_NET=1`, 13 providers, green):
| Host and path | Invalid key answered |
|---|---|
| api.stability.ai `/v1/user/balance` | 401 `{"id","message","name"}` |
| api.stability.ai `/v2beta/results/00000000` | 400 `{"errors":["id: string must contain exactly 64 character(s)"],...}` (the result id is 64 characters) |
| api.bfl.ai `/v1/credits` | 422 `{"detail":"Invalid API key format"}` |
| api.replicate.com `/v1/account` | 401 `{"title","detail","status"}` |
| api.fal.ai `/v1/models/pricing` | 401 `{"error":{"type":"authorization_error",...}}` |
| queue.fal.run `/fal-ai/flux/requests/<id>/status` | 401 `{"detail":"Authentication is required"}` |
| api.fal.ai `/v1/models`, `/v1/meta` | 200 without any key (public) |
| api.ideogram.ai `/v2/account/api-keys` | 401 `{"message":"Access denied..."}`; `/v2/account/usage` answers 400 for the missing `start_time` before auth |
| external.api.recraft.ai `/v1/users/me` | 401 `request unauthorized` (plain text) |
| cloud.leonardo.ai `/api/rest/v2/models`, `/v1/me` | 401 `{"error","path","code":"access-denied"}` |
| api.dev.runwayml.com `/v1/organization` | 401 `{"error":"...did not start with key_","docUrl"}` |
| api.lumalabs.ai `/dream-machine/v1/credits` | 403 `{"detail":"Not authenticated"}` |
| api.minimax.io `/v1/query/video_generation` | 200 with `base_resp.status_code 1004` (converted to an ApiError) |
| api-singapore.klingai.com `/v1/videos/text2video/...` | 401 `{"code":1002,...}`; api.klingai.com answers `{"code":1000,"message":"Auth failed"}` |
| api.higgsfield.ai `/requests/<id>/status` | 401 `{"detail":"Invalid credentials"}` |
No success body of any provider has been seen: no key exists here. Live tests
with keys (`MOLO_LIVE_NET=1` plus the provider variables) run one free read per
provider (`MediaRegistry::account`). MiniMax, Kling and Higgsfield have no free
account read, so a key is never proven for them without a paid or resource call.
## 9. Unverified
Every shape inferred rather than read from a spec or seen:
- All success and result bodies, for every provider (no key). Status words and
result locations come from the specs or SDK types where they exist (BFL
`StatusResponse` enum and `result.sample`, fal `QueueStatus`, Ideogram
`GenerationResponse`, Higgsfield `RequestStatus`, Luma and Runway SDK
types, MiniMax pages) and are exercised by local mapper tests only.
- **Kling**: all six paths, the request fields, the `{code,data:{task_id,
task_status,task_result}}` envelope, the statuses and the JWT claims
(`iss`, `exp`, `nbf`, 1800 s) come from community sources, not from Kling.
Only the paths and the auth refusal were probed. The JWT builder matches an
independent HS256 implementation but Kling's acceptance of it was never seen.
- **Leonardo**: the polling route `GET /api/rest/v1/generations/{id}`, its
`generations_by_pk.{status,generated_images[].url}` shape, `GET /v1/me`, and
the v1 create body are described by quick start and FAQ prose only (the v1
reference pages were removed). Whether a v2 `generationId` is pollable on the
v1 route was not confirmed.
- **Higgsfield**: only five model paths are in the OpenAPI document; any other
path goes through the generic row (`model.submit`) with an open body.
- **fal**: result URL locations differ per model; `OutputScanner` finds URLs
generically. A failed request is assumed to answer the result fetch with an
HTTP 4xx (the queue spec says `COMPLETED` also covers failure).
- **BFL**: `polling_url` may point at a regional host; polling uses
`get_result?id=` on the configured host instead.
- **Recraft** default raster output is lossless WebP (docs); inline results are
labelled `image/webp` on that basis.
- **Replicate** model names listed by `models()` and the other adapters'
`models()` lists are names seen in the docs and specs, not a live catalogue.
- Runway `X-Runway-Version` 2024-11-06 is the SDK default; newer versions may
change bodies.
- Console URLs in section 5 marked as such.
## 10. Gaps
- Foundation (`src/llm`, report to its owner): no unauthenticated requests (so
fal's public model search needs a key through our client); no GET stream (fal
status stream, excluded); per-call timeouts missing; `error_from_response`
leaves `type` empty for `{"message"}`, `{"detail"}` and `{"errors"}`
envelopes (Stability, BFL, Replicate, Higgsfield, Luma, Runway answer
`UNKNOWN`); the 32 MiB request ceiling blocks large video uploads (package
files are capped at 16 MiB each); `post_multipart` is bypassed through
`request_raw`.
- Another suite in the shared test binary unsets `MOLO_LIVE_NET`
(`tests/src/google/optimize/test_gaps.cpp`); the media live tests read it once
at startup to be immune.
- Not implemented: Stability gRPC and v2beta image-to-video (absent from the
published spec); MiniMax Video Agent, H3-Context-IR and regeneration tasks;
fal Platform assets / serverless / compute / keys / workflows; Replicate list
cursors (first page only, `truncated` is set); Ideogram and Replicate webhook
receivers (Replicate verification exists); WebSocket / realtime routes of
Runway beyond session creation.
- Recipes and avatar conversations of Runway are table rows only, with no typed
wrapper: use `MediaService::call_json`.
- Fetching a result needs a second step: `Downloader` is separate from
`poll()` so that callers choose when to spend bandwidth.
+146
View File
@@ -0,0 +1,146 @@
#include "vendors/media/bfl/bfl.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs with_body(const Json& body) {
CallArgs a;
a.body = body;
return a;
}
JobTicket ticket_from(const Json& out, const std::string& model, MediaKind kind) {
JobTicket t;
t.provider = "bfl";
t.model = model;
t.kind = kind;
t.id = AdapterSupport::str(out, "id");
if (t.id.empty()) throw llm::ApiError("bfl", 502, "BAD_RESPONSE", "no id in the submit reply");
t.meta = Json::object();
AdapterSupport::put(t.meta, "cost", AdapterSupport::str(out, "cost"));
return t;
}
JobStatus status_of(BflClient& client, const JobTicket& ticket, MediaKind kind) {
CallArgs a;
a.query["id"] = ticket.id;
Json out = client.call_json("get_result", a);
JobStatus s;
s.raw_state = AdapterSupport::str(out, "status");
s.state = BflImageGenerator::state_of(s.raw_state);
if (const Json* p = out.find("progress"); p && p->is_number()) s.progress = p->as_number();
if (s.state == JobState::kSucceeded) {
if (const Json* r = out.find("result")) s.assets = OutputScanner::scan(*r, kind);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "Ready but no result URL in the reply";
}
} else if (s.state == JobState::kFailed) {
s.error = s.raw_state;
if (const Json* d = out.find("details"); d && d->is_object()) s.error += ": " + d->dump();
}
return s;
}
} // namespace
PreparedCall BflClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
Json BflClient::submit(const std::string& key, const Json& body, bool confirm) {
return call_json(key, with_body(body), confirm);
}
Json BflClient::get_result(const std::string& id) {
CallArgs a;
a.query["id"] = id;
return call_json("get_result", a);
}
Json BflClient::credits() { return call_json("credits"); }
Json BflClient::my_finetunes() { return call_json("my_finetunes"); }
Json BflClient::finetune_details(const std::string& finetune_id) {
CallArgs a;
a.query["finetune_id"] = finetune_id;
return call_json("finetune_details", a);
}
Json BflClient::delete_finetune(const std::string& finetune_id, bool confirm) {
Json b = Json::object();
b.set("finetune_id", Json(finetune_id));
return call_json("delete_finetune", with_body(b), confirm);
}
JobState BflImageGenerator::state_of(const std::string& status) {
if (status == "Ready") return JobState::kSucceeded;
if (status == "Pending" || status == "Reasoning") return JobState::kQueued;
if (status == "Generating") return JobState::kRunning;
if (status == "Task not found" || status == "Error" || status == "Request Moderated" ||
status == "Content Moderated" || status == "No Card Generated")
return JobState::kFailed;
return JobState::kRunning;
}
std::vector<std::string> BflImageGenerator::models() const {
return {"flux-2-pro", "flux-2-max", "flux-2-flex", "flux-2-klein-9b", "flux-2-klein-4b", "flux-kontext-pro",
"flux-kontext-max", "flux-pro-1.1", "flux-pro-1.1-ultra", "flux-dev", "flux-3-image"};
}
Json BflImageGenerator::body(const ImageRequest& r, std::string& key) const {
r.validate();
key = r.model.empty() ? "flux-2-pro" : r.model;
if (r.count != 1) throw llm::ApiError::invalid("bfl", "one image per call (count must be 1)");
Json b = Json::object();
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put_int(b, "width", r.width);
AdapterSupport::put_int(b, "height", r.height);
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put_seed(b, "seed", r.seed);
AdapterSupport::put(b, "input_image", r.source_url);
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json BflImageGenerator::plan(const ImageRequest& request) const {
std::string key;
Json b = body(request, key);
return client_->plan(key, b).summary();
}
JobTicket BflImageGenerator::submit(const ImageRequest& request, bool confirm) {
std::string key;
Json b = body(request, key);
AdapterSupport::need_confirm(confirm, "bfl", "generate");
return ticket_from(client_->submit(key, b, true), key, MediaKind::kImage);
}
JobStatus BflImageGenerator::poll(const JobTicket& ticket) { return status_of(*client_, ticket, MediaKind::kImage); }
Json BflVideoGenerator::body(const VideoRequest& r) const {
r.validate();
Json b = Json::object();
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put_int(b, "duration", r.duration_seconds);
AdapterSupport::put(b, "resolution", r.resolution);
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json BflVideoGenerator::plan(const VideoRequest& request) const {
return client_->plan("flux-3-video", body(request)).summary();
}
JobTicket BflVideoGenerator::submit(const VideoRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "bfl", "generate");
return ticket_from(client_->submit("flux-3-video", b, true), "flux-3-video", MediaKind::kVideo);
}
JobStatus BflVideoGenerator::poll(const JobTicket& ticket) { return status_of(*client_, ticket, MediaKind::kVideo); }
} // namespace molo::media
+76
View File
@@ -0,0 +1,76 @@
#pragma once
// Black Forest Labs (FLUX): submit to a model endpoint, poll /v1/get_result.
// Auth is the x-key header. Models are endpoints (flux-2-pro, flux-kontext-pro,
// flux-pro-1.1-ultra, ...), tools are endpoints under /v1/flux-tools/.
#include "vendors/media/bfl/calls.h"
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
namespace molo::media {
class BflClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.bfl.ai";
static constexpr const char* kKeyEnv = "BFL_API_KEY";
explicit BflClient(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), BflCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::header("bfl", kHost, kKeyEnv, "x-key"); }
static std::shared_ptr<BflClient> from_env() {
return std::make_shared<BflClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<BflClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<BflClient>(ClientFactory::with(std::move(t), config(), key));
}
// key = a POST model / tool row ("flux-2-pro", "tools.erase-v1"). Returns {id, polling_url, cost, ...}.
Json submit(const std::string& key, const Json& body, bool confirm);
PreparedCall plan(const std::string& key, const Json& body) const;
Json get_result(const std::string& id);
Json credits();
Json my_finetunes();
Json finetune_details(const std::string& finetune_id);
// Destructive: no default for confirm.
Json delete_finetune(const std::string& finetune_id, bool confirm);
};
class BflImageGenerator : public ImageGenerator {
public:
explicit BflImageGenerator(std::shared_ptr<BflClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "bfl"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override;
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; } // no cancel endpoint
int suggested_poll_ms() const override { return 1500; }
// Maps the BFL status word onto JobState (shared with the video adapter).
static JobState state_of(const std::string& status);
private:
Json body(const ImageRequest& request, std::string& key) const;
std::shared_ptr<BflClient> client_;
};
class BflVideoGenerator : public VideoGenerator {
public:
explicit BflVideoGenerator(std::shared_ptr<BflClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "bfl"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override { return {"flux-3-video"}; }
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; }
int suggested_poll_ms() const override { return 4000; }
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<BflClient> client_;
};
} // namespace molo::media
+57
View File
@@ -0,0 +1,57 @@
#include "vendors/media/bfl/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"finetune_details", "GET", "/v1/finetune_details", "finetune_details_v1_finetune_details_get", "none", "", "finetune_id", "read"},
{"my_finetunes", "GET", "/v1/my_finetunes", "my_finetunes_v1_my_finetunes_get", "none", "", "", "read"},
{"delete_finetune", "POST", "/v1/delete_finetune", "delete_finetune_v1_delete_finetune_post", "json", "finetune_id", "", "delete"},
{"flux-pro-1.1", "POST", "/v1/flux-pro-1.1", "flux_pro_1_1_v1_flux_pro_1_1_post", "json", "user,prompt,image_prompt,width,height,prompt_upsampling,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-dev", "POST", "/v1/flux-dev", "flux_dev_v1_flux_dev_post", "json", "user,prompt,image_prompt,width,height,steps,prompt_upsampling,seed,guidance,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-pro-1.1-ultra", "POST", "/v1/flux-pro-1.1-ultra", "generate_flux_1_1_ultra_v1_flux_pro_1_1_ultra_post", "json", "user,prompt,prompt_upsampling,seed,aspect_ratio,safety_tolerance,output_format,raw,image_prompt,image_prompt_strength,webhook_url,webhook_secret", "", "paid"},
{"flux-pro-1.0-fill", "POST", "/v1/flux-pro-1.0-fill", "fill_v1_flux_pro_1_0_fill_post", "json", "user,image,mask,prompt,steps,prompt_upsampling,seed,guidance,output_format,safety_tolerance,webhook_url,webhook_secret", "", "paid"},
{"flux-pro-1.0-expand", "POST", "/v1/flux-pro-1.0-expand", "expand_v1_flux_pro_1_0_expand_post", "json", "user,image,top,bottom,left,right,prompt,steps,prompt_upsampling,seed,guidance,output_format,safety_tolerance,webhook_url,webhook_secret", "", "paid"},
{"flux-pro-1.1-ultra-finetuned", "POST", "/v1/flux-pro-1.1-ultra-finetuned", "generate_bigblue_finetuned_v1_flux_pro_1_1_ultra_finetuned_post", "json", "user,finetune_id,finetune_strength,prompt,seed,aspect_ratio,safety_tolerance,output_format,image_prompt,image_prompt_strength,prompt_upsampling,webhook_url,webhook_secret", "", "paid"},
{"flux-2-flex", "POST", "/v1/flux-2-flex", "generate_flux_2_flex_v1_flux_2_flex_post", "json", "user,prompt,prompt_upsampling,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,input_image_blob_path,seed,width,height,guidance,steps,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-2-klein-9b", "POST", "/v1/flux-2-klein-9b", "generate_flux_2_klein_9b_v1_flux_2_klein_9b_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-2-klein-9b-preview", "POST", "/v1/flux-2-klein-9b-preview", "generate_flux_2_klein_9b_kv_v1_flux_2_klein_9b_preview_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-2-klein-4b", "POST", "/v1/flux-2-klein-4b", "generate_flux_2_klein_4b_v1_flux_2_klein_4b_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-2-max", "POST", "/v1/flux-2-max", "generate_flux_2_max_v1_flux_2_max_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-2-pro-preview", "POST", "/v1/flux-2-pro-preview", "generate_flux_2_pro_preview_v1_flux_2_pro_preview_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-2-pro", "POST", "/v1/flux-2-pro", "generate_flux_2_pro_v1_flux_2_pro_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-3-video", "POST", "/v1/flux-3-video", "flux_3_video_v1_flux_3_video_post", "json", "user,prompt,aspect_ratio,duration,resolution,version,generate_audio,safety_tolerance,draft,mode,keyframes,start_video,draft_cache", "", "paid"},
{"flux-3-image", "POST", "/v1/flux-3-image", "flux_3_image_v1_flux_3_image_post", "json", "user,prompt,images,aspect_ratio,resolution,width,height,safety_tolerance,grounding,version", "", "paid"},
{"flux-3-image-klein", "POST", "/v1/flux-3-image-klein", "flux_3_image_klein_v1_flux_3_image_klein_post", "json", "user,prompt,images,aspect_ratio,resolution,width,height,seed,safety_tolerance,output_format,steps", "", "paid"},
{"tools.video-upscale-v1", "POST", "/v1/flux-tools/video-upscale-v1", "generate_flux_tools_video_upscale_v1_v1_flux_tools_video_upscale_v1_post", "json", "user,input_video,prompt,creativity,upscale_factor,safety_tolerance,webhook_url,webhook_secret", "", "paid"},
{"tools.video-edit-v1", "POST", "/v1/flux-tools/video-edit-v1", "generate_flux_tools_video_edit_v1_v1_flux_tools_video_edit_v1_post", "json", "user,video,prompt,safety_tolerance", "", "paid"},
{"tools.outpainting-v1", "POST", "/v1/flux-tools/outpainting-v1", "generate_flux_tools_outpainting_v1_v1_flux_tools_outpainting_v1_post", "json", "user,input_image,width,height,auto_crop,safety_tolerance,output_format,prompt,reference_offset_x,reference_offset_y,mode,disable_pup", "", "paid"},
{"tools.vto-v1", "POST", "/v1/flux-tools/vto-v1", "generate_flux_tools_vto_v1_v1_flux_tools_vto_v1_post", "json", "user,prompt,person,garment,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"tools.vto-v2", "POST", "/v1/flux-tools/vto-v2", "generate_flux_tools_vto_v2_v1_flux_tools_vto_v2_post", "json", "user,prompt,person,garment,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"tools.erase-v1", "POST", "/v1/flux-tools/erase-v1", "generate_flux_tools_erase_v1_v1_flux_tools_erase_v1_post", "json", "user,image,mask,dilate_pixels,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"tools.deblur-v1", "POST", "/v1/flux-tools/deblur-v1", "generate_flux_tools_deblur_v1_v1_flux_tools_deblur_v1_post", "json", "user,image,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
{"flux-kontext-max", "POST", "/v1/flux-kontext-max", "generate_flux_kontext_max_v1_flux_kontext_max_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,aspect_ratio,output_format,webhook_url,webhook_secret,prompt_upsampling,safety_tolerance", "", "paid"},
{"flux-kontext-pro", "POST", "/v1/flux-kontext-pro", "generate_flux_kontext_pro_v1_flux_kontext_pro_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,aspect_ratio,output_format,webhook_url,webhook_secret,prompt_upsampling,safety_tolerance", "", "paid"},
{"get_result", "GET", "/v1/get_result", "get_result_v1_get_result_get", "none", "", "id", "read"},
{"credits", "GET", "/v1/credits", "get_credits_v1_credits_get", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"AsyncResponse", "id,polling_url,cost,input_mp,output_mp"},
{"ResultResponse", "id,status,result,progress,details,preview"},
{"CreditsResponse", "credits"},
};
// FIELDS-END
const CallTable& BflCalls::table() {
static const CallTable t("bfl", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t BflCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* BflCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct BflCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+50
View File
@@ -0,0 +1,50 @@
#pragma once
// Small helpers every generator adapter shares: the confirmation gate, body
// assembly that skips unset values, and extra-field merging.
#include <string>
#include "clients/body/json.h"
#include "llm/error.h"
#include "vendors/media/common/job.h"
namespace molo::media {
struct AdapterSupport {
// Every submit / cancel passes through here first: no confirm, no call.
static void need_confirm(bool confirm, const std::string& provider, const std::string& what) {
if (!confirm)
throw llm::ApiError::invalid(provider, what + " needs explicit confirmation (it costs money or changes a resource)");
}
static void put(clients::body::Json& obj, const std::string& key, const std::string& value) {
if (!value.empty()) obj.set(key, clients::body::Json(value));
}
static void put_int(clients::body::Json& obj, const std::string& key, int value) {
if (value > 0) obj.set(key, clients::body::Json(value));
}
static void put_seed(clients::body::Json& obj, const std::string& key, long seed) {
if (seed >= 0) obj.set(key, clients::body::Json(static_cast<double>(seed)));
}
// Provider fields from `extra` win over derived ones (the caller asked).
static void merge_extra(clients::body::Json& obj, const clients::body::Json& extra) {
if (!extra.is_object()) return;
for (const auto& [k, v] : extra.as_object()) obj.set(k, v);
}
static std::string str(const clients::body::Json& doc, const std::string& key) {
const clients::body::Json* v = doc.is_object() ? doc.find(key) : nullptr;
if (v && v->is_string()) return v->as_string();
if (v && v->is_number()) return v->dump();
return "";
}
static const clients::body::Json* obj_at(const clients::body::Json& doc, const std::string& key) {
const clients::body::Json* v = doc.is_object() ? doc.find(key) : nullptr;
return v && v->is_object() ? v : nullptr;
}
};
} // namespace molo::media
+162
View File
@@ -0,0 +1,162 @@
#include "vendors/media/common/call_table.h"
#include <cstring>
#include <stdexcept>
#include "llm/error.h"
#include "vendors/media/common/caps.h"
namespace molo::media {
namespace {
bool one_of(const char* value, std::initializer_list<const char*> options) {
for (const char* o : options)
if (std::strcmp(value, o) == 0) return true;
return false;
}
} // namespace
CallTable::CallTable(const char* vendor, const CallRow* rows, std::size_t count)
: vendor_(vendor), rows_(rows), count_(count) {
if (count > kMaxRows) throw std::logic_error(vendor_ + ": call table over " + std::to_string(kMaxRows) + " rows");
for (std::size_t i = 0; i < count; ++i) {
const CallRow& r = rows[i];
if (!index_.emplace(r.key, i).second) throw std::logic_error(vendor_ + ": duplicate call key " + r.key);
if (!one_of(r.http_method, {"GET", "POST", "PUT", "PATCH", "DELETE"}))
throw std::logic_error(vendor_ + ": bad method on " + r.key);
if (!one_of(r.body_kind, {"none", "json", "multipart", "either", "form"}))
throw std::logic_error(vendor_ + ": bad body kind on " + r.key);
if (!one_of(r.effect, {"read", "paid", "write", "delete"}))
throw std::logic_error(vendor_ + ": bad effect on " + r.key);
if (r.path[0] != '/') throw std::logic_error(vendor_ + ": path must start with / on " + r.key);
}
}
const CallRow& CallTable::at(std::size_t index) const {
if (index >= count_) throw llm::ApiError::invalid(vendor_, "call index out of range");
return rows_[index];
}
const CallRow* CallTable::find(const std::string& key) const {
auto it = index_.find(key);
return it == index_.end() ? nullptr : &rows_[it->second];
}
const CallRow& CallTable::get(const std::string& key) const {
const CallRow* row = find(key);
if (!row) throw llm::ApiError::invalid(vendor_, "unknown call '" + key + "'");
return *row;
}
Effect CallTable::effect_of(const CallRow& row) {
if (std::strcmp(row.effect, "paid") == 0) return Effect::kPaid;
if (std::strcmp(row.effect, "write") == 0) return Effect::kWrite;
if (std::strcmp(row.effect, "delete") == 0) return Effect::kDelete;
return Effect::kRead;
}
const char* CallTable::effect_name(Effect effect) {
switch (effect) {
case Effect::kPaid: return "paid";
case Effect::kWrite: return "write";
case Effect::kDelete: return "delete";
default: return "read";
}
}
clients::Method CallTable::method_of(const CallRow& row) {
const char* m = row.http_method;
if (std::strcmp(m, "POST") == 0) return clients::Method::kPost;
if (std::strcmp(m, "PUT") == 0) return clients::Method::kPut;
if (std::strcmp(m, "PATCH") == 0) return clients::Method::kPatch;
if (std::strcmp(m, "DELETE") == 0) return clients::Method::kDelete;
return clients::Method::kGet;
}
std::vector<std::string> CallTable::split(const char* csv) {
std::vector<std::string> out;
std::string cur;
for (const char* p = csv; *p; ++p) {
if (*p == ',') {
if (!cur.empty()) out.push_back(cur);
cur.clear();
} else {
cur.push_back(*p);
}
}
if (!cur.empty()) out.push_back(cur);
return out;
}
std::vector<std::string> CallTable::placeholders(const char* path) {
std::vector<std::string> out;
std::string p(path);
std::size_t pos = 0;
while ((pos = p.find('{', pos)) != std::string::npos) {
std::size_t end = p.find('}', pos);
if (end == std::string::npos) break;
out.push_back(p.substr(pos + 1, end - pos - 1));
pos = end + 1;
}
return out;
}
bool CallTable::body_allows(const CallRow& row, const std::string& field) const {
for (const std::string& f : split(row.body_fields))
if (f == "*" || f == field) return true;
return false;
}
bool CallTable::query_allows(const CallRow& row, const std::string& name) const {
for (const std::string& f : split(row.query_params))
if (f == name) return true;
return false;
}
bool CallTable::valid_segment(const std::string& id) {
if (id.empty() || id.size() > MediaCaps::kMaxPathArgChars || id == "." || id == "..") return false;
for (char c : id)
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' ||
c == '_' || c == '.' || c == ':' || c == '=' || c == '~'))
return false;
return true;
}
std::string CallTable::render_path(const CallRow& row,
const std::map<std::string, std::string>& args) const {
std::vector<std::string> names = placeholders(row.path);
for (const auto& [name, value] : args) {
bool known = false;
for (const std::string& n : names) known = known || n == name;
if (!known) throw llm::ApiError::invalid(vendor_, "call '" + std::string(row.key) + "' has no path argument '" + name + "'");
(void)value;
}
std::string path = row.path;
for (const std::string& name : names) {
auto it = args.find(name);
if (it == args.end()) throw llm::ApiError::invalid(vendor_, "call '" + std::string(row.key) + "' needs path argument '" + name + "'");
const std::string& value = it->second;
bool multi = name == "endpoint_id" || name == "model_path" || name == "path";
if (multi) {
std::size_t segments = 0, start = 0;
while (true) {
std::size_t slash = value.find('/', start);
std::string seg = value.substr(start, slash == std::string::npos ? std::string::npos : slash - start);
if (!valid_segment(seg)) throw llm::ApiError::invalid(vendor_, "bad " + name);
if (++segments > MediaCaps::kMaxPathSegments) throw llm::ApiError::invalid(vendor_, name + " has too many segments");
if (slash == std::string::npos) break;
start = slash + 1;
}
} else if (!valid_segment(value)) {
throw llm::ApiError::invalid(vendor_, "bad " + name);
}
std::string token = "{" + name + "}";
std::size_t at = path.find(token);
path.replace(at, token.size(), value);
}
return path;
}
} // namespace molo::media
+70
View File
@@ -0,0 +1,70 @@
#pragma once
// One table row per REST call. Every provider has exactly one table (calls.cpp
// in its directory); tools/check_spec.py checks each row against the
// provider's published spec. Code never builds a path from a literal.
#include <cstddef>
#include <map>
#include <string>
#include <vector>
#include "clients/request.h"
namespace molo::media {
// What a call does to the account. Anything but kRead needs explicit
// confirmation at execution time (MediaService::execute).
enum class Effect { kRead, kPaid, kWrite, kDelete };
struct CallRow {
const char* key; // short stable name, unique within the provider
const char* http_method; // GET POST PUT PATCH DELETE
const char* path; // from the host, "{placeholder}" segments
const char* op; // spec operation id when the spec has one
const char* body_kind; // none | json | multipart | either | form
const char* body_fields; // comma list of accepted body fields; "*" = open
const char* query_params; // comma list of accepted query parameters
const char* effect; // read | paid | write | delete
};
class CallTable {
public:
static constexpr std::size_t kMaxRows = 256;
// Throws std::logic_error on a duplicate key, a bad method / kind / effect,
// or more than kMaxRows rows (a table bug, caught by the unit tests).
CallTable(const char* vendor, const CallRow* rows, std::size_t count);
const std::string& vendor() const { return vendor_; }
std::size_t size() const { return count_; }
const CallRow& at(std::size_t index) const;
const CallRow* find(const std::string& key) const;
// Throws llm::ApiError::invalid for an unknown key.
const CallRow& get(const std::string& key) const;
static Effect effect_of(const CallRow& row);
static const char* effect_name(Effect effect);
static clients::Method method_of(const CallRow& row);
static std::vector<std::string> split(const char* csv);
static std::vector<std::string> placeholders(const char* path);
bool body_allows(const CallRow& row, const std::string& field) const;
bool query_allows(const CallRow& row, const std::string& name) const;
// One URL path segment: 1..256 of [A-Za-z0-9._:=~-], not "." or "..".
static bool valid_segment(const std::string& id);
// Replaces every {placeholder}; each must be supplied, none may be extra.
// Placeholders named endpoint_id, model_path or path accept 1..6 "/"-joined
// segments, every other placeholder exactly one. Throws ApiError::invalid.
std::string render_path(const CallRow& row,
const std::map<std::string, std::string>& args) const;
private:
std::string vendor_;
const CallRow* rows_;
std::size_t count_;
std::map<std::string, std::size_t> index_;
};
} // namespace molo::media
+47
View File
@@ -0,0 +1,47 @@
#pragma once
// Static ceilings for package media. Never autotuned. Request and response
// byte ceilings that come from src/llm (32 MiB / 64 MiB) are not repeated here;
// ours are the ones the media workflows add: files, polls, downloads, pages.
#include <cstddef>
namespace molo::media {
struct MediaCaps {
// Request shaping.
static constexpr std::size_t kMaxPromptChars = 32000;
static constexpr std::size_t kMaxBodyFields = 128;
static constexpr std::size_t kMaxFieldChars = 1u << 20; // one text field, 1 MiB
static constexpr std::size_t kMaxFileBytes = 16u * 1024u * 1024u; // one uploaded file
static constexpr std::size_t kMaxFiles = 16;
static constexpr std::size_t kMaxQueryArgs = 16;
static constexpr std::size_t kMaxQueryValueChars = 1024;
static constexpr std::size_t kMaxPathArgChars = 256;
static constexpr std::size_t kMaxPathSegments = 6; // multi-segment ids (fal endpoints)
static constexpr std::size_t kMaxHeaders = 8;
static constexpr std::size_t kMaxHeaderValueChars = 512;
// Generation limits we enforce before spending money.
static constexpr int kMaxImagesPerRequest = 8;
static constexpr int kMaxVideoSeconds = 60;
static constexpr int kMaxDimension = 8192;
// Job polling: floors and ceilings (the machine never goes below / above).
static constexpr int kMinPollIntervalMs = 500;
static constexpr int kMaxPollIntervalMs = 30000;
static constexpr int kMaxPollAttempts = 240;
static constexpr int kMaxWaitSeconds = 1800;
// Outputs.
static constexpr std::size_t kMaxAssetsPerJob = 16;
static constexpr std::size_t kMaxDownloadBytes = 128u * 1024u * 1024u;
static constexpr std::size_t kMaxUrlChars = 4096;
static constexpr std::size_t kMaxScanDepth = 8;
static constexpr std::size_t kMaxScanNodes = 4096;
// Listings.
static constexpr std::size_t kMaxPages = 20;
static constexpr std::size_t kMaxItemsReturned = 1000;
// Webhooks.
static constexpr long kWebhookToleranceSeconds = 300;
static constexpr std::size_t kMaxWebhookBodyBytes = 1u * 1024u * 1024u;
};
} // namespace molo::media
+58
View File
@@ -0,0 +1,58 @@
#pragma once
// Builds the llm::ApiClient every provider client wraps. One place, so the
// credential lookup (environment, then .env.json.local, then .env.json via
// llm::Config) and the transport choice are identical for all providers.
#include <initializer_list>
#include <memory>
#include <string>
#include "clients/transports/http_transport.h"
#include "llm/api_client.h"
#include "llm/config.h"
namespace molo::media {
struct ClientFactory {
// First non-empty value among `names` (llm::Config::get), else "".
static std::string env(std::initializer_list<const char*> names) {
for (const char* n : names) {
std::string v = llm::Config::get(n);
if (!v.empty()) return v;
}
return "";
}
// Real HTTPS. An empty key leaves the client "not configured".
static std::shared_ptr<llm::ApiClient> real(llm::ApiClientConfig config, const std::string& key) {
return std::make_shared<llm::ApiClient>(std::make_unique<clients::transports::HttpTransport>(),
std::move(config), key);
}
// Any transport (tests record the requests the real client builds).
static std::shared_ptr<llm::ApiClient> with(std::unique_ptr<clients::Transport> transport,
llm::ApiClientConfig config, const std::string& key) {
return std::make_shared<llm::ApiClient>(std::move(transport), std::move(config), key);
}
static llm::ApiClientConfig bearer(const std::string& vendor, const std::string& base_url,
const std::string& key_env) {
llm::ApiClientConfig c;
c.vendor = vendor;
c.base_url = base_url;
c.key_env = key_env;
c.auth = llm::AuthStyle::kBearer;
return c;
}
static llm::ApiClientConfig header(const std::string& vendor, const std::string& base_url,
const std::string& key_env, const std::string& header_name) {
llm::ApiClientConfig c = bearer(vendor, base_url, key_env);
c.auth = llm::AuthStyle::kHeader;
c.auth_header = header_name;
return c;
}
};
} // namespace molo::media
+74
View File
@@ -0,0 +1,74 @@
#include "vendors/media/common/downloader.h"
#include "clients/transports/http_transport.h"
#include "llm/error.h"
#include "security/crypto/Base64.h"
#include "vendors/media/common/caps.h"
namespace molo::media {
namespace {
clients::ClientConfig plain_config() {
clients::ClientConfig c;
c.persist_cookies = false;
return c;
}
std::unique_ptr<clients::Transport> real_or(std::unique_ptr<clients::Transport> t) {
if (t) return t;
return std::make_unique<clients::transports::HttpTransport>();
}
} // namespace
Downloader::Downloader(std::unique_ptr<clients::Transport> transport)
: client_(real_or(std::move(transport)), plain_config()) {}
void Downloader::validate_url(const std::string& url) {
auto bad = [](const std::string& why) { throw llm::ApiError::invalid("media", why); };
if (url.empty() || url.size() > MediaCaps::kMaxUrlChars) bad("download url length");
if (url.rfind("https://", 0) != 0) bad("download url must be https");
for (unsigned char c : url)
if (c <= 0x20 || c == 0x7f) bad("download url has whitespace or control characters");
std::size_t host_end = url.find_first_of("/?#", 8);
std::string authority = url.substr(8, host_end == std::string::npos ? std::string::npos : host_end - 8);
if (authority.empty()) bad("download url has no host");
if (authority.find('@') != std::string::npos) bad("download url must not carry credentials");
}
std::string Downloader::fetch_url(const std::string& url, std::string* content_type) {
validate_url(url);
try {
clients::ClientRequest req(clients::Method::kGet, clients::Url::parse(url));
clients::ClientResponse res = client_.send_blocking(std::move(req));
if (!res.is_success())
throw llm::ApiError("media", res.status(), "DOWNLOAD_FAILED", "download answered HTTP " + std::to_string(res.status()));
if (res.body().size() > MediaCaps::kMaxDownloadBytes)
throw llm::ApiError("media", 502, "BAD_RESPONSE", "download over the byte cap");
if (content_type) *content_type = res.content_type();
return res.body();
} catch (const llm::ApiError&) {
throw;
} catch (const std::exception& e) {
throw llm::ApiError("media", 502, "UNAVAILABLE", e.what());
}
}
FetchedAsset Downloader::fetch(const MediaAsset& asset) {
FetchedAsset out;
out.asset = asset;
if (!asset.data_base64.empty()) {
if (!security::crypto::Base64::decode(asset.data_base64, out.bytes))
throw llm::ApiError::invalid("media", "inline asset is not valid base64");
if (out.bytes.size() > MediaCaps::kMaxDownloadBytes)
throw llm::ApiError("media", 502, "BAD_RESPONSE", "inline asset over the byte cap");
out.content_type = asset.mime;
return out;
}
if (asset.url.empty()) throw llm::ApiError::invalid("media", "asset has neither url nor data");
out.bytes = fetch_url(asset.url, &out.content_type);
return out;
}
} // namespace molo::media
+38
View File
@@ -0,0 +1,38 @@
#pragma once
// Downloader: fetch an output (https URL or inline base64) as bytes under the
// download cap. Sends no credential: result URLs are public or pre-signed.
#include <memory>
#include <string>
#include "clients/client.h"
#include "vendors/media/common/job.h"
namespace molo::media {
struct FetchedAsset {
MediaAsset asset;
std::string bytes;
std::string content_type;
};
class Downloader {
public:
// Null transport = real HTTPS.
explicit Downloader(std::unique_ptr<clients::Transport> transport = nullptr);
// https only, 1..4096 chars, no userinfo, no whitespace or control chars.
// Throws ApiError::invalid("media", ...).
static void validate_url(const std::string& url);
// Decodes inline data or downloads the URL. Throws ApiError on a bad status
// (the body is never returned for a failure) or when the cap is exceeded.
FetchedAsset fetch(const MediaAsset& asset);
std::string fetch_url(const std::string& url, std::string* content_type = nullptr);
private:
clients::Client client_;
};
} // namespace molo::media
+145
View File
@@ -0,0 +1,145 @@
#include "vendors/media/common/job.h"
#include "llm/error.h"
#include "vendors/media/common/caps.h"
namespace molo::media {
using clients::body::Json;
const char* media_kind_name(MediaKind kind) {
switch (kind) {
case MediaKind::kImage: return "image";
case MediaKind::kVideo: return "video";
case MediaKind::kAudio: return "audio";
case MediaKind::kModel3d: return "model3d";
default: return "other";
}
}
const char* job_state_name(JobState state) {
switch (state) {
case JobState::kQueued: return "queued";
case JobState::kRunning: return "running";
case JobState::kSucceeded: return "succeeded";
case JobState::kFailed: return "failed";
default: return "cancelled";
}
}
namespace {
MediaKind kind_from(const std::string& s) {
if (s == "image") return MediaKind::kImage;
if (s == "video") return MediaKind::kVideo;
if (s == "audio") return MediaKind::kAudio;
if (s == "model3d") return MediaKind::kModel3d;
return MediaKind::kOther;
}
std::string str(const Json& doc, const char* key) {
const Json* v = doc.is_object() ? doc.find(key) : nullptr;
return v && v->is_string() ? v->as_string() : "";
}
Json assets_json(const std::vector<MediaAsset>& assets) {
Json a = Json::array();
for (const MediaAsset& x : assets) a.push(x.to_json());
return a;
}
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("media", why); }
void check_text(const std::string& what, const std::string& v, std::size_t cap) {
if (v.size() > cap) bad(what + " is over " + std::to_string(cap) + " characters");
}
} // namespace
Json MediaAsset::to_json() const {
Json o = Json::object();
o.set("kind", Json(media_kind_name(kind)));
if (!url.empty()) o.set("url", Json(url));
if (!data_base64.empty()) o.set("data_base64_bytes", Json(static_cast<double>(data_base64.size())));
if (!mime.empty()) o.set("mime", Json(mime));
if (!label.empty()) o.set("label", Json(label));
return o;
}
Json JobTicket::to_json() const {
Json o = Json::object();
o.set("provider", Json(provider));
o.set("id", Json(id));
o.set("model", Json(model));
o.set("kind", Json(media_kind_name(kind)));
o.set("immediate", Json(immediate));
o.set("assets", assets_json(assets));
if (!meta.is_null()) o.set("meta", meta);
return o;
}
JobTicket JobTicket::from_json(const Json& doc) {
if (!doc.is_object()) bad("ticket must be a JSON object");
JobTicket t;
t.provider = str(doc, "provider");
t.id = str(doc, "id");
t.model = str(doc, "model");
t.kind = kind_from(str(doc, "kind"));
if (t.provider.empty() || t.id.empty()) bad("ticket needs provider and id");
if (const Json* m = doc.find("meta"); m && m->is_object()) t.meta = *m;
if (const Json* im = doc.find("immediate"); im && im->is_bool()) t.immediate = im->as_bool();
// Only URL assets travel back: inline (base64) outputs are not echoed by
// to_json(), so an immediate ticket restored from JSON carries its URLs only.
if (const Json* a = doc.find("assets"); a && a->is_array())
for (const Json& x : a->as_array()) {
std::string url = str(x, "url");
if (url.empty() || t.assets.size() >= MediaCaps::kMaxAssetsPerJob) continue;
MediaAsset asset;
asset.url = url;
asset.kind = kind_from(str(x, "kind"));
asset.mime = str(x, "mime");
asset.label = str(x, "label");
t.assets.push_back(asset);
}
return t;
}
Json JobStatus::to_json() const {
Json o = Json::object();
o.set("state", Json(job_state_name(state)));
o.set("raw_state", Json(raw_state));
if (progress >= 0) o.set("progress", Json(progress));
if (!error.empty()) o.set("error", Json(error));
o.set("assets", assets_json(assets));
return o;
}
void ImageRequest::validate() const {
if (prompt.empty()) bad("prompt is required");
check_text("prompt", prompt, MediaCaps::kMaxPromptChars);
check_text("negative_prompt", negative_prompt, MediaCaps::kMaxPromptChars);
check_text("model", model, 256);
check_text("source_url", source_url, MediaCaps::kMaxUrlChars);
if (count < 1 || count > MediaCaps::kMaxImagesPerRequest)
bad("count must be 1.." + std::to_string(MediaCaps::kMaxImagesPerRequest));
if (width < 0 || width > MediaCaps::kMaxDimension || height < 0 || height > MediaCaps::kMaxDimension)
bad("width and height must be 0.." + std::to_string(MediaCaps::kMaxDimension));
if ((width == 0) != (height == 0)) bad("width and height go together");
if (seed < -1 || seed > 4294967295L) bad("seed must be -1 or 0..4294967295");
if (!extra.is_null() && !extra.is_object()) bad("extra must be a JSON object");
}
void VideoRequest::validate() const {
if (prompt.empty() && image_url.empty()) bad("prompt or image_url is required");
check_text("prompt", prompt, MediaCaps::kMaxPromptChars);
check_text("negative_prompt", negative_prompt, MediaCaps::kMaxPromptChars);
check_text("model", model, 256);
check_text("image_url", image_url, MediaCaps::kMaxUrlChars);
check_text("end_image_url", end_image_url, MediaCaps::kMaxUrlChars);
if (duration_seconds < 0 || duration_seconds > MediaCaps::kMaxVideoSeconds)
bad("duration_seconds must be 0.." + std::to_string(MediaCaps::kMaxVideoSeconds));
if (seed < -1 || seed > 4294967295L) bad("seed must be -1 or 0..4294967295");
if (!extra.is_null() && !extra.is_object()) bad("extra must be a JSON object");
}
} // namespace molo::media
+124
View File
@@ -0,0 +1,124 @@
#pragma once
// The neutral generation seam of package media: submit a request, poll the
// job, fetch the outputs. Every provider adapter implements ImageGenerator
// and / or VideoGenerator; callers (the demo, tests, other packages) code
// against these types only. Nothing here knows a provider's wire format.
//
// Money rule: submit() and cancel() take an explicit confirm flag with no
// default. Without it they throw before any network traffic.
#include <string>
#include <vector>
#include "clients/body/json.h"
namespace molo::media {
enum class MediaKind { kImage, kVideo, kAudio, kModel3d, kOther };
enum class JobState { kQueued, kRunning, kSucceeded, kFailed, kCancelled };
const char* media_kind_name(MediaKind kind);
const char* job_state_name(JobState state);
// One output. Exactly one of url / data_base64 is set.
struct MediaAsset {
MediaKind kind = MediaKind::kOther;
std::string url;
std::string data_base64;
std::string mime;
std::string label; // provider note: seed, finish reason, model
clients::body::Json to_json() const;
};
// Handle of a submitted job. `immediate` jobs (synchronous providers) carry
// their assets already; poll() answers from the ticket without a network call.
struct JobTicket {
std::string provider;
std::string id;
std::string model;
MediaKind kind = MediaKind::kImage;
bool immediate = false;
std::vector<MediaAsset> assets;
clients::body::Json meta; // provider extras the poll needs (endpoint, urls, cost)
clients::body::Json to_json() const;
// Rebuilds a ticket from to_json() output (the demo round-trips tickets).
static JobTicket from_json(const clients::body::Json& doc);
};
struct JobStatus {
JobState state = JobState::kQueued;
std::string raw_state; // the provider's own word
double progress = -1; // 0..1, -1 = unknown
std::string error;
std::vector<MediaAsset> assets;
clients::body::Json meta;
bool terminal() const { return state == JobState::kSucceeded || state == JobState::kFailed || state == JobState::kCancelled; }
clients::body::Json to_json() const;
};
struct ImageRequest {
std::string prompt;
std::string negative_prompt;
std::string model; // provider model / endpoint id; empty = adapter default
int width = 0; // 0 = provider default
int height = 0;
std::string aspect_ratio; // "16:9"; empty = default
int count = 1;
long seed = -1; // -1 = random
std::string source_url; // reference / source image for edit styles
clients::body::Json extra; // provider fields merged last, checked against the call table
// Throws llm::ApiError::invalid(vendor "media") on a bad value or cap breach.
void validate() const;
};
struct VideoRequest {
std::string prompt;
std::string negative_prompt;
std::string model;
int duration_seconds = 0; // 0 = provider default
std::string aspect_ratio;
std::string resolution; // "720p", "1080p"; empty = default
long seed = -1;
std::string image_url; // start frame for image-to-video
std::string end_image_url; // end frame
clients::body::Json extra;
void validate() const;
};
class MediaGenerator {
public:
virtual ~MediaGenerator() = default;
virtual std::string provider() const = 0;
virtual bool configured() const = 0;
// Model / endpoint ids the adapter knows by name (not a live catalogue).
virtual std::vector<std::string> models() const = 0;
// One status check. A failed job is a JobStatus with state kFailed, a
// transport or auth failure is an ApiError.
virtual JobStatus poll(const JobTicket& ticket) = 0;
// Cancels where the provider supports it; returns false (without any call)
// where it does not. Explicit confirm, no default.
virtual bool cancel(const JobTicket& ticket, bool confirm) = 0;
// Suggested first poll interval; JobRunner clamps it to the caps.
virtual int suggested_poll_ms() const { return 2000; }
};
class ImageGenerator : public MediaGenerator {
public:
// Exact request the submit would send, as JSON; no network, no credential.
virtual clients::body::Json plan(const ImageRequest& request) const = 0;
virtual JobTicket submit(const ImageRequest& request, bool confirm) = 0;
};
class VideoGenerator : public MediaGenerator {
public:
virtual clients::body::Json plan(const VideoRequest& request) const = 0;
virtual JobTicket submit(const VideoRequest& request, bool confirm) = 0;
};
} // namespace molo::media
+65
View File
@@ -0,0 +1,65 @@
#include "vendors/media/common/job_runner.h"
#include <algorithm>
#include <chrono>
#include <thread>
#include "llm/error.h"
#include "vendors/media/common/caps.h"
namespace molo::media {
void WaitPolicy::validate() const {
auto bad = [](const std::string& why) { throw llm::ApiError::invalid("media", why); };
if (first_interval_ms < 0 || first_interval_ms > MediaCaps::kMaxPollIntervalMs) bad("first_interval_ms out of range");
if (backoff < 1.0 || backoff > 3.0) bad("backoff must be 1.0..3.0");
if (max_interval_ms < MediaCaps::kMinPollIntervalMs || max_interval_ms > MediaCaps::kMaxPollIntervalMs)
bad("max_interval_ms must be " + std::to_string(MediaCaps::kMinPollIntervalMs) + ".." +
std::to_string(MediaCaps::kMaxPollIntervalMs));
if (max_attempts < 1 || max_attempts > MediaCaps::kMaxPollAttempts)
bad("max_attempts must be 1.." + std::to_string(MediaCaps::kMaxPollAttempts));
if (max_wait_seconds < 1 || max_wait_seconds > MediaCaps::kMaxWaitSeconds)
bad("max_wait_seconds must be 1.." + std::to_string(MediaCaps::kMaxWaitSeconds));
}
JobRunner::JobRunner(WaitPolicy policy, Sleeper sleeper, Clock clock)
: policy_(policy), sleeper_(std::move(sleeper)), clock_(std::move(clock)) {
policy_.validate();
if (!sleeper_)
sleeper_ = [](int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); };
if (!clock_)
clock_ = [] {
return static_cast<long>(std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count());
};
}
JobStatus JobRunner::wait(MediaGenerator& generator, const JobTicket& ticket) const {
if (ticket.immediate) {
JobStatus s;
s.state = JobState::kSucceeded;
s.raw_state = "immediate";
s.progress = 1;
s.assets = ticket.assets;
return s;
}
const long start = clock_();
const long deadline_ms = static_cast<long>(policy_.max_wait_seconds) * 1000L;
int interval = policy_.first_interval_ms > 0 ? policy_.first_interval_ms : generator.suggested_poll_ms();
interval = std::clamp(interval, MediaCaps::kMinPollIntervalMs, policy_.max_interval_ms);
for (int attempt = 1; attempt <= policy_.max_attempts; ++attempt) {
JobStatus s = generator.poll(ticket);
if (s.terminal()) return s;
if (clock_() - start + interval > deadline_ms) break;
sleeper_(interval);
interval = std::clamp(static_cast<int>(interval * policy_.backoff), MediaCaps::kMinPollIntervalMs,
policy_.max_interval_ms);
}
throw llm::ApiError("media", 504, "WAIT_EXPIRED",
generator.provider() + " job " + ticket.id + " is not finished after the poll caps (" +
std::to_string(policy_.max_attempts) + " polls, " + std::to_string(policy_.max_wait_seconds) +
" s); it was NOT cancelled, check it with poll");
}
} // namespace molo::media
+45
View File
@@ -0,0 +1,45 @@
#pragma once
// JobRunner: poll a job to a terminal state under hard caps. The caps are a
// floor on the interval (never hammer a provider) and ceilings on attempts
// and total wait (never hang). When a cap is hit the runner throws; it never
// cancels the job on its own (a human decides, the money may already be spent).
#include <functional>
#include "vendors/media/common/job.h"
namespace molo::media {
struct WaitPolicy {
int first_interval_ms = 0; // 0 = the generator's suggestion
double backoff = 1.5; // interval multiplier per attempt, 1.0 .. 3.0
int max_interval_ms = 15000; // <= MediaCaps::kMaxPollIntervalMs
int max_attempts = 120; // <= MediaCaps::kMaxPollAttempts
int max_wait_seconds = 900; // <= MediaCaps::kMaxWaitSeconds
// Throws ApiError::invalid when a value is outside the caps.
void validate() const;
};
class JobRunner {
public:
using Sleeper = std::function<void(int milliseconds)>;
using Clock = std::function<long()>; // monotonic milliseconds
// Null sleeper / clock use the real ones. Tests inject fakes.
explicit JobRunner(WaitPolicy policy = {}, Sleeper sleeper = nullptr, Clock clock = nullptr);
const WaitPolicy& policy() const { return policy_; }
// Immediate tickets answer without a poll. Otherwise polls until terminal.
// Throws ApiError (504, WAIT_EXPIRED) when attempts or wait are exhausted.
JobStatus wait(MediaGenerator& generator, const JobTicket& ticket) const;
private:
WaitPolicy policy_;
Sleeper sleeper_;
Clock clock_;
};
} // namespace molo::media
+171
View File
@@ -0,0 +1,171 @@
#include "vendors/media/common/media_service.h"
#include <cstring>
#include "clients/body/form.h"
#include "clients/url.h"
#include "llm/error.h"
#include "vendors/media/common/caps.h"
namespace molo::media {
namespace {
bool valid_header_name(const std::string& n) {
if (n.empty() || n.size() > 64) return false;
for (char c : n)
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-')) return false;
return true;
}
std::string field_text(const Json& v) {
if (v.is_string()) return v.as_string();
if (v.is_bool()) return v.as_bool() ? "true" : "false";
return v.dump(); // numbers, arrays, objects travel as JSON text
}
} // namespace
Json PreparedCall::summary() const {
Json o = Json::object();
o.set("vendor", Json(vendor));
o.set("key", Json(key));
o.set("method", Json(method));
o.set("target", Json(target));
o.set("effect", Json(CallTable::effect_name(effect)));
if (!content_type.empty()) o.set("content_type", Json(content_type.substr(0, content_type.find(';'))));
if (!preview.is_null()) o.set("body", preview);
return o;
}
MediaService::MediaService(std::shared_ptr<llm::ApiClient> client, const CallTable& table)
: llm::Service(std::move(client)), table_(table) {}
PreparedCall MediaService::prepare(const std::string& key, const CallArgs& args) const {
const std::string v = table_.vendor();
const CallRow& row = table_.get(key);
PreparedCall out;
out.vendor = v;
out.key = key;
out.method = row.http_method;
out.effect = CallTable::effect_of(row);
std::string target = table_.render_path(row, args.path);
if (args.query.size() > MediaCaps::kMaxQueryArgs) throw llm::ApiError::invalid(v, "too many query parameters");
std::string qs;
for (const auto& [name, value] : args.query) {
if (!table_.query_allows(row, name))
throw llm::ApiError::invalid(v, "call '" + key + "' does not accept query parameter '" + name + "'");
if (value.size() > MediaCaps::kMaxQueryValueChars) throw llm::ApiError::invalid(v, "query value too long: " + name);
qs += qs.empty() ? '?' : '&';
qs += clients::Url::encode_component(name) + "=" + clients::Url::encode_component(value);
}
out.target = target + qs;
if (args.headers.size() > MediaCaps::kMaxHeaders) throw llm::ApiError::invalid(v, "too many headers");
for (const auto& [k, val] : args.headers) {
if (!valid_header_name(k) || val.size() > MediaCaps::kMaxHeaderValueChars ||
val.find_first_of("\r\n") != std::string::npos)
throw llm::ApiError::invalid(v, "bad header '" + k + "'");
out.headers[k] = val;
}
const std::string kind = row.body_kind;
const bool has_body = !args.body.is_null() && !(args.body.is_object() && args.body.as_object().empty());
if (kind == "none") {
if (has_body || !args.files.empty()) throw llm::ApiError::invalid(v, "call '" + key + "' takes no body");
return out;
}
if (kind == "form") throw llm::ApiError::invalid(v, "form bodies are not supported by this package");
if (!args.body.is_null() && !args.body.is_object()) throw llm::ApiError::invalid(v, "body must be a JSON object");
Json fields = args.body.is_null() ? Json::object() : args.body;
if (fields.as_object().size() > MediaCaps::kMaxBodyFields) throw llm::ApiError::invalid(v, "too many body fields");
for (const auto& [name, value] : fields.as_object()) {
if (!table_.body_allows(row, name))
throw llm::ApiError::invalid(v, "call '" + key + "' does not accept body field '" + name + "'");
if (value.is_string() && value.as_string().size() > MediaCaps::kMaxFieldChars)
throw llm::ApiError::invalid(v, "field '" + name + "' is over the size cap");
}
const bool multipart = kind == "multipart" || (kind == "either" && !args.files.empty());
if (!args.files.empty() && !multipart) throw llm::ApiError::invalid(v, "call '" + key + "' does not take files");
if (args.files.size() > MediaCaps::kMaxFiles) throw llm::ApiError::invalid(v, "too many files");
if (!multipart) {
out.body = fields.dump();
out.content_type = "application/json";
out.preview = fields;
return out;
}
clients::body::MultipartBody form;
Json preview = Json::object();
for (const auto& [name, value] : fields.as_object()) {
form.add_text(name, field_text(value));
preview.set(name, value);
}
Json files = Json::array();
for (const FilePart& f : args.files) {
if (!table_.body_allows(row, f.field))
throw llm::ApiError::invalid(v, "call '" + key + "' does not accept file field '" + f.field + "'");
if (f.data.empty()) throw llm::ApiError::invalid(v, "file '" + f.field + "' is empty");
if (f.data.size() > MediaCaps::kMaxFileBytes) throw llm::ApiError::invalid(v, "file '" + f.field + "' is over the size cap");
if (f.filename.find_first_of("\"\r\n") != std::string::npos || f.content_type.find_first_of("\r\n") != std::string::npos)
throw llm::ApiError::invalid(v, "bad file name or content type");
form.add_file(f.field, f.filename.empty() ? f.field : f.filename,
f.content_type.empty() ? "application/octet-stream" : f.content_type, f.data);
Json info = Json::object();
info.set("field", Json(f.field));
info.set("filename", Json(f.filename));
info.set("bytes", Json(static_cast<double>(f.data.size())));
files.push(info);
}
if (!args.files.empty()) preview.set("_files", files);
out.body = form.encode();
out.content_type = form.content_type();
out.preview = preview;
return out;
}
llm::RawResponse MediaService::execute(const PreparedCall& call, bool confirmed) {
if (call.effect != Effect::kRead && !confirmed)
throw llm::ApiError::invalid(table_.vendor(), "call '" + call.key + "' has effect " +
CallTable::effect_name(call.effect) +
" and needs explicit confirmation");
const CallRow& row = table_.get(call.key);
const bool has_body = !call.body.empty();
return client_->request_raw(CallTable::method_of(row), call.target, has_body ? &call.body : nullptr,
call.content_type, call.headers);
}
llm::RawResponse MediaService::call_raw(const std::string& key, const CallArgs& args, bool confirmed) {
return execute(prepare(key, args), confirmed);
}
Json MediaService::parse_json(const std::string& vendor, const llm::RawResponse& raw) {
if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object();
try {
return Json::parse(raw.body);
} catch (const std::exception& e) {
throw llm::ApiError(vendor, 502, "BAD_RESPONSE", std::string("not JSON: ") + e.what());
}
}
Json MediaService::call_json(const std::string& key, const CallArgs& args, bool confirmed) {
return parse_json(table_.vendor(), call_raw(key, args, confirmed));
}
const Json* MediaService::member(const Json& doc, const std::string& key) {
return doc.is_object() ? doc.find(key) : nullptr;
}
std::string MediaService::text(const Json& doc, const std::string& key, const std::string& fallback) {
const Json* v = member(doc, key);
if (v && v->is_string()) return v->as_string();
if (v && v->is_number()) return v->dump();
return fallback;
}
} // namespace molo::media
+78
View File
@@ -0,0 +1,78 @@
#pragma once
// MediaService: the validated path from "I want call X with these arguments"
// to bytes on the wire. prepare() is pure (no network): it checks the key,
// path arguments, query parameters, body fields and files against the call
// table row and builds the exact request. execute() sends it and refuses any
// call whose effect is not "read" unless the caller confirms explicitly.
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "clients/body/json.h"
#include "llm/api_client.h"
#include "vendors/media/common/call_table.h"
namespace molo::media {
using Json = clients::body::Json;
struct FilePart {
std::string field;
std::string filename;
std::string content_type;
std::string data;
};
struct CallArgs {
std::map<std::string, std::string> path;
std::map<std::string, std::string> query;
Json body; // null = no body; otherwise a JSON object
std::vector<FilePart> files; // multipart rows only
llm::Headers headers; // extra request headers (Accept, Prefer, ...)
};
struct PreparedCall {
std::string vendor;
std::string key;
std::string method; // GET POST ...
std::string target; // path plus query string
std::string body; // exact bytes ("" = none)
std::string content_type; // "" when no body
llm::Headers headers;
Effect effect = Effect::kRead;
Json preview; // body as JSON, or the multipart part list
// What a dry run shows: method, target, effect, content type, body preview.
Json summary() const;
};
class MediaService : public llm::Service {
public:
MediaService(std::shared_ptr<llm::ApiClient> client, const CallTable& table);
const CallTable& table() const { return table_; }
const std::string& vendor() const { return table_.vendor(); }
PreparedCall prepare(const std::string& key, const CallArgs& args = {}) const;
// Non-read effects throw ApiError::invalid unless confirmed is true. The
// missing-credential error surfaces here, before any network traffic.
llm::RawResponse execute(const PreparedCall& call, bool confirmed);
// prepare + execute + parse. Empty body -> {}. Non-JSON body -> ApiError.
Json call_json(const std::string& key, const CallArgs& args = {}, bool confirmed = false);
llm::RawResponse call_raw(const std::string& key, const CallArgs& args = {}, bool confirmed = false);
// Helpers for typed wrappers.
static Json parse_json(const std::string& vendor, const llm::RawResponse& raw);
static const Json* member(const Json& doc, const std::string& key);
static std::string text(const Json& doc, const std::string& key, const std::string& fallback = "");
private:
const CallTable& table_;
};
} // namespace molo::media
+119
View File
@@ -0,0 +1,119 @@
#include "vendors/media/common/output_scanner.h"
#include <algorithm>
#include <set>
#include "vendors/media/common/caps.h"
namespace molo::media {
using clients::body::Json;
namespace {
std::string lower_ext(const std::string& name) {
std::string n = name.substr(0, name.find_first_of("?#"));
std::size_t dot = n.rfind('.');
if (dot == std::string::npos || n.size() - dot > 8) return "";
std::string e = n.substr(dot + 1);
std::transform(e.begin(), e.end(), e.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return e;
}
struct Walk {
std::vector<MediaAsset> out;
std::set<std::string> seen;
std::size_t nodes = 0;
MediaKind fallback;
void add(const std::string& s, const std::string& key) {
if (out.size() >= MediaCaps::kMaxAssetsPerJob) return;
bool http = s.rfind("https://", 0) == 0 || s.rfind("http://", 0) == 0;
bool data = s.rfind("data:", 0) == 0;
if (!http && !data) return;
if (s.size() > MediaCaps::kMaxUrlChars && !data) return;
if (!seen.insert(s).second) return;
MediaAsset a;
if (data) {
std::size_t comma = s.find(',');
std::size_t semi = s.find(';');
if (comma == std::string::npos) return;
a.mime = s.substr(5, std::min(semi, comma) - 5);
a.data_base64 = s.substr(comma + 1);
a.kind = fallback;
if (a.mime.rfind("video/", 0) == 0) a.kind = MediaKind::kVideo;
else if (a.mime.rfind("image/", 0) == 0) a.kind = MediaKind::kImage;
else if (a.mime.rfind("audio/", 0) == 0) a.kind = MediaKind::kAudio;
} else {
a.url = s;
a.kind = OutputScanner::kind_from_name(s, fallback);
if (a.kind == fallback) {
if (key.find("video") != std::string::npos) a.kind = MediaKind::kVideo;
else if (key.find("image") != std::string::npos) a.kind = MediaKind::kImage;
else if (key.find("audio") != std::string::npos) a.kind = MediaKind::kAudio;
}
a.mime = OutputScanner::mime_from_name(s);
}
out.push_back(a);
}
void go(const Json& v, const std::string& key, std::size_t depth) {
if (depth > MediaCaps::kMaxScanDepth || ++nodes > MediaCaps::kMaxScanNodes) return;
if (v.is_string()) {
add(v.as_string(), key);
} else if (v.is_array()) {
for (const Json& x : v.as_array()) go(x, key, depth + 1);
} else if (v.is_object()) {
for (const auto& [k, x] : v.as_object()) {
if (OutputScanner::skipped_key(k)) continue;
go(x, k, depth + 1);
}
}
}
};
} // namespace
bool OutputScanner::skipped_key(const std::string& key) {
static const std::set<std::string> skip = {
"input", "request", "urls", "logs", "metrics", "webhook", "webhook_url", "callback_url",
"status_url", "cancel_url", "response_url", "polling_url", "get", "cancel", "stream", "web",
"params", "parameters", "meta", "links", "self", "thumbnail_url", "model_url"};
return skip.count(key) > 0;
}
MediaKind OutputScanner::kind_from_name(const std::string& name, MediaKind fallback) {
std::string e = lower_ext(name);
if (e == "mp4" || e == "webm" || e == "mov" || e == "mkv" || e == "m4v") return MediaKind::kVideo;
if (e == "png" || e == "jpg" || e == "jpeg" || e == "webp" || e == "gif" || e == "svg" || e == "avif") return MediaKind::kImage;
if (e == "mp3" || e == "wav" || e == "flac" || e == "ogg" || e == "m4a" || e == "aac") return MediaKind::kAudio;
if (e == "glb" || e == "gltf" || e == "obj" || e == "ply" || e == "usdz" || e == "fbx" || e == "stl") return MediaKind::kModel3d;
return fallback;
}
std::string OutputScanner::mime_from_name(const std::string& name) {
std::string e = lower_ext(name);
if (e == "mp4" || e == "m4v") return "video/mp4";
if (e == "webm") return "video/webm";
if (e == "mov") return "video/quicktime";
if (e == "png") return "image/png";
if (e == "jpg" || e == "jpeg") return "image/jpeg";
if (e == "webp") return "image/webp";
if (e == "gif") return "image/gif";
if (e == "svg") return "image/svg+xml";
if (e == "mp3") return "audio/mpeg";
if (e == "wav") return "audio/wav";
if (e == "flac") return "audio/flac";
if (e == "ogg") return "audio/ogg";
if (e == "glb") return "model/gltf-binary";
return "";
}
std::vector<MediaAsset> OutputScanner::scan(const Json& doc, MediaKind fallback) {
Walk w;
w.fallback = fallback;
w.go(doc, "", 0);
return std::move(w.out);
}
} // namespace molo::media
+29
View File
@@ -0,0 +1,29 @@
#pragma once
// OutputScanner: finds result URLs / inline data in a provider's result JSON.
// Providers shape results differently (fal: images[].url, Replicate: output as
// string or array, Runway: output[]), so the adapters share one bounded walk
// instead of seven hand-rolled ones. Keys that carry request echoes or
// control links (input, request, urls, webhook, status_url, ...) are skipped.
#include <vector>
#include "vendors/media/common/job.h"
namespace molo::media {
class OutputScanner {
public:
// Walks `doc` (depth and node caps in MediaCaps) and returns de-duplicated
// assets in document order, at most MediaCaps::kMaxAssetsPerJob.
static std::vector<MediaAsset> scan(const clients::body::Json& doc, MediaKind fallback);
// Kind from a URL / file name extension, else `fallback`.
static MediaKind kind_from_name(const std::string& name, MediaKind fallback);
// MIME for a kind + extension, "" when unknown.
static std::string mime_from_name(const std::string& name);
// True for keys whose subtree is request echo or control metadata.
static bool skipped_key(const std::string& key);
};
} // namespace molo::media
+195
View File
@@ -0,0 +1,195 @@
#include "vendors/media/common/registry.h"
#include <ctime>
#include "llm/config.h"
#include "llm/error.h"
#include "vendors/media/bfl/bfl.h"
#include "vendors/media/fal/fal.h"
#include "vendors/media/higgsfield/higgsfield.h"
#include "vendors/media/ideogram/ideogram.h"
#include "vendors/media/kling/kling.h"
#include "vendors/media/leonardo/leonardo.h"
#include "vendors/media/luma/luma.h"
#include "vendors/media/minimax/minimax.h"
#include "vendors/media/recraft/recraft.h"
#include "vendors/media/replicate/replicate.h"
#include "vendors/media/runway/runway.h"
#include "vendors/media/stability/stability.h"
namespace molo::media {
const std::vector<ProviderInfo>& MediaRegistry::providers() {
static const std::vector<ProviderInfo> list = {
{"stability", "Stability AI", {"STABILITY_API_KEY"}, true, false, Evidence::kOpenApi,
"https://platform.stability.ai/account/keys", "v1.user.balance",
"v2beta image (generate, edit, upscale, control), 3D, audio; v1 engines. No video route in the published spec."},
{"bfl", "Black Forest Labs", {"BFL_API_KEY"}, true, true, Evidence::kOpenApi,
"https://dashboard.bfl.ai/", "credits", "FLUX models and tools, polling via get_result, fine-tunes."},
{"replicate", "Replicate", {"REPLICATE_API_TOKEN"}, true, true, Evidence::kOpenApi,
"https://replicate.com/account/api-tokens", "account.get",
"Predictions, models, deployments, files, trainings, webhook verification. Any model by owner/name."},
{"fal", "fal", {"FAL_KEY"}, true, true, Evidence::kOpenApi, "https://fal.ai/dashboard/keys",
"platform.models.pricing", "Queue API (any endpoint id) and the model-related Platform API."},
{"ideogram", "Ideogram", {"IDEOGRAM_API_KEY"}, true, true, Evidence::kOpenApi,
"https://ideogram.ai/manage-api", "account.usage",
"v2 image, SVG, video and ad tools; dry_run=true gives a free price quote."},
{"recraft", "Recraft", {"RECRAFT_API_TOKEN"}, true, false, Evidence::kOpenApi,
"https://www.recraft.ai/profile/api", "users.me", "Generation, vector, background, upscale, styles."},
{"leonardo", "Leonardo.Ai", {"LEONARDO_API_KEY"}, true, true, Evidence::kDocs,
"https://app.leonardo.ai/api-access", "v2.models",
"v2 generations for image, video, audio and 3D models. v1 reference pages are gone from the docs."},
{"runway", "Runway", {"RUNWAY_API_KEY", "RUNWAYML_API_SECRET"}, true, true, Evidence::kSdk,
"https://dev.runwayml.com/", "organization.get",
"Text / image / video generation, tasks, uploads, avatars, voices, documents, routers, workflows."},
{"luma", "Luma Dream Machine", {"LUMA_API_KEY"}, true, true, Evidence::kSdk,
"https://lumalabs.ai/api/keys", "credits.get", "Ray video, Photon image, reframe, modify, upscale, audio."},
{"minimax", "MiniMax Hailuo", {"MINIMAX_API_KEY"}, false, true, Evidence::kDocs,
"https://platform.minimax.io/user-center/basic-information/interface-key", "",
"Hailuo video v1 and v2, file retrieval. No free account read exists."},
{"kling", "Kling AI", {"KLING_ACCESS_KEY", "KLING_SECRET_KEY"}, false, true, Evidence::kProbeOnly,
"https://app.klingai.com/global/dev/api-key", "",
"Official API with JWT auth; docs are a JS app with no machine-readable spec: paths are unverified."},
{"higgsfield", "Higgsfield", {"HF_API_KEY_ID", "HF_API_KEY_SECRET"}, true, true, Evidence::kOpenApi,
"https://cloud.higgsfield.ai/", "",
"One lifecycle for every model path; the OpenAPI document is explicitly supplementary."},
};
return list;
}
const std::vector<UnsupportedProduct>& MediaRegistry::unsupported() {
static const std::vector<UnsupportedProduct> list = {
{"Midjourney",
"No public API. Midjourney documents only its web and Discord products; third-party relays are unofficial "
"and violate its terms, so none is implemented."},
{"Pika",
"pika.art/api advertises a membership API that resells other vendors' models, but no endpoint, schema or "
"authentication document was found (2026-10-25), so nothing is implemented. Pika models are reachable "
"through fal."},
};
return list;
}
const ProviderInfo* MediaRegistry::find(const std::string& id) {
for (const ProviderInfo& p : providers())
if (p.id == id) return &p;
return nullptr;
}
const char* MediaRegistry::evidence_name(Evidence e) {
switch (e) {
case Evidence::kOpenApi: return "openapi";
case Evidence::kSdk: return "sdk";
case Evidence::kDocs: return "docs";
default: return "probe-only";
}
}
const CallTable& MediaRegistry::table(const std::string& id) {
if (id == "stability") return StabilityCalls::table();
if (id == "bfl") return BflCalls::table();
if (id == "replicate") return ReplicateCalls::table();
if (id == "fal") return FalCalls::table();
if (id == "ideogram") return IdeogramCalls::table();
if (id == "recraft") return RecraftCalls::table();
if (id == "leonardo") return LeonardoCalls::table();
if (id == "runway") return RunwayCalls::table();
if (id == "luma") return LumaCalls::table();
if (id == "minimax") return MinimaxCalls::table();
if (id == "kling") return KlingCalls::table();
if (id == "higgsfield") return HiggsfieldCalls::table();
throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
}
bool MediaRegistry::configured(const std::string& id) {
const ProviderInfo* p = find(id);
if (!p) throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
if (id == "runway") return !llm::Config::get("RUNWAY_API_KEY").empty() || !llm::Config::get("RUNWAYML_API_SECRET").empty();
for (const std::string& e : p->env)
if (llm::Config::get(e.c_str()).empty()) return false;
return true;
}
std::shared_ptr<MediaService> MediaRegistry::service(const std::string& id, const std::string& key) {
std::string slot = id;
if (id == "fal") slot = key.rfind("platform.", 0) == 0 ? "fal.platform" : "fal.queue";
auto it = services_.find(slot);
if (it != services_.end()) return it->second;
std::shared_ptr<MediaService> s;
if (id == "stability") s = StabilityClient::from_env();
else if (id == "bfl") s = BflClient::from_env();
else if (id == "replicate") s = ReplicateClient::from_env();
else if (id == "fal") s = slot == "fal.platform" ? std::shared_ptr<MediaService>(FalPlatform::from_env())
: std::shared_ptr<MediaService>(FalQueue::from_env());
else if (id == "ideogram") s = IdeogramClient::from_env();
else if (id == "recraft") s = RecraftClient::from_env();
else if (id == "leonardo") s = LeonardoClient::from_env();
else if (id == "runway") s = RunwayClient::from_env();
else if (id == "luma") s = LumaClient::from_env();
else if (id == "minimax") s = MinimaxClient::from_env();
else if (id == "kling") s = KlingClient::from_env();
else if (id == "higgsfield") s = HiggsfieldClient::from_env();
else throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
services_[slot] = s;
return s;
}
std::shared_ptr<ImageGenerator> MediaRegistry::image(const std::string& id) {
auto it = images_.find(id);
if (it != images_.end()) return it->second;
std::shared_ptr<ImageGenerator> g;
if (id == "stability") g = std::make_shared<StabilityImageGenerator>(std::static_pointer_cast<StabilityClient>(service(id)));
else if (id == "bfl") g = std::make_shared<BflImageGenerator>(std::static_pointer_cast<BflClient>(service(id)));
else if (id == "replicate") g = std::make_shared<ReplicateImageGenerator>(std::static_pointer_cast<ReplicateClient>(service(id)));
else if (id == "fal") g = std::make_shared<FalImageGenerator>(std::static_pointer_cast<FalQueue>(service(id)));
else if (id == "ideogram") g = std::make_shared<IdeogramImageGenerator>(std::static_pointer_cast<IdeogramClient>(service(id)));
else if (id == "recraft") g = std::make_shared<RecraftImageGenerator>(std::static_pointer_cast<RecraftClient>(service(id)));
else if (id == "leonardo") g = std::make_shared<LeonardoImageGenerator>(std::static_pointer_cast<LeonardoClient>(service(id)));
else if (id == "runway") g = std::make_shared<RunwayImageGenerator>(std::static_pointer_cast<RunwayClient>(service(id)));
else if (id == "luma") g = std::make_shared<LumaImageGenerator>(std::static_pointer_cast<LumaClient>(service(id)));
else if (id == "higgsfield") g = std::make_shared<HiggsfieldImageGenerator>(std::static_pointer_cast<HiggsfieldClient>(service(id)));
else throw llm::ApiError::invalid("media", "provider '" + id + "' has no image generator");
images_[id] = g;
return g;
}
std::shared_ptr<VideoGenerator> MediaRegistry::video(const std::string& id) {
auto it = videos_.find(id);
if (it != videos_.end()) return it->second;
std::shared_ptr<VideoGenerator> g;
if (id == "bfl") g = std::make_shared<BflVideoGenerator>(std::static_pointer_cast<BflClient>(service(id)));
else if (id == "replicate") g = std::make_shared<ReplicateVideoGenerator>(std::static_pointer_cast<ReplicateClient>(service(id)));
else if (id == "fal") g = std::make_shared<FalVideoGenerator>(std::static_pointer_cast<FalQueue>(service(id)));
else if (id == "ideogram") g = std::make_shared<IdeogramVideoGenerator>(std::static_pointer_cast<IdeogramClient>(service(id)));
else if (id == "leonardo") g = std::make_shared<LeonardoVideoGenerator>(std::static_pointer_cast<LeonardoClient>(service(id)));
else if (id == "runway") g = std::make_shared<RunwayVideoGenerator>(std::static_pointer_cast<RunwayClient>(service(id)));
else if (id == "luma") g = std::make_shared<LumaVideoGenerator>(std::static_pointer_cast<LumaClient>(service(id)));
else if (id == "minimax") g = std::make_shared<MinimaxVideoGenerator>(std::static_pointer_cast<MinimaxClient>(service(id)));
else if (id == "kling") g = std::make_shared<KlingVideoGenerator>(std::static_pointer_cast<KlingClient>(service(id)));
else if (id == "higgsfield") g = std::make_shared<HiggsfieldVideoGenerator>(std::static_pointer_cast<HiggsfieldClient>(service(id)));
else throw llm::ApiError::invalid("media", "provider '" + id + "' has no video generator");
videos_[id] = g;
return g;
}
Json MediaRegistry::account(const std::string& id) {
const ProviderInfo* p = find(id);
if (!p) throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
if (p->account_key.empty())
throw llm::ApiError::invalid("media", id + " has no free account read; use a generation poll to prove the credential");
std::shared_ptr<MediaService> s = service(id, p->account_key);
CallArgs args;
if (id == "fal") args.query["endpoint_id"] = "fal-ai/flux/dev";
if (id == "stability") args.headers["Accept"] = "application/json";
if (id == "ideogram") { // usage requires start_time (spec: required, date-time)
std::time_t t = std::time(nullptr) - 24 * 3600;
std::tm tm{};
gmtime_r(&t, &tm);
char buf[32];
std::strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%SZ", &tm);
args.query["start_time"] = buf;
}
return s->call_json(p->account_key, args);
}
} // namespace molo::media
+72
View File
@@ -0,0 +1,72 @@
#pragma once
// MediaRegistry: the one list of providers in package media, what each one
// can do, which environment variables it needs and how well its API surface
// is verified. Builds generators and call services from the environment on
// demand and caches them. Products without a public API are listed too, with
// the reason, so nothing is silently missing.
#include <map>
#include <memory>
#include <string>
#include <vector>
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
namespace molo::media {
// How the provider's call table was checked (see each SPEC).
enum class Evidence {
kOpenApi, // vendor-published OpenAPI document, checked by tools/check_spec.py
kSdk, // official SDK source, checked by tools/check_spec.py
kDocs, // vendor doc pages, checked by tools/check_spec.py
kProbeOnly // no machine-readable source: community sources plus live probes
};
struct ProviderInfo {
std::string id;
std::string name;
std::vector<std::string> env; // every variable the credential needs
bool image = false;
bool video = false;
Evidence evidence = Evidence::kOpenApi;
std::string console_url; // where the key is created
std::string account_key; // table key of a free read that proves the credential ("" = none)
std::string note;
};
struct UnsupportedProduct {
std::string name;
std::string reason;
};
class MediaRegistry {
public:
static const std::vector<ProviderInfo>& providers();
static const std::vector<UnsupportedProduct>& unsupported();
static const ProviderInfo* find(const std::string& id);
static const char* evidence_name(Evidence evidence);
// Table of one provider (no credential needed). Throws ApiError::invalid.
static const CallTable& table(const std::string& id);
// True when every env variable the provider needs is set (llm::Config).
static bool configured(const std::string& id);
// Built from the environment on first use, then cached for this registry.
// `key` picks the host for providers with two (fal: queue.* / platform.*).
std::shared_ptr<MediaService> service(const std::string& id, const std::string& key = "");
std::shared_ptr<ImageGenerator> image(const std::string& id);
std::shared_ptr<VideoGenerator> video(const std::string& id);
// Free read that proves the credential works (balance, account, models).
// Returns the provider's JSON. Throws NotConfiguredError without a key.
Json account(const std::string& id);
private:
std::map<std::string, std::shared_ptr<MediaService>> services_;
std::map<std::string, std::shared_ptr<ImageGenerator>> images_;
std::map<std::string, std::shared_ptr<VideoGenerator>> videos_;
};
} // namespace molo::media
+41
View File
@@ -0,0 +1,41 @@
#include "vendors/media/fal/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"queue.submit", "POST", "/{owner}/{app}", "submitQueueRequest", "json", "*", "fal_webhook,fal_max_queue_length", "paid"},
{"queue.submit_path", "POST", "/{owner}/{app}/{path}", "submitQueueRequestWithPath", "json", "*", "fal_webhook,fal_max_queue_length", "paid"},
{"queue.status", "GET", "/{owner}/{app}/requests/{request_id}/status", "getQueueRequestStatus", "none", "", "logs", "read"},
{"queue.result", "GET", "/{owner}/{app}/requests/{request_id}", "getQueueRequestResult", "none", "", "", "read"},
{"queue.cancel", "PUT", "/{owner}/{app}/requests/{request_id}/cancel", "cancelQueueRequest", "none", "", "", "write"},
{"platform.models.insights", "GET", "/v1/models/insights", "getModelInsights", "none", "", "endpoint_id,q,limit", "read"},
{"platform.models.search", "GET", "/v1/models", "getModels", "none", "", "limit,cursor,endpoint_id,q,category,status,sort,expand", "read"},
{"platform.models.pricing", "GET", "/v1/models/pricing", "getPricing", "none", "", "endpoint_id", "read"},
{"platform.models.estimate", "POST", "/v1/models/pricing/estimate", "estimatePricing", "json", "estimate_type,endpoints", "", "read"},
{"platform.models.usage", "GET", "/v1/models/usage", "getUsage", "none", "", "limit,cursor,start,end,timezone,timeframe,bound_to_timeframe,source,endpoint_id,api_key_id,login_username,tag,expand", "read"},
{"platform.models.analytics", "GET", "/v1/models/analytics", "getAnalytics", "none", "", "limit,cursor,start,end,timezone,timeframe,bound_to_timeframe,endpoint_id,expand", "read"},
{"platform.models.billing_events", "GET", "/v1/models/billing-events", "getBillingEvents", "none", "", "limit,cursor,start,end,source,endpoint_id,request_id,tag,api_key_id,login_username,expand", "read"},
{"platform.models.delete_payloads", "DELETE", "/v1/models/requests/{request_id}/payloads", "deleteRequestPayloads", "none", "", "", "delete"},
{"platform.models.requests_by_endpoint", "GET", "/v1/models/requests/by-endpoint", "listRequestsByEndpoint", "none", "", "limit,cursor,endpoint_id,start,end,status,request_id,expand,sort_by", "read"},
{"platform.models.requests_search", "GET", "/v1/models/requests/search", "searchRequests", "none", "", "limit,cursor,query,image_url,video_url,endpoint_id,endpoint,exclude_api_requests,only_api_requests,min_similarity", "read"},
{"platform.meta", "GET", "/v1/meta", "getMeta", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"QueueStatus", "status,request_id,response_url,status_url,cancel_url,logs,metrics,queue_position"},
};
// FIELDS-END
const CallTable& FalCalls::table() {
static const CallTable t("fal", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t FalCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* FalCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct FalCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+260
View File
@@ -0,0 +1,260 @@
#include "vendors/media/fal/fal.h"
#include <algorithm>
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/caps.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs ids(const FalEndpoint& e, const std::string& request_id = "") {
CallArgs a;
a.path["owner"] = e.owner;
a.path["app"] = e.app;
if (!request_id.empty()) a.path["request_id"] = request_id;
return a;
}
} // namespace
FalEndpoint FalEndpoint::parse(const std::string& id) {
std::vector<std::string> parts;
std::size_t start = 0;
while (true) {
std::size_t slash = id.find('/', start);
parts.push_back(id.substr(start, slash == std::string::npos ? std::string::npos : slash - start));
if (slash == std::string::npos) break;
start = slash + 1;
}
if (parts.size() < 2 || parts.size() > 5)
throw llm::ApiError::invalid("fal", "endpoint must look like owner/app or owner/app/path");
for (const std::string& p : parts)
if (!CallTable::valid_segment(p)) throw llm::ApiError::invalid("fal", "bad endpoint id");
FalEndpoint e;
e.owner = parts[0];
e.app = parts[1];
// Everything below owner/app is the {path} of the submit_path row; the
// call table accepts several "/"-joined segments for it.
for (std::size_t i = 2; i < parts.size(); ++i) e.path += (i > 2 ? "/" : "") + parts[i];
return e;
}
PreparedCall FalQueue::plan(const FalEndpoint& e, const Json& body, const FalSubmitOptions& o) const {
CallArgs a = ids(e);
a.body = body;
if (!e.path.empty()) {
a.path["path"] = e.path;
}
if (!o.webhook.empty()) a.query["fal_webhook"] = o.webhook;
if (o.max_queue_length > 0) a.query["fal_max_queue_length"] = std::to_string(o.max_queue_length);
if (!o.priority.empty()) {
if (o.priority != "low" && o.priority != "normal" && o.priority != "high")
throw llm::ApiError::invalid("fal", "priority must be low, normal or high");
a.headers["X-Fal-Queue-Priority"] = o.priority;
}
return prepare(e.path.empty() ? "queue.submit" : "queue.submit_path", a);
}
Json FalQueue::submit(const FalEndpoint& e, const Json& body, const FalSubmitOptions& o, bool confirm) {
return parse_json("fal", execute(plan(e, body, o), confirm));
}
Json FalQueue::status(const FalEndpoint& e, const std::string& request_id, bool logs) {
CallArgs a = ids(e, request_id);
if (logs) a.query["logs"] = "1";
return call_json("queue.status", a);
}
Json FalQueue::result(const FalEndpoint& e, const std::string& request_id) {
return call_json("queue.result", ids(e, request_id));
}
Json FalQueue::cancel(const FalEndpoint& e, const std::string& request_id, bool confirm) {
return call_json("queue.cancel", ids(e, request_id), confirm);
}
// ---------------------------------------------------------------- platform
Json FalPlatform::models(const std::string& query, const std::string& category, std::size_t max_pages,
std::size_t max_items) {
if (max_pages < 1 || max_pages > MediaCaps::kMaxPages) throw llm::ApiError::invalid("fal", "max_pages out of range");
if (max_items < 1 || max_items > MediaCaps::kMaxItemsReturned) throw llm::ApiError::invalid("fal", "max_items out of range");
Json all = Json::array();
std::string cursor;
bool more = false;
std::size_t pages = 0;
while (pages < max_pages) {
CallArgs a;
a.query["limit"] = std::to_string(std::min<std::size_t>(max_items - all.as_array().size(), 100));
if (!query.empty()) a.query["q"] = query;
if (!category.empty()) a.query["category"] = category;
if (!cursor.empty()) a.query["cursor"] = cursor;
Json page = call_json("platform.models.search", a);
++pages;
if (const Json* m = page.find("models"); m && m->is_array())
for (const Json& x : m->as_array())
if (all.as_array().size() < max_items) all.push(x);
const Json* hm = page.find("has_more");
more = hm && hm->is_bool() && hm->as_bool();
cursor = AdapterSupport::str(page, "next_cursor");
if (!more || cursor.empty() || all.as_array().size() >= max_items) break;
}
Json out = Json::object();
out.set("models", all);
out.set("pages", Json(static_cast<double>(pages)));
out.set("truncated", Json(more));
return out;
}
Json FalPlatform::pricing(const std::string& endpoint_id) {
CallArgs a;
a.query["endpoint_id"] = endpoint_id;
return call_json("platform.models.pricing", a);
}
Json FalPlatform::estimate(const Json& body) {
CallArgs a;
a.body = body;
return call_json("platform.models.estimate", a);
}
Json FalPlatform::usage(const std::map<std::string, std::string>& query) {
CallArgs a;
a.query = query;
return call_json("platform.models.usage", a);
}
Json FalPlatform::analytics(const std::map<std::string, std::string>& query) {
CallArgs a;
a.query = query;
return call_json("platform.models.analytics", a);
}
Json FalPlatform::meta() { return call_json("platform.meta"); }
Json FalPlatform::delete_payloads(const std::string& request_id, bool confirm) {
CallArgs a;
a.path["request_id"] = request_id;
return call_json("platform.models.delete_payloads", a, confirm);
}
// -------------------------------------------------------------------- jobs
JobTicket FalJobs::ticket_from(const Json& reply, const FalEndpoint& e, MediaKind kind) {
JobTicket t;
t.provider = "fal";
t.model = e.full();
t.kind = kind;
t.id = AdapterSupport::str(reply, "request_id");
if (t.id.empty()) throw llm::ApiError("fal", 502, "BAD_RESPONSE", "no request_id in the submit reply");
t.meta = Json::object();
t.meta.set("endpoint", Json(e.full()));
return t;
}
JobStatus FalJobs::poll(FalQueue& queue, const JobTicket& ticket) {
FalEndpoint e = FalEndpoint::parse(AdapterSupport::str(ticket.meta, "endpoint"));
Json st = queue.status(e, ticket.id);
JobStatus s;
s.raw_state = AdapterSupport::str(st, "status");
if (s.raw_state == "IN_QUEUE") {
s.state = JobState::kQueued;
return s;
}
if (s.raw_state != "COMPLETED") {
s.state = JobState::kRunning;
return s;
}
// COMPLETED is final for success, failure and cancellation alike: the
// result fetch tells which (a failed request answers with an HTTP error).
try {
Json res = queue.result(e, ticket.id);
s.state = JobState::kSucceeded;
s.assets = OutputScanner::scan(res, ticket.kind);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "completed without a result URL: " + res.dump().substr(0, 300);
}
} catch (const llm::ApiError& err) {
if (err.http_status() == 401 || err.http_status() == 403 || err.http_status() >= 500 || err.http_status() == 429) throw;
s.state = JobState::kFailed;
s.error = err.message();
}
return s;
}
bool FalJobs::cancel(FalQueue& queue, const JobTicket& ticket, bool confirm) {
AdapterSupport::need_confirm(confirm, "fal", "cancel");
queue.cancel(FalEndpoint::parse(AdapterSupport::str(ticket.meta, "endpoint")), ticket.id, true);
return true;
}
Json FalImageGenerator::body(const ImageRequest& r) const {
r.validate();
Json b = Json::object();
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
if (r.width && r.height) {
Json size = Json::object();
size.set("width", Json(r.width));
size.set("height", Json(r.height));
b.set("image_size", size);
}
AdapterSupport::put_seed(b, "seed", r.seed);
if (r.count > 1) b.set("num_images", Json(r.count));
AdapterSupport::put(b, "image_url", r.source_url);
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json FalImageGenerator::plan(const ImageRequest& r) const {
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
return queue_->plan(FalEndpoint::parse(r.model), body(r), {}).summary();
}
JobTicket FalImageGenerator::submit(const ImageRequest& r, bool confirm) {
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
FalEndpoint e = FalEndpoint::parse(r.model);
Json b = body(r);
AdapterSupport::need_confirm(confirm, "fal", "queue submit");
return FalJobs::ticket_from(queue_->submit(e, b, {}, true), e, MediaKind::kImage);
}
JobStatus FalImageGenerator::poll(const JobTicket& t) { return FalJobs::poll(*queue_, t); }
bool FalImageGenerator::cancel(const JobTicket& t, bool confirm) { return FalJobs::cancel(*queue_, t, confirm); }
Json FalVideoGenerator::body(const VideoRequest& r) const {
r.validate();
Json b = Json::object();
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds)));
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put(b, "resolution", r.resolution);
AdapterSupport::put_seed(b, "seed", r.seed);
AdapterSupport::put(b, "image_url", r.image_url);
AdapterSupport::put(b, "end_image_url", r.end_image_url);
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json FalVideoGenerator::plan(const VideoRequest& r) const {
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
return queue_->plan(FalEndpoint::parse(r.model), body(r), {}).summary();
}
JobTicket FalVideoGenerator::submit(const VideoRequest& r, bool confirm) {
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
FalEndpoint e = FalEndpoint::parse(r.model);
Json b = body(r);
AdapterSupport::need_confirm(confirm, "fal", "queue submit");
return FalJobs::ticket_from(queue_->submit(e, b, {}, true), e, MediaKind::kVideo);
}
JobStatus FalVideoGenerator::poll(const JobTicket& t) { return FalJobs::poll(*queue_, t); }
bool FalVideoGenerator::cancel(const JobTicket& t, bool confirm) { return FalJobs::cancel(*queue_, t, confirm); }
} // namespace molo::media
+128
View File
@@ -0,0 +1,128 @@
#pragma once
// fal: the queue API (queue.fal.run: submit, status, result, cancel) and the
// model-related Platform API (api.fal.ai/v1: model search, pricing, estimate,
// usage, analytics, billing events, request listing). Auth is
// "Authorization: Key <FAL_KEY>" on both hosts. Platform groups outside model
// inference (assets, serverless, compute, keys, organization) are out of scope
// and listed as exclusions in tools/check_spec.py.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/fal/calls.h"
namespace molo::media {
// "owner/app" or "owner/app/path" (fal-ai/flux/dev, fal-ai/flux/dev/image-to-image).
struct FalEndpoint {
std::string owner, app, path;
static FalEndpoint parse(const std::string& id); // throws ApiError::invalid
std::string app_id() const { return owner + "/" + app; }
std::string full() const { return path.empty() ? app_id() : app_id() + "/" + path; }
};
struct FalSubmitOptions {
std::string priority; // "", "low", "normal", "high" -> X-Fal-Queue-Priority
std::string webhook; // https URL -> fal_webhook
int max_queue_length = 0; // 0 = unset -> fal_max_queue_length
};
class FalQueue : public MediaService {
public:
static constexpr const char* kHost = "https://queue.fal.run";
static constexpr const char* kKeyEnv = "FAL_KEY";
explicit FalQueue(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), FalCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::header("fal", kHost, kKeyEnv, "Authorization"); }
// fal sends the key as "Key <value>".
static std::string header_value(const std::string& key) { return key.empty() ? "" : "Key " + key; }
static std::shared_ptr<FalQueue> from_env() {
return std::make_shared<FalQueue>(ClientFactory::real(config(), header_value(ClientFactory::env({kKeyEnv}))));
}
static std::shared_ptr<FalQueue> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<FalQueue>(ClientFactory::with(std::move(t), config(), header_value(key)));
}
PreparedCall plan(const FalEndpoint& endpoint, const Json& body, const FalSubmitOptions& options) const;
// Paid. Returns {request_id, response_url, status_url, cancel_url, ...}.
Json submit(const FalEndpoint& endpoint, const Json& body, const FalSubmitOptions& options, bool confirm);
Json status(const FalEndpoint& endpoint, const std::string& request_id, bool logs = false);
Json result(const FalEndpoint& endpoint, const std::string& request_id);
Json cancel(const FalEndpoint& endpoint, const std::string& request_id, bool confirm);
};
class FalPlatform : public MediaService {
public:
static constexpr const char* kHost = "https://api.fal.ai";
explicit FalPlatform(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), FalCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::header("fal", kHost, FalQueue::kKeyEnv, "Authorization"); }
static std::shared_ptr<FalPlatform> from_env() {
return std::make_shared<FalPlatform>(
ClientFactory::real(config(), FalQueue::header_value(ClientFactory::env({FalQueue::kKeyEnv}))));
}
static std::shared_ptr<FalPlatform> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<FalPlatform>(ClientFactory::with(std::move(t), config(), FalQueue::header_value(key)));
}
// Search; walks next_cursor up to max_pages / max_items and says so loudly
// ("truncated": true) when the server had more.
Json models(const std::string& query, const std::string& category, std::size_t max_pages = 3,
std::size_t max_items = 200);
Json pricing(const std::string& endpoint_id);
Json estimate(const Json& body); // pure computation on fal's side, no charge
Json usage(const std::map<std::string, std::string>& query);
Json analytics(const std::map<std::string, std::string>& query);
Json meta();
Json delete_payloads(const std::string& request_id, bool confirm); // destructive, no default
};
class FalImageGenerator : public ImageGenerator {
public:
explicit FalImageGenerator(std::shared_ptr<FalQueue> q) : queue_(std::move(q)) {}
std::string provider() const override { return "fal"; }
bool configured() const override { return queue_->configured(); }
std::vector<std::string> models() const override {
return {"fal-ai/flux/dev", "fal-ai/flux/schnell", "fal-ai/flux-pro/v1.1", "fal-ai/recraft/v3/text-to-image",
"fal-ai/ideogram/v3", "fal-ai/stable-diffusion-v35-large"};
}
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 1500; }
private:
Json body(const ImageRequest& request) const;
std::shared_ptr<FalQueue> queue_;
};
class FalVideoGenerator : public VideoGenerator {
public:
explicit FalVideoGenerator(std::shared_ptr<FalQueue> q) : queue_(std::move(q)) {}
std::string provider() const override { return "fal"; }
bool configured() const override { return queue_->configured(); }
std::vector<std::string> models() const override {
return {"fal-ai/kling-video/v2.1/master/text-to-video", "fal-ai/minimax/hailuo-02/standard/text-to-video",
"fal-ai/veo3", "fal-ai/luma-dream-machine"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 4000; }
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<FalQueue> queue_;
};
// Shared queue polling for both fal adapters.
struct FalJobs {
static JobTicket ticket_from(const Json& submit_reply, const FalEndpoint& endpoint, MediaKind kind);
static JobStatus poll(FalQueue& queue, const JobTicket& ticket);
static bool cancel(FalQueue& queue, const JobTicket& ticket, bool confirm);
};
} // namespace molo::media
+35
View File
@@ -0,0 +1,35 @@
#include "vendors/media/higgsfield/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"higgsfield-ai.soul.standard", "POST", "/higgsfield-ai/soul/standard", "post_higgsfield_ai_soul_standard", "json", "prompt,num_images,resolution,aspect_ratio", "", "paid"},
{"requests.id.status", "GET", "/requests/{request_id}/status", "get_request_status", "none", "", "", "read"},
{"requests.id.cancel", "POST", "/requests/{request_id}/cancel", "cancel_request", "none", "", "", "write"},
{"kling-video.v2.5-turbo.pro.image-to-video", "POST", "/kling-video/v2.5-turbo/pro/image-to-video", "post_kling_video_v2_5_turbo_pro_image_to_video", "json", "prompt,duration,cfg_scale,image_url,negative_prompt", "", "paid"},
{"kling-video.v2.5-turbo.pro.text-to-video", "POST", "/kling-video/v2.5-turbo/pro/text-to-video", "post_kling_video_v2_5_turbo_pro_text_to_video", "json", "prompt,duration,cfg_scale,negative_prompt", "", "paid"},
{"kling-video.v2.5-turbo.standard.image-to-video", "POST", "/kling-video/v2.5-turbo/standard/image-to-video", "post_kling_video_v2_5_turbo_standard_image_to_video", "json", "prompt,duration,cfg_scale,image_url,negative_prompt", "", "paid"},
{"minimax.hailuo-2.3.standard.image-to-video", "POST", "/minimax/hailuo-2.3/standard/image-to-video", "post_minimax_hailuo_2_3_standard_image_to_video", "json", "prompt,duration,image_url,prompt_optimizer", "", "paid"},
{"minimax.hailuo-2.3.standard.text-to-video", "POST", "/minimax/hailuo-2.3/standard/text-to-video", "post_minimax_hailuo_2_3_standard_text_to_video", "json", "prompt,duration,prompt_optimizer", "", "paid"},
{"requests.id", "DELETE", "/requests/{request_id}", "delete_request", "none", "", "", "delete"},
{"model.submit", "POST", "/{model_path}", "", "json", "*", "", "paid"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"RequestStatus", "status,request_id,status_url,cancel_url,error,images,video,url"},
};
// FIELDS-END
const CallTable& HiggsfieldCalls::table() {
static const CallTable t("higgsfield", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t HiggsfieldCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* HiggsfieldCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct HiggsfieldCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+156
View File
@@ -0,0 +1,156 @@
#include "vendors/media/higgsfield/higgsfield.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs for_model(const std::string& model_path, const Json& body) {
CallArgs a;
a.body = body;
if (HiggsfieldClient::row_for(model_path) == "model.submit") a.path["model_path"] = model_path;
return a;
}
CallArgs by_request(const std::string& id) {
CallArgs a;
a.path["request_id"] = id;
return a;
}
} // namespace
std::string HiggsfieldClient::row_for(const std::string& model_path) {
std::string key = model_path;
for (char& c : key)
if (c == '/') c = '.';
return HiggsfieldCalls::table().find(key) ? key : "model.submit";
}
PreparedCall HiggsfieldClient::plan(const std::string& model_path, const Json& body) const {
return prepare(row_for(model_path), for_model(model_path, body));
}
Json HiggsfieldClient::submit(const std::string& model_path, const Json& body, bool confirm) {
return call_json(row_for(model_path), for_model(model_path, body), confirm);
}
Json HiggsfieldClient::status(const std::string& id) { return call_json("requests.id.status", by_request(id)); }
Json HiggsfieldClient::cancel(const std::string& id, bool confirm) {
return call_json("requests.id.cancel", by_request(id), confirm);
}
Json HiggsfieldClient::remove(const std::string& id, bool confirm) {
return call_json("requests.id", by_request(id), confirm);
}
JobState HiggsfieldJobs::state_of(const std::string& status) {
if (status == "completed") return JobState::kSucceeded;
if (status == "failed" || status == "nsfw") return JobState::kFailed;
if (status == "canceled") return JobState::kCancelled;
if (status == "in_progress") return JobState::kRunning;
return JobState::kQueued; // queued
}
JobTicket HiggsfieldJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) {
JobTicket t;
t.provider = "higgsfield";
t.model = model;
t.kind = kind;
t.id = AdapterSupport::str(reply, "request_id");
if (t.id.empty()) throw llm::ApiError("higgsfield", 502, "BAD_RESPONSE", "no request_id in the reply");
return t;
}
JobStatus HiggsfieldJobs::poll(HiggsfieldClient& client, const JobTicket& ticket) {
Json st = client.status(ticket.id);
JobStatus s;
s.raw_state = AdapterSupport::str(st, "status");
s.state = state_of(s.raw_state);
if (s.raw_state == "nsfw") s.error = "rejected by the content filter (nsfw)";
if (s.state == JobState::kFailed && s.error.empty()) s.error = AdapterSupport::str(st, "error");
if (s.state == JobState::kSucceeded) {
for (const char* key : {"images", "video"})
if (const Json* x = st.find(key)) {
auto found = OutputScanner::scan(*x, ticket.kind);
s.assets.insert(s.assets.end(), found.begin(), found.end());
}
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "completed without images or video";
}
}
return s;
}
Json HiggsfieldImageGenerator::body(const ImageRequest& r, std::string& model) const {
r.validate();
model = r.model.empty() ? "higgsfield-ai/soul/standard" : r.model;
Json b = Json::object();
AdapterSupport::put(b, "prompt", r.prompt);
if (r.count > 1) b.set("num_images", Json(r.count));
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put_seed(b, "seed", r.seed);
AdapterSupport::put(b, "image_url", r.source_url);
if (r.width || r.height || !r.negative_prompt.empty())
throw llm::ApiError::invalid("higgsfield", "width / height / negative_prompt are model-specific: pass them in extra");
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json HiggsfieldImageGenerator::plan(const ImageRequest& request) const {
std::string model;
Json b = body(request, model);
return client_->plan(model, b).summary();
}
JobTicket HiggsfieldImageGenerator::submit(const ImageRequest& request, bool confirm) {
std::string model;
Json b = body(request, model);
AdapterSupport::need_confirm(confirm, "higgsfield", "submit");
return HiggsfieldJobs::ticket_from(client_->submit(model, b, true), model, MediaKind::kImage);
}
bool HiggsfieldImageGenerator::cancel(const JobTicket& t, bool confirm) {
AdapterSupport::need_confirm(confirm, "higgsfield", "cancel");
client_->cancel(t.id, true);
return true;
}
Json HiggsfieldVideoGenerator::body(const VideoRequest& r, std::string& model) const {
r.validate();
if (r.model.empty()) throw llm::ApiError::invalid("higgsfield", "model path is required (see models())");
model = r.model;
Json b = Json::object();
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put(b, "resolution", r.resolution);
AdapterSupport::put_int(b, "duration", r.duration_seconds);
AdapterSupport::put(b, "image_url", r.image_url);
AdapterSupport::put(b, "end_image_url", r.end_image_url);
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json HiggsfieldVideoGenerator::plan(const VideoRequest& request) const {
std::string model;
Json b = body(request, model);
return client_->plan(model, b).summary();
}
JobTicket HiggsfieldVideoGenerator::submit(const VideoRequest& request, bool confirm) {
std::string model;
Json b = body(request, model);
AdapterSupport::need_confirm(confirm, "higgsfield", "submit");
return HiggsfieldJobs::ticket_from(client_->submit(model, b, true), model, MediaKind::kVideo);
}
bool HiggsfieldVideoGenerator::cancel(const JobTicket& t, bool confirm) {
AdapterSupport::need_confirm(confirm, "higgsfield", "cancel");
client_->cancel(t.id, true);
return true;
}
} // namespace molo::media
+94
View File
@@ -0,0 +1,94 @@
#pragma once
// Higgsfield cloud API (https://api.higgsfield.ai): every model is an
// endpoint path ("higgsfield-ai/soul/standard", "kling-video/v2.5-turbo/pro/
// text-to-video", ...); the shared lifecycle is submit -> GET
// /requests/{id}/status -> optional cancel / delete. Auth is
// "Authorization: Key <HF_API_KEY_ID>:<HF_API_KEY_SECRET>". The published
// OpenAPI document is explicitly supplementary: a model absent from it is not
// unavailable, so submit() accepts any model path through the generic row.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/higgsfield/calls.h"
namespace molo::media {
class HiggsfieldClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.higgsfield.ai";
static constexpr const char* kIdEnv = "HF_API_KEY_ID";
static constexpr const char* kSecretEnv = "HF_API_KEY_SECRET";
explicit HiggsfieldClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), HiggsfieldCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::header("higgsfield", kHost, kIdEnv, "Authorization"); }
// "Key id:secret"; empty when either half is missing.
static std::string header_value(const std::string& id, const std::string& secret) {
return id.empty() || secret.empty() ? "" : "Key " + id + ":" + secret;
}
static std::shared_ptr<HiggsfieldClient> from_env() {
return std::make_shared<HiggsfieldClient>(ClientFactory::real(
config(), header_value(ClientFactory::env({kIdEnv}), ClientFactory::env({kSecretEnv}))));
}
static std::shared_ptr<HiggsfieldClient> with(std::unique_ptr<clients::Transport> t, const std::string& id,
const std::string& secret) {
return std::make_shared<HiggsfieldClient>(ClientFactory::with(std::move(t), config(), header_value(id, secret)));
}
// Uses the spec's typed row when the path is one of them, else model.submit.
Json submit(const std::string& model_path, const Json& body, bool confirm);
PreparedCall plan(const std::string& model_path, const Json& body) const;
Json status(const std::string& request_id);
Json cancel(const std::string& request_id, bool confirm); // queued requests only
Json remove(const std::string& request_id, bool confirm); // deletes files and parameters
static std::string row_for(const std::string& model_path);
};
class HiggsfieldJobs {
public:
static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind);
static JobStatus poll(HiggsfieldClient& client, const JobTicket& ticket);
static JobState state_of(const std::string& status);
};
class HiggsfieldImageGenerator : public ImageGenerator {
public:
explicit HiggsfieldImageGenerator(std::shared_ptr<HiggsfieldClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "higgsfield"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override { return {"higgsfield-ai/soul/standard"}; }
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override { return HiggsfieldJobs::poll(*client_, ticket); }
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 3000; }
private:
Json body(const ImageRequest& request, std::string& model) const;
std::shared_ptr<HiggsfieldClient> client_;
};
class HiggsfieldVideoGenerator : public VideoGenerator {
public:
explicit HiggsfieldVideoGenerator(std::shared_ptr<HiggsfieldClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "higgsfield"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"kling-video/v2.5-turbo/pro/text-to-video", "kling-video/v2.5-turbo/pro/image-to-video",
"kling-video/v2.5-turbo/standard/image-to-video", "minimax/hailuo-2.3/standard/text-to-video",
"minimax/hailuo-2.3/standard/image-to-video"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override { return HiggsfieldJobs::poll(*client_, ticket); }
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 5000; }
private:
Json body(const VideoRequest& request, std::string& model) const;
std::shared_ptr<HiggsfieldClient> client_;
};
} // namespace molo::media
+96
View File
@@ -0,0 +1,96 @@
#include "vendors/media/ideogram/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"image.precise-edit.ideogram-4-5", "POST", "/v2/image/precise-edit/ideogram-4-5", "imagesPreciseEdit_post_precise_edit_image_v2_ideogram45", "multipart", "prompt,image,reference_images,mask,context_window,quality,seed,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.auto", "POST", "/v2/image/generate/auto", "autoModel_post_generate_image_v2_auto", "multipart", "prompt,images,negative_prompt,aspect_ratio,resolution,resolution_tier,magic_prompt,num_images,seed,private,async", "dry_run", "paid"},
{"image.generate.ideogram-4-5", "POST", "/v2/image/generate/ideogram-4-5", "imagesGenerate_post_generate_image_v2_ideogram45", "multipart", "prompt,magic_prompt,images,mask,size,background,quality,seed,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-4", "POST", "/v2/image/generate/ideogram-4", "imagesGenerate_post_generate_image_v2_ideogram_v4", "json", "prompt,magic_prompt,seed,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-4-custom-model", "POST", "/v2/image/generate/ideogram-4-custom-model", "imagesGenerate_post_generate_image_v2_ideogram_v4_custom_model", "json", "prompt,custom_model_uri,transparent_background,magic_prompt,seed,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-4-transparent", "POST", "/v2/image/generate/ideogram-4-transparent", "imagesGenerate_post_generate_image_v2_ideogram_v4_transparent", "json", "prompt,magic_prompt,seed,num_images,aspect_ratio,output_resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-3", "POST", "/v2/image/generate/ideogram-3", "imagesGenerate_post_generate_image_v2_ideogram_v3", "multipart", "prompt,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_type,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-3-character", "POST", "/v2/image/generate/ideogram-3-character", "imagesGenerate_post_generate_image_v2_ideogram_v3_character", "multipart", "prompt,negative_prompt,character_reference_images,character_reference_mask,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-3-custom-model", "POST", "/v2/image/generate/ideogram-3-custom-model", "imagesGenerate_post_generate_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,transparent_background,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-3-transparent", "POST", "/v2/image/generate/ideogram-3-transparent", "imagesGenerate_post_generate_image_v2_ideogram_v3_transparent", "json", "prompt,negative_prompt,seed,upscale_factor,aspect_ratio,rendering_speed,magic_prompt,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-2a", "POST", "/v2/image/generate/ideogram-2a", "imagesGenerate_post_generate_image_v2_ideogram_v2_a", "json", "prompt,seed,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.ideogram-2", "POST", "/v2/image/generate/ideogram-2", "imagesGenerate_post_generate_image_v2_ideogram_v2", "json", "prompt,negative_prompt,seed,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.gpt-image-2-5-flare", "POST", "/v2/image/generate/gpt-image-2-5-flare", "imagesGenerate_post_generate_image_v2_gpt_image25_flare", "multipart", "prompt,images,mask,background,quality,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"},
{"image.generate.gpt-image-2-5-sunburst", "POST", "/v2/image/generate/gpt-image-2-5-sunburst", "imagesGenerate_post_generate_image_v2_gpt_image25_sunburst", "multipart", "prompt,images,mask,background,quality,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"},
{"image.generate.gpt-image-2", "POST", "/v2/image/generate/gpt-image-2", "imagesGenerate_post_generate_image_v2_gpt_image2", "multipart", "prompt,images,mask,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"},
{"image.generate.nano-banana-2", "POST", "/v2/image/generate/nano-banana-2", "imagesGenerate_post_generate_image_v2_nano_banana2", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"},
{"image.generate.nano-banana-2-1", "POST", "/v2/image/generate/nano-banana-2-1", "imagesGenerate_post_generate_image_v2_nano_banana21", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"},
{"image.generate.nano-banana-pro", "POST", "/v2/image/generate/nano-banana-pro", "imagesGenerate_post_generate_image_v2_nano_banana_pro", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"},
{"image.generate.p-image-ideogram", "POST", "/v2/image/generate/p-image-ideogram", "imagesGenerate_post_generate_image_v2_p_image_ideogram", "json", "prompt,magic_prompt,seed,num_images,quality,resolution,aspect_ratio,custom_width,custom_height,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.generate.z-image", "POST", "/v2/image/generate/z-image", "imagesGenerate_post_generate_image_v2_z_image", "json", "prompt,resolution,num_inference_steps,seed,num_images,async,webhook_url", "dry_run", "paid"},
{"image.remix.auto", "POST", "/v2/image/remix/auto", "autoModel_post_tool_remix", "multipart", "prompt,image,image_weight,negative_prompt,resolution,aspect_ratio,resolution_tier,magic_prompt,seed,style_preset,color_palette,style_codes,style_type,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.remix.ideogram-4", "POST", "/v2/image/remix/ideogram-4", "imagesRemix_post_remix_image_v2_ideogram_v4", "multipart", "prompt,image,image_weight,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.remix.ideogram-3", "POST", "/v2/image/remix/ideogram-3", "imagesRemix_post_remix_image_v2_ideogram_v3", "multipart", "prompt,image,image_weight,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_type,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.remix.ideogram-3-character", "POST", "/v2/image/remix/ideogram-3-character", "imagesRemix_post_remix_image_v2_ideogram_v3_character", "multipart", "prompt,image,image_weight,negative_prompt,character_reference_images,character_reference_mask,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.remix.ideogram-3-custom-model", "POST", "/v2/image/remix/ideogram-3-custom-model", "imagesRemix_post_remix_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,image,image_weight,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.inpaint.ideogram-3", "POST", "/v2/image/inpaint/ideogram-3", "imagesInpaint_post_inpaint_image_v2_ideogram_v3", "multipart", "prompt,image,mask,magic_prompt,num_images,seed,rendering_speed,style_type,style_preset,style_codes,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.inpaint.ideogram-3-character", "POST", "/v2/image/inpaint/ideogram-3-character", "imagesInpaint_post_inpaint_image_v2_ideogram_v3_character", "multipart", "prompt,image,mask,character_reference_images,character_reference_mask,magic_prompt,num_images,seed,rendering_speed,style_type,style_codes,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
{"image.inpaint.ideogram-3-custom-model", "POST", "/v2/image/inpaint/ideogram-3-custom-model", "imagesInpaint_post_inpaint_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,image,mask,magic_prompt,num_images,seed,rendering_speed,style_preset,style_codes,style_reference_images,enable_copyright_detection,async", "dry_run", "paid"},
{"image.reframe.ideogram-3", "POST", "/v2/image/reframe/ideogram-3", "reframe_ideogramV3", "multipart", "image,resolution,num_images,seed,rendering_speed,style_preset,color_palette,style_codes,style_reference_images,async", "dry_run", "paid"},
{"image.reframe.nano-banana-2", "POST", "/v2/image/reframe/nano-banana-2", "reframe_nanoBanana2", "multipart", "image,aspect_ratio,private,num_images,seed", "dry_run", "paid"},
{"image.upscale.auto", "POST", "/v2/image/upscale/auto", "tools_post_upscale", "multipart", "image,upscale_factor,seed,async,webhook_url", "dry_run", "paid"},
{"image.upscale.nano-banana-pro", "POST", "/v2/image/upscale/nano-banana-pro", "imagesUpscale_post_upscale_image_nano_banana_pro", "multipart", "image,prompt,resolution_tier,seed,async,webhook_url", "dry_run", "paid"},
{"image.upscale.topaz-bloom-2", "POST", "/v2/image/upscale/topaz-bloom-2", "imagesUpscale_post_upscale_image_topaz_bloom2", "multipart", "image,upscale_factor,prompt,creativity,color_preservation,seed,async,webhook_url", "dry_run", "paid"},
{"image.upscale.topaz-redefine", "POST", "/v2/image/upscale/topaz-redefine", "imagesUpscale_post_upscale_image_topaz_redefine", "multipart", "image,upscale_factor,prompt,creativity,autoprompt,texture,detail,detail_strength,sharpen,denoise,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,seed,async,webhook_url", "dry_run", "paid"},
{"image.upscale.topaz-standard-2", "POST", "/v2/image/upscale/topaz-standard-2", "imagesUpscale_post_upscale_image_topaz_standard_v2", "multipart", "image,upscale_factor,strength,sharpen,denoise,fix_compression,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,async,webhook_url", "dry_run", "paid"},
{"image.upscale.topaz-text-refine", "POST", "/v2/image/upscale/topaz-text-refine", "imagesUpscale_post_upscale_image_topaz_text_refine", "multipart", "image,upscale_factor,strength,sharpen,denoise,fix_compression,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,seed,async,webhook_url", "dry_run", "paid"},
{"image.upscale.topaz-wonder-3-5", "POST", "/v2/image/upscale/topaz-wonder-3-5", "imagesUpscale_post_upscale_image_topaz_wonder35", "multipart", "image,upscale_factor,enhancement_strength,seed,async,webhook_url", "dry_run", "paid"},
{"image.replace-background.ideogram-3", "POST", "/v2/image/replace-background/ideogram-3", "tool_replaceBackgroundIdeogramV3", "multipart", "image,prompt,rendering_speed,num_images,private,webhook_url", "dry_run", "paid"},
{"image.replace-background.gpt-image-2", "POST", "/v2/image/replace-background/gpt-image-2", "tool_replaceBackground", "multipart", "image,prompt,quality,num_images,private,webhook_url", "dry_run", "paid"},
{"image.remove-background.ideogram-1", "POST", "/v2/image/remove-background/ideogram-1", "tool_removeBackground", "multipart", "image,private,async", "dry_run", "paid"},
{"image.remove-object.ideogram-1", "POST", "/v2/image/remove-object/ideogram-1", "objectRemoval_post_remove_object_from_v2_assets", "multipart", "image,mask,seed", "dry_run", "paid"},
{"image.describe.ideogram-4", "POST", "/v2/image/describe/ideogram-4", "imageDescribe_post_describe_image_ideogram_v4", "multipart", "image,include_bbox,include_style_descriptions,include_tags", "", "paid"},
{"image.describe.ideogram-3", "POST", "/v2/image/describe/ideogram-3", "imageDescribe_post_describe_image_ideogram_v3", "multipart", "image", "", "paid"},
{"design.layerize.ideogram-3", "POST", "/v2/design/layerize/ideogram-3", "design_post_layerize_design_ideogram_v3", "multipart", "image,prompt,seed,font_candidate_files,async,webhook_url", "dry_run", "paid"},
{"video.generate.kling-3-standard-image-to-video", "POST", "/v2/video/generate/kling-3-standard-image-to-video", "videoGenerate_post_generate_video_kling_v3_standard_image_to_video", "multipart", "image,end_image,prompt,negative_prompt,duration,generate_audio,shot_type,cfg_scale,webhook_url", "dry_run", "paid"},
{"video.generate.kling-3-standard-text-to-video", "POST", "/v2/video/generate/kling-3-standard-text-to-video", "videoGenerate_post_generate_video_kling_v3_standard_text_to_video", "json", "prompt,negative_prompt,aspect_ratio,duration,generate_audio,shot_type,cfg_scale,webhook_url", "dry_run", "paid"},
{"video.generate.minimax-h3-image-to-video", "POST", "/v2/video/generate/minimax-h3-image-to-video", "videoGenerate_post_generate_video_minimax_h3_image_to_video", "multipart", "image,end_image,prompt,resolution,duration,webhook_url", "dry_run", "paid"},
{"video.generate.minimax-h3-reference-to-video", "POST", "/v2/video/generate/minimax-h3-reference-to-video", "videoEdit_post_edit_video_minimax_h3_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,prompt_expansion_mode,webhook_url", "dry_run", "paid"},
{"video.generate.minimax-h3-text-to-video", "POST", "/v2/video/generate/minimax-h3-text-to-video", "videoGenerate_post_generate_video_minimax_h3_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,webhook_url", "dry_run", "paid"},
{"video.generate.seedance-2-5-image-to-video", "POST", "/v2/video/generate/seedance-2-5-image-to-video", "videoGenerate_post_generate_video_seed_dance25_image_to_video", "multipart", "image,end_image,prompt,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
{"video.generate.seedance-2-5-reference-to-video", "POST", "/v2/video/generate/seedance-2-5-reference-to-video", "videoGenerate_post_generate_video_seed_dance25_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
{"video.generate.seedance-2-5-text-to-video", "POST", "/v2/video/generate/seedance-2-5-text-to-video", "videoGenerate_post_generate_video_seed_dance25_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
{"video.generate.seedance-2-image-to-video", "POST", "/v2/video/generate/seedance-2-image-to-video", "videoGenerate_post_generate_video_seed_dance2_image_to_video", "multipart", "image,end_image,prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
{"video.generate.seedance-2-reference-to-video", "POST", "/v2/video/generate/seedance-2-reference-to-video", "videoGenerate_post_generate_video_seed_dance2_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
{"video.generate.seedance-2-text-to-video", "POST", "/v2/video/generate/seedance-2-text-to-video", "videoGenerate_post_generate_video_seed_dance2_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
{"svg.generate.ideogram-vector-1", "POST", "/v2/svg/generate/ideogram-vector-1", "svgGenerate_post_generate_svg", "multipart", "prompt,image_asset_identifiers,images,aspect_ratio,quality,num_images,seed,private,target_collection_id,webhook_url", "dry_run", "paid"},
{"svg.vectorize.ideogram-vector-1", "POST", "/v2/svg/vectorize/ideogram-vector-1", "svgGenerate_post_vectorizer", "multipart", "image,image_asset_identifier,mode,private,target_collection_id,webhook_url", "dry_run", "paid"},
{"tool.ad-localizer", "POST", "/v2/tool/ad-localizer", "editWorkflow_post_ad_localizer", "multipart", "image,language,exact_copy,prompt,quality,seed,num_images,webhook_url", "dry_run", "paid"},
{"tool.ad-resizer", "POST", "/v2/tool/ad-resizer", "editWorkflow_post_ad_resizer", "multipart", "image,resolution,platform,prompt,quality,num_images,webhook_url", "dry_run", "paid"},
{"tool.ad-variations", "POST", "/v2/tool/ad-variations", "editWorkflow_post_ad_variations", "multipart", "image,variation_type,prompt,quality,num_images,webhook_url", "", "paid"},
{"tool.colorways", "POST", "/v2/tool/colorways", "editWorkflow_post_colorways", "multipart", "image,masks,colors,aspect_ratio,quality,webhook_url", "dry_run", "paid"},
{"tool.ghost-mannequin", "POST", "/v2/tool/ghost-mannequin", "editWorkflow_post_ghost_mannequin", "multipart", "garment_images,front_image,back_image,left_image,right_image,top_image,bottom_image,output_reference_asset_identifier,output_reference_image,view,instruction,metadata,aspect_ratio,quality,seed,webhook_url", "dry_run", "paid"},
{"tool.material-swap", "POST", "/v2/tool/material-swap", "editWorkflow_post_material_swap", "multipart", "image,masks,materials,aspect_ratio,quality,webhook_url", "dry_run", "paid"},
{"tool.model-pose-variants", "POST", "/v2/tool/model-pose-variants", "editWorkflow_post_model_pose_variants", "multipart", "source_image,instruction,aspect_ratio,quality,seed,num_images,webhook_url", "", "paid"},
{"tool.sketch-to-render", "POST", "/v2/tool/sketch-to-render", "editWorkflow_post_sketch_to_render", "multipart", "sketch_image,instruction,aspect_ratio,quality,seed,num_images,webhook_url", "", "paid"},
{"generations.id", "GET", "/v2/generations/{generation_id}", "generations_get_generation_v2", "none", "", "", "read"},
{"account.api-keys", "GET", "/v2/account/api-keys", "account_get_account_api_keys", "none", "", "", "read"},
{"account.invoices", "GET", "/v2/account/invoices", "account_get_account_invoices", "none", "", "", "read"},
{"account.usage", "GET", "/v2/account/usage", "account_get_account_usage", "none", "", "start_time,end_time,bucket_width,sources", "read"},
{"image.replace-background.ideogram-4-5", "POST", "/v2/image/replace-background/ideogram-4-5", "tool_replaceBackgroundIdeogram45", "multipart", "image,background_reference,num_images,private,webhook_url", "dry_run", "paid"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"GenerateImageIdeogramV3Response", "generation_id,data,seed"},
{"GenerationResponse", "generation_id,status,created,response_type,usage_cost_usd_micros,failure_reason,data"},
};
// FIELDS-END
const CallTable& IdeogramCalls::table() {
static const CallTable t("ideogram", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t IdeogramCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* IdeogramCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct IdeogramCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+174
View File
@@ -0,0 +1,174 @@
#include "vendors/media/ideogram/ideogram.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs args(const Json& fields, const std::vector<FilePart>& files, bool dry_run) {
CallArgs a;
a.body = fields;
a.files = files;
if (dry_run) a.query["dry_run"] = "true";
return a;
}
// Image routes take `async` in the body; video routes are always asynchronous.
bool is_async(const Json& fields) {
const Json* a = fields.find("async");
return a && a->is_bool() && a->as_bool();
}
JobStatus status_from(const Json& g, MediaKind kind) {
JobStatus s;
s.raw_state = AdapterSupport::str(g, "status");
if (s.raw_state == "completed") {
s.state = JobState::kSucceeded;
if (const Json* d = g.find("data")) s.assets = OutputScanner::scan(*d, kind);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "completed without a URL (images that fail the safety check have an empty url)";
}
} else if (s.raw_state == "failed") {
s.state = JobState::kFailed;
s.error = AdapterSupport::str(g, "failure_reason");
} else {
s.state = s.raw_state == "pending" ? JobState::kRunning : JobState::kQueued;
}
return s;
}
} // namespace
PreparedCall IdeogramClient::plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
bool dry_run) const {
return prepare(key, args(fields, files, dry_run));
}
Json IdeogramClient::generate(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
bool confirm) {
return call_json(key, args(fields, files, false), confirm);
}
Json IdeogramClient::quote(const std::string& key, const Json& fields, const std::vector<FilePart>& files) {
PreparedCall c = plan(key, fields, files, true);
c.effect = Effect::kRead; // dry_run=true is defined by the spec as not billed
return parse_json("ideogram", execute(c, false));
}
Json IdeogramClient::generation(const std::string& id) {
CallArgs a;
a.path["generation_id"] = id;
return call_json("generations.id", a);
}
Json IdeogramClient::usage(const std::map<std::string, std::string>& query) {
CallArgs a;
a.query = query;
return call_json("account.usage", a);
}
Json IdeogramClient::api_keys() { return call_json("account.api-keys"); }
Json IdeogramClient::invoices() { return call_json("account.invoices"); }
std::string IdeogramImageGenerator::key_for(const std::string& model) {
std::string m = model.empty() ? "ideogram-3" : model;
return "image.generate." + m;
}
Json IdeogramImageGenerator::fields(const ImageRequest& r, std::string& key) const {
r.validate();
key = key_for(r.model);
Json f = Json::object();
AdapterSupport::put(f, "prompt", r.prompt);
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
AdapterSupport::put_seed(f, "seed", r.seed > 2147483647L ? 2147483647L : r.seed);
if (r.count > 1) f.set("num_images", Json(r.count));
AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio);
if (r.width && r.height) f.set("resolution", Json(std::to_string(r.width) + "x" + std::to_string(r.height)));
if (!r.source_url.empty())
throw llm::ApiError::invalid("ideogram", "source images are files: use IdeogramClient::generate with a FilePart and an edit row");
AdapterSupport::merge_extra(f, r.extra);
return f;
}
Json IdeogramImageGenerator::plan(const ImageRequest& request) const {
std::string key;
Json f = fields(request, key);
return client_->plan(key, f, {}).summary();
}
JobTicket IdeogramImageGenerator::submit(const ImageRequest& request, bool confirm) {
std::string key;
Json f = fields(request, key);
AdapterSupport::need_confirm(confirm, "ideogram", "generate");
Json out = client_->generate(key, f, {}, true);
JobTicket t;
t.provider = "ideogram";
t.model = request.model.empty() ? "ideogram-3" : request.model;
t.kind = MediaKind::kImage;
t.id = AdapterSupport::str(out, "generation_id");
if (is_async(f)) {
if (t.id.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no generation_id in the async reply");
return t;
}
if (t.id.empty()) t.id = "immediate";
t.immediate = true;
if (const Json* d = out.find("data")) t.assets = OutputScanner::scan(*d, MediaKind::kImage);
if (t.assets.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no image URL in the reply (safety check?)");
return t;
}
JobStatus IdeogramImageGenerator::poll(const JobTicket& ticket) {
if (ticket.immediate) {
JobStatus s;
s.state = JobState::kSucceeded;
s.raw_state = "immediate";
s.assets = ticket.assets;
return s;
}
return status_from(client_->generation(ticket.id), MediaKind::kImage);
}
Json IdeogramVideoGenerator::fields(const VideoRequest& r, std::string& key) const {
r.validate();
if (r.model.empty()) throw llm::ApiError::invalid("ideogram", "model is required (for example kling-3-standard-text-to-video)");
key = "video.generate." + r.model;
Json f = Json::object();
AdapterSupport::put(f, "prompt", r.prompt);
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put(f, "resolution", r.resolution);
AdapterSupport::put_int(f, "duration", r.duration_seconds);
if (!r.image_url.empty())
throw llm::ApiError::invalid("ideogram", "start frames are files: use IdeogramClient::generate with an image-to-video row");
AdapterSupport::merge_extra(f, r.extra);
return f;
}
Json IdeogramVideoGenerator::plan(const VideoRequest& request) const {
std::string key;
Json f = fields(request, key);
return client_->plan(key, f, {}).summary();
}
JobTicket IdeogramVideoGenerator::submit(const VideoRequest& request, bool confirm) {
std::string key;
Json f = fields(request, key);
AdapterSupport::need_confirm(confirm, "ideogram", "generate");
Json out = client_->generate(key, f, {}, true);
JobTicket t;
t.provider = "ideogram";
t.model = request.model;
t.kind = MediaKind::kVideo;
t.id = AdapterSupport::str(out, "generation_id");
if (t.id.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no generation_id in the reply");
return t;
}
JobStatus IdeogramVideoGenerator::poll(const JobTicket& ticket) {
return status_from(client_->generation(ticket.id), MediaKind::kVideo);
}
} // namespace molo::media
+84
View File
@@ -0,0 +1,84 @@
#pragma once
// Ideogram API v2: image generate / remix / inpaint / reframe / upscale /
// background tools, SVG, video models, ad tools, generation polling, account.
// Auth is the Api-Key header. Every POST accepts dry_run=true, which the spec
// defines as "validated and priced but not run: nothing generated, stored or
// billed": quote() uses it, so a price check is free.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/ideogram/calls.h"
namespace molo::media {
class IdeogramClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.ideogram.ai";
static constexpr const char* kKeyEnv = "IDEOGRAM_API_KEY";
explicit IdeogramClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), IdeogramCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::header("ideogram", kHost, kKeyEnv, "Api-Key"); }
static std::shared_ptr<IdeogramClient> from_env() {
return std::make_shared<IdeogramClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<IdeogramClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<IdeogramClient>(ClientFactory::with(std::move(t), config(), key));
}
// Paid: any POST row of the table (image.*, video.*, svg.*, tool.*).
Json generate(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm);
PreparedCall plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
bool dry_run = false) const;
// Free price quote (dry_run=true): the only non-read call that needs no confirm.
Json quote(const std::string& key, const Json& fields, const std::vector<FilePart>& files);
Json generation(const std::string& generation_id);
Json usage(const std::map<std::string, std::string>& query);
Json api_keys();
Json invoices();
};
class IdeogramImageGenerator : public ImageGenerator {
public:
explicit IdeogramImageGenerator(std::shared_ptr<IdeogramClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "ideogram"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"ideogram-4-5", "ideogram-4", "ideogram-3", "ideogram-3-transparent", "ideogram-2a", "ideogram-2", "auto"};
}
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the API
int suggested_poll_ms() const override { return 2000; }
static std::string key_for(const std::string& model);
private:
Json fields(const ImageRequest& request, std::string& key) const;
std::shared_ptr<IdeogramClient> client_;
};
class IdeogramVideoGenerator : public VideoGenerator {
public:
explicit IdeogramVideoGenerator(std::shared_ptr<IdeogramClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "ideogram"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"kling-3-standard-text-to-video", "minimax-h3-text-to-video", "seedance-2-text-to-video",
"seedance-2-5-text-to-video"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; }
int suggested_poll_ms() const override { return 4000; }
private:
Json fields(const VideoRequest& request, std::string& key) const;
std::shared_ptr<IdeogramClient> client_;
};
} // namespace molo::media
+30
View File
@@ -0,0 +1,30 @@
#include "vendors/media/kling/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"text2video.create", "POST", "/v1/videos/text2video", "", "json", "model_name,prompt,negative_prompt,cfg_scale,mode,aspect_ratio,duration,callback_url,external_task_id", "", "paid"},
{"text2video.get", "GET", "/v1/videos/text2video/{id}", "", "none", "", "", "read"},
{"image2video.create", "POST", "/v1/videos/image2video", "", "json", "model_name,image,image_tail,prompt,negative_prompt,cfg_scale,mode,duration,callback_url,external_task_id", "", "paid"},
{"image2video.get", "GET", "/v1/videos/image2video/{id}", "", "none", "", "", "read"},
{"images.create", "POST", "/v1/images/generations", "", "json", "model_name,prompt,negative_prompt,image,image_reference,image_fidelity,n,aspect_ratio,callback_url", "", "paid"},
{"images.get", "GET", "/v1/images/generations/{id}", "", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
};
// FIELDS-END
const CallTable& KlingCalls::table() {
static const CallTable t("kling", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t KlingCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* KlingCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct KlingCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+156
View File
@@ -0,0 +1,156 @@
#include "vendors/media/kling/kling.h"
#include <ctime>
#include <functional>
#include "security/crypto/Base64.h"
#include "security/crypto/Hmac.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
std::shared_ptr<llm::ApiClient> build_client(const std::string& access, const std::string& secret, long now,
const std::function<std::unique_ptr<clients::Transport>()>& factory) {
std::string token = KlingClient::credentials_present(access, secret) ? KlingJwt::make(access, secret, now) : "";
if (factory) return ClientFactory::with(factory(), KlingClient::config(), token);
return ClientFactory::real(KlingClient::config(), token);
}
CallArgs with_body(const Json& body) {
CallArgs a;
a.body = body;
return a;
}
} // namespace
std::string KlingJwt::make(const std::string& access_key, const std::string& secret_key, long now) {
Json header = Json::object();
header.set("alg", Json("HS256"));
header.set("typ", Json("JWT"));
Json payload = Json::object();
payload.set("iss", Json(access_key));
payload.set("exp", Json(static_cast<double>(now + kTtlSeconds)));
payload.set("nbf", Json(static_cast<double>(now - 5)));
std::string signing = security::crypto::Base64::encode_url(header.dump()) + "." +
security::crypto::Base64::encode_url(payload.dump());
auto mac = security::crypto::Hmac::sha256(secret_key, signing);
return signing + "." + security::crypto::Base64::encode_url(reinterpret_cast<const std::uint8_t*>(mac.data()), mac.size());
}
KlingClient::KlingClient(std::string access_key, std::string secret_key,
std::function<std::unique_ptr<clients::Transport>()> transport)
: MediaService(build_client(access_key, secret_key, static_cast<long>(std::time(nullptr)), transport),
KlingCalls::table()),
access_(std::move(access_key)),
secret_(std::move(secret_key)),
transport_(std::move(transport)),
minted_at_(static_cast<long>(std::time(nullptr))) {}
std::shared_ptr<KlingClient> KlingClient::from_env() {
return std::make_shared<KlingClient>(ClientFactory::env({kAccessEnv}), ClientFactory::env({kSecretEnv}));
}
void KlingClient::renew() {
long now = static_cast<long>(std::time(nullptr));
if (!credentials_present(access_, secret_) || now - minted_at_ < KlingJwt::kRenewAfterSeconds) return;
client_ = build_client(access_, secret_, now, transport_);
minted_at_ = now;
}
void KlingClient::check_envelope(const Json& doc) {
const Json* code = doc.is_object() ? doc.find("code") : nullptr;
if (!code || !code->is_number() || code->as_int() == 0) return;
throw llm::ApiError("kling", 400, std::to_string(code->as_int()), AdapterSupport::str(doc, "message"),
AdapterSupport::str(doc, "request_id"));
}
PreparedCall KlingClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
Json KlingClient::submit(const std::string& key, const Json& body, bool confirm) {
renew();
Json out = call_json(key, with_body(body), confirm);
check_envelope(out);
return out;
}
Json KlingClient::get(const std::string& key, const std::string& task_id) {
renew();
CallArgs a;
a.path["id"] = task_id;
Json out = call_json(key, a);
check_envelope(out);
return out;
}
JobState KlingVideoGenerator::state_of(const std::string& status) {
if (status == "succeed") return JobState::kSucceeded;
if (status == "failed") return JobState::kFailed;
if (status == "processing") return JobState::kRunning;
return JobState::kQueued; // submitted
}
Json KlingVideoGenerator::body(const VideoRequest& r) const {
r.validate();
Json b = Json::object();
b.set("model_name", Json(r.model.empty() ? "kling-v1-6" : r.model));
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds)));
AdapterSupport::put(b, "image", r.image_url);
AdapterSupport::put(b, "image_tail", r.end_image_url);
if (r.seed >= 0) throw llm::ApiError::invalid("kling", "no seed parameter is documented");
if (!r.resolution.empty()) throw llm::ApiError::invalid("kling", "resolution is set through mode (std / pro): pass mode in extra");
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json KlingVideoGenerator::plan(const VideoRequest& request) const {
return client_->plan(request.image_url.empty() ? "text2video.create" : "image2video.create", body(request)).summary();
}
JobTicket KlingVideoGenerator::submit(const VideoRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "kling", "generate");
const bool image = !request.image_url.empty();
Json out = client_->submit(image ? "image2video.create" : "text2video.create", b, true);
const Json* data = out.find("data");
JobTicket t;
t.provider = "kling";
t.model = b.find("model_name")->as_string();
t.kind = MediaKind::kVideo;
t.id = data ? AdapterSupport::str(*data, "task_id") : "";
if (t.id.empty()) throw llm::ApiError("kling", 502, "BAD_RESPONSE", "no data.task_id in the reply");
t.meta = Json::object();
t.meta.set("route", Json(image ? "image2video" : "text2video"));
return t;
}
JobStatus KlingVideoGenerator::poll(const JobTicket& ticket) {
std::string route = AdapterSupport::str(ticket.meta, "route");
if (route != "image2video" && route != "text2video") throw llm::ApiError::invalid("kling", "ticket has no route");
Json out = client_->get(route + ".get", ticket.id);
const Json* data = out.find("data");
JobStatus s;
if (!data) {
s.state = JobState::kQueued;
return s;
}
s.raw_state = AdapterSupport::str(*data, "task_status");
s.state = state_of(s.raw_state);
if (s.state == JobState::kFailed) s.error = AdapterSupport::str(*data, "task_status_msg");
if (s.state == JobState::kSucceeded) {
if (const Json* r = data->find("task_result")) s.assets = OutputScanner::scan(*r, MediaKind::kVideo);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "succeed without videos";
}
}
return s;
}
} // namespace molo::media
+80
View File
@@ -0,0 +1,80 @@
#pragma once
// Kling AI (Kuaishou) API. Kling publishes its developer docs as a JavaScript
// app with no machine-readable spec, so this adapter is built from community
// sources and live probes only: see SPEC "Unverified". Auth is a short-lived
// HS256 JWT built from KLING_ACCESS_KEY (iss) and KLING_SECRET_KEY. Hosts
// api-singapore.klingai.com and api.klingai.com both answered the documented
// path with `{"code":1001,"message":"Authorization is empty"}` unauthenticated.
#include <chrono>
#include <functional>
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/kling/calls.h"
namespace molo::media {
class KlingJwt {
public:
static constexpr long kTtlSeconds = 1800; // token lifetime
static constexpr long kRenewAfterSeconds = 1500; // the client rebuilds before this
// header {"alg":"HS256","typ":"JWT"}, payload {iss, exp, nbf}; base64url, no padding.
static std::string make(const std::string& access_key, const std::string& secret_key, long now_seconds);
};
class KlingClient : public MediaService {
public:
static constexpr const char* kHost = "https://api-singapore.klingai.com";
static constexpr const char* kAccessEnv = "KLING_ACCESS_KEY";
static constexpr const char* kSecretEnv = "KLING_SECRET_KEY";
// The token is fixed per ApiClient, so a KlingClient holds a factory and
// renews the ApiClient when the token nears expiry.
KlingClient(std::string access_key, std::string secret_key,
std::function<std::unique_ptr<clients::Transport>()> transport = nullptr);
static llm::ApiClientConfig config() { return ClientFactory::bearer("kling", kHost, kAccessEnv); }
static std::shared_ptr<KlingClient> from_env();
static bool credentials_present(const std::string& access, const std::string& secret) {
return !access.empty() && !secret.empty();
}
// Paid submits: text2video.create, image2video.create, images.create.
Json submit(const std::string& key, const Json& body, bool confirm);
PreparedCall plan(const std::string& key, const Json& body) const;
Json get(const std::string& key, const std::string& task_id); // *.get rows
// Throws ApiError when the envelope's code is not 0.
static void check_envelope(const Json& doc);
private:
void renew();
std::string access_, secret_;
std::function<std::unique_ptr<clients::Transport>()> transport_;
long minted_at_ = 0;
};
class KlingVideoGenerator : public VideoGenerator {
public:
explicit KlingVideoGenerator(std::shared_ptr<KlingClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "kling"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"kling-v1", "kling-v1-6", "kling-v2-master", "kling-v2-1", "kling-v2-5-turbo"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; } // not documented
int suggested_poll_ms() const override { return 6000; }
// submitted / processing / succeed / failed (community sources).
static JobState state_of(const std::string& status);
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<KlingClient> client_;
};
} // namespace molo::media
+30
View File
@@ -0,0 +1,30 @@
#include "vendors/media/leonardo/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"v2.generations.create", "POST", "/v2/generations", "createGeneration", "json", "model,parameters,public", "", "paid"},
{"v2.generations.sync", "POST", "/v2/generationssync", "createSyncGeneration", "json", "model,parameters,public", "", "paid"},
{"v2.models", "GET", "/v2/models", "getModels", "none", "", "", "read"},
{"v1.generations.create", "POST", "/v1/generations", "", "json", "prompt,modelId,width,height,num_images,alchemy,ultra,contrast,styleUUID,negative_prompt,seed,public", "", "paid"},
{"v1.generations.get", "GET", "/v1/generations/{id}", "getGenerationById", "none", "", "", "read"},
{"v1.me", "GET", "/v1/me", "getUserSelf", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
};
// FIELDS-END
const CallTable& LeonardoCalls::table() {
static const CallTable t("leonardo", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t LeonardoCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* LeonardoCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct LeonardoCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+129
View File
@@ -0,0 +1,129 @@
#include "vendors/media/leonardo/leonardo.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs with_body(const Json& body) {
CallArgs a;
a.body = body;
return a;
}
} // namespace
PreparedCall LeonardoClient::plan(const Json& body) const { return prepare("v2.generations.create", with_body(body)); }
Json LeonardoClient::create(const Json& body, bool confirm) {
return call_json("v2.generations.create", with_body(body), confirm);
}
Json LeonardoClient::create_sync(const Json& body, bool confirm) {
return call_json("v2.generations.sync", with_body(body), confirm);
}
Json LeonardoClient::models() { return call_json("v2.models"); }
Json LeonardoClient::generation(const std::string& id) {
CallArgs a;
a.path["id"] = id;
return call_json("v1.generations.get", a);
}
Json LeonardoClient::me() { return call_json("v1.me"); }
Json LeonardoJobs::body(const std::string& model, const Json& parameters, bool is_public) {
Json b = Json::object();
b.set("model", Json(model));
b.set("parameters", parameters);
b.set("public", Json(is_public)); // never publish to the community feed by default
return b;
}
JobTicket LeonardoJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) {
JobTicket t;
t.provider = "leonardo";
t.model = model;
t.kind = kind;
t.id = AdapterSupport::str(reply, "generationId");
if (t.id.empty()) throw llm::ApiError("leonardo", 502, "BAD_RESPONSE", "no generationId in the reply");
t.meta = Json::object();
AdapterSupport::put(t.meta, "apiCreditCost", AdapterSupport::str(reply, "apiCreditCost"));
return t;
}
// v1 GET /generations/{id}: {"generations_by_pk": {"status": "PENDING|COMPLETE|FAILED",
// "generated_images": [{"url": ...}], ...}} (status words from the FAQ; shape unverified).
JobStatus LeonardoJobs::status_of(const Json& doc, MediaKind kind) {
const Json* g = doc.find("generations_by_pk");
const Json& gen = g && g->is_object() ? *g : doc;
JobStatus s;
s.raw_state = AdapterSupport::str(gen, "status");
if (s.raw_state == "COMPLETE") {
s.state = JobState::kSucceeded;
for (const char* key : {"generated_images", "generated_videos", "generated_assets"})
if (const Json* a = gen.find(key)) {
auto found = OutputScanner::scan(*a, kind);
s.assets.insert(s.assets.end(), found.begin(), found.end());
}
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "COMPLETE without generated images";
}
} else if (s.raw_state == "FAILED") {
s.state = JobState::kFailed;
s.error = "FAILED";
} else {
s.state = JobState::kRunning; // PENDING
}
return s;
}
JobStatus LeonardoJobs::poll(LeonardoClient& client, const JobTicket& ticket) {
return status_of(client.generation(ticket.id), ticket.kind);
}
Json LeonardoImageGenerator::body(const ImageRequest& r) const {
r.validate();
if (r.model.empty()) throw llm::ApiError::invalid("leonardo", "model is required (see LeonardoClient::models)");
Json p = Json::object();
AdapterSupport::put(p, "prompt", r.prompt);
AdapterSupport::put_int(p, "width", r.width);
AdapterSupport::put_int(p, "height", r.height);
p.set("quantity", Json(r.count));
AdapterSupport::put_seed(p, "seed", r.seed);
if (!r.aspect_ratio.empty() || !r.negative_prompt.empty() || !r.source_url.empty())
throw llm::ApiError::invalid("leonardo", "aspect_ratio / negative_prompt / source images are model-specific: pass them in extra (parameters)");
AdapterSupport::merge_extra(p, r.extra);
return LeonardoJobs::body(r.model, p, false);
}
Json LeonardoImageGenerator::plan(const ImageRequest& request) const { return client_->plan(body(request)).summary(); }
JobTicket LeonardoImageGenerator::submit(const ImageRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "leonardo", "generate");
return LeonardoJobs::ticket_from(client_->create(b, true), request.model, MediaKind::kImage);
}
Json LeonardoVideoGenerator::body(const VideoRequest& r) const {
r.validate();
if (r.model.empty()) throw llm::ApiError::invalid("leonardo", "model is required (see LeonardoClient::models)");
Json p = Json::object();
AdapterSupport::put(p, "prompt", r.prompt);
AdapterSupport::put_int(p, "duration", r.duration_seconds);
AdapterSupport::put(p, "resolution", r.resolution);
AdapterSupport::put_seed(p, "seed", r.seed);
if (!r.aspect_ratio.empty() || !r.negative_prompt.empty() || !r.image_url.empty() || !r.end_image_url.empty())
throw llm::ApiError::invalid("leonardo", "frames / aspect_ratio / negative_prompt are model-specific: pass them in extra (parameters)");
AdapterSupport::merge_extra(p, r.extra);
return LeonardoJobs::body(r.model, p, false);
}
Json LeonardoVideoGenerator::plan(const VideoRequest& request) const { return client_->plan(body(request)).summary(); }
JobTicket LeonardoVideoGenerator::submit(const VideoRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "leonardo", "generate");
return LeonardoJobs::ticket_from(client_->create(b, true), request.model, MediaKind::kVideo);
}
} // namespace molo::media
+89
View File
@@ -0,0 +1,89 @@
#pragma once
// Leonardo.Ai production API. v2 (https://cloud.leonardo.ai/api/rest/v2):
// POST /generations (async, any image / video / audio / 3D model by id),
// POST /generationssync, GET /models. Polling uses the documented v1
// GET /generations/{id} (the v1 reference pages are gone from the docs; the
// shape is described in prose only, see SPEC Unverified). Bearer auth with
// LEONARDO_API_KEY.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/leonardo/calls.h"
namespace molo::media {
class LeonardoClient : public MediaService {
public:
static constexpr const char* kHost = "https://cloud.leonardo.ai/api/rest";
static constexpr const char* kKeyEnv = "LEONARDO_API_KEY";
explicit LeonardoClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), LeonardoCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::bearer("leonardo", kHost, kKeyEnv); }
static std::shared_ptr<LeonardoClient> from_env() {
return std::make_shared<LeonardoClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<LeonardoClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<LeonardoClient>(ClientFactory::with(std::move(t), config(), key));
}
// body = {"model": "<id>", "parameters": {...}, "public": false}
Json create(const Json& body, bool confirm);
PreparedCall plan(const Json& body) const;
Json create_sync(const Json& body, bool confirm); // up to ~27 s, finished result in the reply
Json models(); // v2, parameter specs per model
Json generation(const std::string& id); // v1
Json me(); // v1, account and token balance
};
// Shared by the image and video adapters (the API does not tell them apart).
class LeonardoJobs {
public:
static Json body(const std::string& model, const Json& parameters, bool is_public);
static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind);
static JobStatus poll(LeonardoClient& client, const JobTicket& ticket);
static JobStatus status_of(const Json& generation, MediaKind kind);
};
class LeonardoImageGenerator : public ImageGenerator {
public:
explicit LeonardoImageGenerator(std::shared_ptr<LeonardoClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "leonardo"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"flux-pro-2.0", "flux-kontext-pro", "phoenix-v1.0", "lucid-origin", "ideogram-v3.0", "gpt-image-2",
"nano-banana-2", "seedream-4.5"};
}
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override { return LeonardoJobs::poll(*client_, ticket); }
bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the v2 spec
int suggested_poll_ms() const override { return 3000; }
private:
Json body(const ImageRequest& request) const;
std::shared_ptr<LeonardoClient> client_;
};
class LeonardoVideoGenerator : public VideoGenerator {
public:
explicit LeonardoVideoGenerator(std::shared_ptr<LeonardoClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "leonardo"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"motion_2.0", "veo-3.1-fast-generate-001", "kling-3.0", "hailuo-03", "seedance-2.0", "wan-2.7"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override { return LeonardoJobs::poll(*client_, ticket); }
bool cancel(const JobTicket&, bool) override { return false; }
int suggested_poll_ms() const override { return 5000; }
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<LeonardoClient> client_;
};
} // namespace molo::media
+39
View File
@@ -0,0 +1,39 @@
#include "vendors/media/luma/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"generations.create_video", "POST", "/generations/video", "", "json", "model,aspect_ratio,callback_url,concepts,duration,generation_type,keyframes,loop,prompt,resolution", "", "paid"},
{"generations.create_image", "POST", "/generations/image", "", "json", "model,aspect_ratio,callback_url,character_ref,format,generation_type,image_ref,modify_image_ref,prompt,style_ref,sync,sync_timeout", "", "paid"},
{"generations.reframe_image", "POST", "/generations/image/reframe", "", "json", "aspect_ratio,generation_type,media,model,callback_url,format,grid_position_x,grid_position_y,prompt,resized_height,resized_width,x_end,x_start,y_end,y_start", "", "paid"},
{"generations.reframe_video", "POST", "/generations/video/reframe", "", "json", "aspect_ratio,generation_type,media,model,callback_url,first_frame,grid_position_x,grid_position_y,prompt,resized_height,resized_width,x_end,x_start,y_end,y_start", "", "paid"},
{"generations.modify_video", "POST", "/generations/video/modify", "", "json", "generation_type,media,mode,model,callback_url,first_frame,prompt", "", "paid"},
{"generations.upscale", "POST", "/generations/{id}/upscale", "", "json", "callback_url,generation_type,resolution", "", "paid"},
{"generations.audio", "POST", "/generations/{id}/audio", "", "json", "callback_url,generation_type,negative_prompt,prompt", "", "paid"},
{"generations.get", "GET", "/generations/{id}", "", "none", "", "", "read"},
{"generations.list", "GET", "/generations", "", "none", "", "limit,offset", "read"},
{"generations.delete", "DELETE", "/generations/{id}", "", "none", "", "", "delete"},
{"concepts.list", "GET", "/generations/concepts/list", "", "none", "", "", "read"},
{"credits.get", "GET", "/credits", "", "none", "", "", "read"},
{"ping", "GET", "/ping", "", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"Generation", "id,state,failure_reason,created_at,assets,generation_type,model,request,image,video,progress_video"},
{"CreditGetResponse", "credit_balance"},
};
// FIELDS-END
const CallTable& LumaCalls::table() {
static const CallTable t("luma", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t LumaCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* LumaCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct LumaCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+174
View File
@@ -0,0 +1,174 @@
#include "vendors/media/luma/luma.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs with_body(const Json& body) {
CallArgs a;
a.body = body;
return a;
}
CallArgs by_id(const std::string& id, const Json& body = Json()) {
CallArgs a;
a.path["id"] = id;
a.body = body;
return a;
}
} // namespace
PreparedCall LumaClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
Json LumaClient::create(const std::string& key, const Json& body, bool confirm) {
return call_json(key, with_body(body), confirm);
}
Json LumaClient::upscale(const std::string& id, const Json& body, bool confirm) {
return call_json("generations.upscale", by_id(id, body), confirm);
}
Json LumaClient::add_audio(const std::string& id, const Json& body, bool confirm) {
return call_json("generations.audio", by_id(id, body), confirm);
}
Json LumaClient::generation(const std::string& id) { return call_json("generations.get", by_id(id)); }
Json LumaClient::list(int limit, int offset) {
CallArgs a;
if (limit > 0) a.query["limit"] = std::to_string(limit);
if (offset > 0) a.query["offset"] = std::to_string(offset);
return call_json("generations.list", a);
}
Json LumaClient::remove(const std::string& id, bool confirm) { return call_json("generations.delete", by_id(id), confirm); }
Json LumaClient::credits() { return call_json("credits.get"); }
Json LumaClient::concepts() { return call_json("concepts.list"); }
Json LumaClient::ping() { return call_json("ping"); }
JobTicket LumaJobs::ticket_from(const Json& g, const std::string& model, MediaKind kind) {
JobTicket t;
t.provider = "luma";
t.model = model;
t.kind = kind;
t.id = AdapterSupport::str(g, "id");
if (t.id.empty()) throw llm::ApiError("luma", 502, "BAD_RESPONSE", "no id in the generation");
return t;
}
JobStatus LumaJobs::status_of(const Json& g, MediaKind kind) {
JobStatus s;
s.raw_state = AdapterSupport::str(g, "state");
if (s.raw_state == "completed") {
s.state = JobState::kSucceeded;
if (const Json* a = g.find("assets")) s.assets = OutputScanner::scan(*a, kind);
// progress_video is a preview, not the result
std::vector<MediaAsset> keep;
for (const MediaAsset& x : s.assets)
if (x.url.find("progress") == std::string::npos) keep.push_back(x);
if (!keep.empty()) s.assets = keep;
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "completed without assets";
}
} else if (s.raw_state == "failed") {
s.state = JobState::kFailed;
s.error = AdapterSupport::str(g, "failure_reason");
} else if (s.raw_state == "dreaming") {
s.state = JobState::kRunning;
} else {
s.state = JobState::kQueued;
}
return s;
}
JobStatus LumaJobs::poll(LumaClient& client, const JobTicket& ticket) {
return status_of(client.generation(ticket.id), ticket.kind);
}
Json LumaVideoGenerator::body(const VideoRequest& r) const {
r.validate();
Json b = Json::object();
b.set("model", Json(r.model.empty() ? "ray-2" : r.model));
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put(b, "resolution", r.resolution);
if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds) + "s"));
if (!r.image_url.empty() || !r.end_image_url.empty()) {
Json kf = Json::object();
if (!r.image_url.empty()) {
Json f0 = Json::object();
f0.set("type", Json("image"));
f0.set("url", Json(r.image_url));
kf.set("frame0", f0);
}
if (!r.end_image_url.empty()) {
Json f1 = Json::object();
f1.set("type", Json("image"));
f1.set("url", Json(r.end_image_url));
kf.set("frame1", f1);
}
b.set("keyframes", kf);
}
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json LumaVideoGenerator::plan(const VideoRequest& request) const {
return client_->plan("generations.create_video", body(request)).summary();
}
JobTicket LumaVideoGenerator::submit(const VideoRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "luma", "generate");
return LumaJobs::ticket_from(client_->create("generations.create_video", b, true),
b.find("model")->as_string(), MediaKind::kVideo);
}
JobStatus LumaVideoGenerator::poll(const JobTicket& ticket) { return LumaJobs::poll(*client_, ticket); }
bool LumaVideoGenerator::cancel(const JobTicket& ticket, bool confirm) {
AdapterSupport::need_confirm(confirm, "luma", "delete generation");
client_->remove(ticket.id, true);
return true;
}
Json LumaImageGenerator::body(const ImageRequest& r) const {
r.validate();
if (r.count != 1) throw llm::ApiError::invalid("luma", "one image per call (count must be 1)");
if (r.width || r.height) throw llm::ApiError::invalid("luma", "use aspect_ratio, not width / height");
Json b = Json::object();
b.set("model", Json(r.model.empty() ? "photon-1" : r.model));
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
if (!r.source_url.empty()) {
Json ref = Json::array();
Json one = Json::object();
one.set("url", Json(r.source_url));
ref.push(one);
b.set("image_ref", ref);
}
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json LumaImageGenerator::plan(const ImageRequest& request) const {
return client_->plan("generations.create_image", body(request)).summary();
}
JobTicket LumaImageGenerator::submit(const ImageRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "luma", "generate");
return LumaJobs::ticket_from(client_->create("generations.create_image", b, true),
b.find("model")->as_string(), MediaKind::kImage);
}
JobStatus LumaImageGenerator::poll(const JobTicket& ticket) { return LumaJobs::poll(*client_, ticket); }
bool LumaImageGenerator::cancel(const JobTicket& ticket, bool confirm) {
AdapterSupport::need_confirm(confirm, "luma", "delete generation");
client_->remove(ticket.id, true);
return true;
}
} // namespace molo::media
+82
View File
@@ -0,0 +1,82 @@
#pragma once
// Luma Dream Machine API (https://api.lumalabs.ai/dream-machine/v1): video
// (Ray models), images (Photon), reframe, modify, upscale, audio, concepts,
// credits. Bearer auth with LUMA_API_KEY. States: queued, dreaming,
// completed, failed (from the official SDK types).
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/luma/calls.h"
namespace molo::media {
class LumaClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.lumalabs.ai/dream-machine/v1";
static constexpr const char* kKeyEnv = "LUMA_API_KEY";
explicit LumaClient(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), LumaCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::bearer("luma", kHost, kKeyEnv); }
static std::shared_ptr<LumaClient> from_env() {
return std::make_shared<LumaClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<LumaClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<LumaClient>(ClientFactory::with(std::move(t), config(), key));
}
// key: generations.create_video | create_image | reframe_image | reframe_video | modify_video.
Json create(const std::string& key, const Json& body, bool confirm);
PreparedCall plan(const std::string& key, const Json& body) const;
Json upscale(const std::string& generation_id, const Json& body, bool confirm);
Json add_audio(const std::string& generation_id, const Json& body, bool confirm);
Json generation(const std::string& id);
Json list(int limit, int offset);
Json remove(const std::string& id, bool confirm); // destructive, no default
Json credits();
Json concepts();
Json ping();
};
class LumaVideoGenerator : public VideoGenerator {
public:
explicit LumaVideoGenerator(std::shared_ptr<LumaClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "luma"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override { return {"ray-2", "ray-flash-2"}; }
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket& ticket, bool confirm) override; // deletes the generation
int suggested_poll_ms() const override { return 4000; }
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<LumaClient> client_;
};
class LumaImageGenerator : public ImageGenerator {
public:
explicit LumaImageGenerator(std::shared_ptr<LumaClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "luma"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override { return {"photon-1", "photon-flash-1"}; }
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 2000; }
private:
Json body(const ImageRequest& request) const;
std::shared_ptr<LumaClient> client_;
};
struct LumaJobs {
static JobTicket ticket_from(const Json& generation, const std::string& model, MediaKind kind);
static JobStatus poll(LumaClient& client, const JobTicket& ticket);
static JobStatus status_of(const Json& generation, MediaKind kind);
};
} // namespace molo::media
+31
View File
@@ -0,0 +1,31 @@
#include "vendors/media/minimax/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"video.create", "POST", "/v1/video_generation", "videoGeneration", "json", "model,prompt,first_frame_image,last_frame_image,subject_reference,prompt_optimizer,fast_pretreatment,duration,resolution,callback_url", "", "paid"},
{"video.query", "GET", "/v1/query/video_generation", "", "none", "", "task_id", "read"},
{"files.retrieve", "GET", "/v1/files/retrieve", "", "none", "", "file_id", "read"},
{"v2.video.create", "POST", "/v2/video_generation", "", "json", "model,content,resolution,duration,ratio,extra,callback_url", "", "paid"},
{"v2.video.query", "GET", "/v2/query/video_generation/{task_id}", "", "none", "", "", "read"},
{"v2.video.list", "GET", "/v2/query/video_generation", "", "none", "", "page_num,page_size", "read"},
{"v2.video.delete", "DELETE", "/v2/video_generation/{task_id}", "", "none", "", "", "delete"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
};
// FIELDS-END
const CallTable& MinimaxCalls::table() {
static const CallTable t("minimax", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t MinimaxCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* MinimaxCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct MinimaxCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+138
View File
@@ -0,0 +1,138 @@
#include "vendors/media/minimax/minimax.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs with_body(const Json& body) {
CallArgs a;
a.body = body;
return a;
}
CallArgs task(const std::string& id) {
CallArgs a;
a.path["task_id"] = id;
return a;
}
} // namespace
void MinimaxClient::check_base_resp(const Json& doc) {
const Json* br = doc.is_object() ? doc.find("base_resp") : nullptr;
if (!br || !br->is_object()) return;
const Json* code = br->find("status_code");
if (!code || !code->is_number() || code->as_int() == 0) return;
throw llm::ApiError("minimax", 400, std::to_string(code->as_int()), AdapterSupport::str(*br, "status_msg"));
}
PreparedCall MinimaxClient::plan_video(const Json& body) const { return prepare("video.create", with_body(body)); }
Json MinimaxClient::create_video(const Json& body, bool confirm) {
Json out = call_json("video.create", with_body(body), confirm);
check_base_resp(out);
return out;
}
Json MinimaxClient::query_video(const std::string& task_id) {
CallArgs a;
a.query["task_id"] = task_id;
Json out = call_json("video.query", a);
check_base_resp(out);
return out;
}
Json MinimaxClient::retrieve_file(const std::string& file_id) {
CallArgs a;
a.query["file_id"] = file_id;
Json out = call_json("files.retrieve", a);
check_base_resp(out);
return out;
}
Json MinimaxClient::create_video_v2(const Json& body, bool confirm) {
return call_json("v2.video.create", with_body(body), confirm);
}
Json MinimaxClient::query_video_v2(const std::string& id) { return call_json("v2.video.query", task(id)); }
Json MinimaxClient::list_video_v2(int page_num, int page_size) {
CallArgs a;
if (page_num > 0) a.query["page_num"] = std::to_string(page_num);
if (page_size > 0) a.query["page_size"] = std::to_string(page_size);
return call_json("v2.video.list", a);
}
Json MinimaxClient::delete_video_v2(const std::string& id, bool confirm) {
return call_json("v2.video.delete", task(id), confirm);
}
JobState MinimaxVideoGenerator::state_of(const std::string& status) {
if (status == "Success") return JobState::kSucceeded;
if (status == "Fail") return JobState::kFailed;
if (status == "Processing") return JobState::kRunning;
return JobState::kQueued; // Preparing, Queueing
}
Json MinimaxVideoGenerator::body(const VideoRequest& r) const {
r.validate();
Json b = Json::object();
b.set("model", Json(r.model.empty() ? "MiniMax-Hailuo-2.3" : r.model));
AdapterSupport::put(b, "prompt", r.prompt);
AdapterSupport::put_int(b, "duration", r.duration_seconds);
if (!r.resolution.empty()) {
std::string res = r.resolution;
for (char& c : res) c = static_cast<char>(std::toupper(static_cast<unsigned char>(c)));
b.set("resolution", Json(res)); // docs: 768P / 1080P
}
AdapterSupport::put(b, "first_frame_image", r.image_url);
AdapterSupport::put(b, "last_frame_image", r.end_image_url);
if (!r.aspect_ratio.empty()) throw llm::ApiError::invalid("minimax", "v1 video_generation has no aspect_ratio");
if (r.seed >= 0) throw llm::ApiError::invalid("minimax", "v1 video_generation has no seed");
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json MinimaxVideoGenerator::plan(const VideoRequest& request) const {
return client_->plan_video(body(request)).summary();
}
JobTicket MinimaxVideoGenerator::submit(const VideoRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "minimax", "generate");
Json out = client_->create_video(b, true);
JobTicket t;
t.provider = "minimax";
t.model = b.find("model")->as_string();
t.kind = MediaKind::kVideo;
t.id = AdapterSupport::str(out, "task_id");
if (t.id.empty()) throw llm::ApiError("minimax", 502, "BAD_RESPONSE", "no task_id in the reply");
return t;
}
JobStatus MinimaxVideoGenerator::poll(const JobTicket& ticket) {
Json q = client_->query_video(ticket.id);
JobStatus s;
s.raw_state = AdapterSupport::str(q, "status");
s.state = state_of(s.raw_state);
if (s.state == JobState::kFailed) s.error = "Fail";
if (s.state == JobState::kSucceeded) {
std::string file_id = AdapterSupport::str(q, "file_id");
if (file_id.empty()) {
s.state = JobState::kFailed;
s.error = "Success without a file_id";
return s;
}
Json f = client_->retrieve_file(file_id);
if (const Json* file = f.find("file")) s.assets = OutputScanner::scan(*file, MediaKind::kVideo);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "file retrieved without a download_url";
}
}
return s;
}
} // namespace molo::media
+66
View File
@@ -0,0 +1,66 @@
#pragma once
// MiniMax (Hailuo) video API on https://api.minimax.io: v1 video_generation
// (text, image, first/last frame, subject reference), the query endpoint, file
// retrieval for the download URL, and the v2 content-array endpoints.
// Bearer auth with MINIMAX_API_KEY. Errors can arrive as HTTP 200 with
// base_resp.status_code != 0: MinimaxClient turns those into ApiError.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/minimax/calls.h"
namespace molo::media {
class MinimaxClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.minimax.io";
static constexpr const char* kKeyEnv = "MINIMAX_API_KEY";
explicit MinimaxClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), MinimaxCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::bearer("minimax", kHost, kKeyEnv); }
static std::shared_ptr<MinimaxClient> from_env() {
return std::make_shared<MinimaxClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<MinimaxClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<MinimaxClient>(ClientFactory::with(std::move(t), config(), key));
}
Json create_video(const Json& body, bool confirm); // v1, paid
PreparedCall plan_video(const Json& body) const;
Json query_video(const std::string& task_id); // v1
Json retrieve_file(const std::string& file_id); // v1
Json create_video_v2(const Json& body, bool confirm);
Json query_video_v2(const std::string& task_id);
Json list_video_v2(int page_num, int page_size);
Json delete_video_v2(const std::string& task_id, bool confirm); // no default
// Throws ApiError when base_resp.status_code is present and not 0.
static void check_base_resp(const Json& doc);
};
class MinimaxVideoGenerator : public VideoGenerator {
public:
explicit MinimaxVideoGenerator(std::shared_ptr<MinimaxClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "minimax"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"MiniMax-Hailuo-2.3", "MiniMax-Hailuo-02", "T2V-01-Director", "T2V-01"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; } // v1 has no cancel
int suggested_poll_ms() const override { return 8000; }
// Preparing / Queueing / Processing / Success / Fail (v1 query).
static JobState state_of(const std::string& status);
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<MinimaxClient> client_;
};
} // namespace molo::media
+48
View File
@@ -0,0 +1,48 @@
#include "vendors/media/recraft/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"images.creativeUpscale", "POST", "/v1/images/creativeUpscale", "CreativeUpscale", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"},
{"images.crispUpscale", "POST", "/v1/images/crispUpscale", "CrispUpscale", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"},
{"images.enhance", "POST", "/v1/images/enhance", "EnhanceImage", "either", "image_format,refinement,response_format,zdr,image_url,image", "", "paid"},
{"images.eraseRegion", "POST", "/v1/images/eraseRegion", "EraseRegion", "either", "image_format,response_format,zdr,image_url,mask_url,image,mask", "", "paid"},
{"images.generateBackground", "POST", "/v1/images/generateBackground", "GenerateBackground", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,mask_url,image,mask", "", "paid"},
{"images.generations", "POST", "/v1/images/generations", "GenerateImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"},
{"images.generations.raster", "POST", "/v1/images/generations/raster", "GenerateImageRaster", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"},
{"images.generations.vector", "POST", "/v1/images/generations/vector", "GenerateImageVector", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"},
{"images.imageToImage", "POST", "/v1/images/imageToImage", "ImageToImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,strength,image", "", "paid"},
{"images.inpaint", "POST", "/v1/images/inpaint", "InpaintImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,mask_url,image,mask", "", "paid"},
{"images.outpaint", "POST", "/v1/images/outpaint", "OutpaintImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,expand_bottom,expand_left,expand_right,expand_top,size,zoom_out_percentage,image_url,image", "", "paid"},
{"images.removeBackground", "POST", "/v1/images/removeBackground", "RemoveBackground", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"},
{"images.replaceBackground", "POST", "/v1/images/replaceBackground", "ReplaceBackground", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,image", "", "paid"},
{"images.variateImage", "POST", "/v1/images/variateImage", "VariateImage", "either", "image_format,model,n,random_seed,response_format,size,zdr,image_url,image", "", "paid"},
{"images.vectorize", "POST", "/v1/images/vectorize", "VectorizeImage", "either", "image_format,response_format,upscale,zdr,color_reduction,limit_num_shapes,max_num_colors,max_num_shapes,return_gradients,shape_stacking,small_shape_filter,strict_color_palette,svg_compression,image_url,image", "", "paid"},
{"styles", "GET", "/v1/styles", "ListStyles", "none", "", "", "read"},
{"styles.create", "POST", "/v1/styles", "CreateStyle", "either", "image_weights,match,mix_policy,model,palette,private,prompt,source_style_weights,source_styles,style,image_urls,images", "", "paid"},
{"styles.id.delete", "DELETE", "/v1/styles/{style_id}", "DeleteStyle", "none", "", "", "delete"},
{"styles.id", "GET", "/v1/styles/{style_id}", "GetStyle", "none", "", "", "read"},
{"styles.basic", "GET", "/v1/styles/basic", "ListBasicStyles", "none", "", "", "read"},
{"users.me", "GET", "/v1/users/me", "GetCurrentUser", "none", "", "", "read"},
{"prompts.enhance", "POST", "/v1/prompts/enhance", "EnhancePrompt", "json", "prompt", "", "paid"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"GenerateImageResponse", "created,data,credits,style_id"},
{"User", "id,email,name,credits"},
};
// FIELDS-END
const CallTable& RecraftCalls::table() {
static const CallTable t("recraft", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t RecraftCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* RecraftCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct RecraftCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+102
View File
@@ -0,0 +1,102 @@
#include "vendors/media/recraft/recraft.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs args(const Json& fields, const std::vector<FilePart>& files) {
CallArgs a;
a.body = fields;
a.files = files;
return a;
}
CallArgs style_id(const std::string& id) {
CallArgs a;
a.path["style_id"] = id;
return a;
}
} // namespace
PreparedCall RecraftClient::plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files) const {
return prepare(key, args(fields, files));
}
Json RecraftClient::run(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm) {
return call_json(key, args(fields, files), confirm);
}
Json RecraftClient::me() { return call_json("users.me"); }
Json RecraftClient::styles() { return call_json("styles"); }
Json RecraftClient::basic_styles() { return call_json("styles.basic"); }
Json RecraftClient::style(const std::string& id) { return call_json("styles.id", style_id(id)); }
Json RecraftClient::create_style(const Json& fields, const std::vector<FilePart>& files, bool confirm) {
return call_json("styles.create", args(fields, files), confirm);
}
Json RecraftClient::delete_style(const std::string& id, bool confirm) {
return call_json("styles.id.delete", style_id(id), confirm);
}
Json RecraftImageGenerator::fields(const ImageRequest& r) const {
r.validate();
Json f = Json::object();
AdapterSupport::put(f, "prompt", r.prompt);
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
AdapterSupport::put(f, "model", r.model);
if (r.count > 1) f.set("n", Json(r.count));
if (r.width && r.height) f.set("size", Json(std::to_string(r.width) + "x" + std::to_string(r.height)));
AdapterSupport::put_seed(f, "random_seed", r.seed);
if (!r.aspect_ratio.empty())
throw llm::ApiError::invalid("recraft", "Recraft sizes images with size (WxH), not aspect_ratio");
if (!r.source_url.empty())
throw llm::ApiError::invalid("recraft", "source images use the images.* edit rows: RecraftClient::run with image_url");
AdapterSupport::merge_extra(f, r.extra);
return f;
}
Json RecraftImageGenerator::plan(const ImageRequest& request) const {
return client_->plan("images.generations", fields(request), {}).summary();
}
JobTicket RecraftImageGenerator::submit(const ImageRequest& request, bool confirm) {
Json f = fields(request);
AdapterSupport::need_confirm(confirm, "recraft", "generate");
Json out = client_->run("images.generations", f, {}, true);
JobTicket t;
t.provider = "recraft";
t.model = request.model;
t.kind = MediaKind::kImage;
t.id = "immediate";
t.immediate = true;
if (const Json* d = out.find("data"); d && d->is_array()) {
for (const Json& item : d->as_array()) {
std::string b64 = AdapterSupport::str(item, "b64_json");
if (!b64.empty()) {
MediaAsset a;
a.kind = MediaKind::kImage;
a.data_base64 = b64;
a.mime = "image/webp"; // Recraft's default raster format is lossless WebP
t.assets.push_back(a);
}
}
if (t.assets.empty()) t.assets = OutputScanner::scan(*d, MediaKind::kImage);
}
if (t.assets.empty()) throw llm::ApiError("recraft", 502, "BAD_RESPONSE", "no image in the reply");
t.meta = Json::object();
AdapterSupport::put(t.meta, "credits", AdapterSupport::str(out, "credits"));
return t;
}
JobStatus RecraftImageGenerator::poll(const JobTicket& ticket) {
JobStatus s;
s.state = JobState::kSucceeded;
s.raw_state = "immediate";
s.assets = ticket.assets;
return s;
}
} // namespace molo::media
+61
View File
@@ -0,0 +1,61 @@
#pragma once
// Recraft external API: generation (general, raster, vector), image tools
// (vectorize, background, upscale, erase, remix, inpaint, outpaint, enhance),
// styles, prompt enhancement and the user endpoint. Bearer auth with
// RECRAFT_API_TOKEN, base https://external.api.recraft.ai. Every image call
// is synchronous: the reply carries the result, so tickets are immediate.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/recraft/calls.h"
namespace molo::media {
class RecraftClient : public MediaService {
public:
static constexpr const char* kHost = "https://external.api.recraft.ai";
static constexpr const char* kKeyEnv = "RECRAFT_API_TOKEN";
explicit RecraftClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), RecraftCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::bearer("recraft", kHost, kKeyEnv); }
static std::shared_ptr<RecraftClient> from_env() {
return std::make_shared<RecraftClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<RecraftClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<RecraftClient>(ClientFactory::with(std::move(t), config(), key));
}
// Paid image rows (images.*) and prompts.enhance. JSON bodies take image_url
// fields; multipart is used when files are given.
Json run(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm);
PreparedCall plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files) const;
Json me(); // credits + identity: the free read
Json styles();
Json basic_styles();
Json style(const std::string& style_id);
Json create_style(const Json& fields, const std::vector<FilePart>& files, bool confirm); // $0.005
Json delete_style(const std::string& style_id, bool confirm); // destructive
};
class RecraftImageGenerator : public ImageGenerator {
public:
explicit RecraftImageGenerator(std::shared_ptr<RecraftClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "recraft"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"recraftv4_1", "recraftv4_1_pro", "recraftv4", "recraftv4_pro", "recraftv3", "recraftv3_vector", "recraftv2"};
}
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; } // synchronous API
private:
Json fields(const ImageRequest& request) const;
std::shared_ptr<RecraftClient> client_;
};
} // namespace molo::media
+61
View File
@@ -0,0 +1,61 @@
#include "vendors/media/replicate/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"account.get", "GET", "/account", "account.get", "none", "", "", "read"},
{"collections.list", "GET", "/collections", "collections.list", "none", "", "", "read"},
{"collections.get", "GET", "/collections/{collection_slug}", "collections.get", "none", "", "", "read"},
{"deployments.list", "GET", "/deployments", "deployments.list", "none", "", "", "read"},
{"deployments.create", "POST", "/deployments", "deployments.create", "json", "hardware,max_instances,min_instances,model,name,version", "", "write"},
{"deployments.delete", "DELETE", "/deployments/{deployment_owner}/{deployment_name}", "deployments.delete", "none", "", "", "delete"},
{"deployments.get", "GET", "/deployments/{deployment_owner}/{deployment_name}", "deployments.get", "none", "", "", "read"},
{"deployments.update", "PATCH", "/deployments/{deployment_owner}/{deployment_name}", "deployments.update", "json", "hardware,max_instances,min_instances,version", "", "write"},
{"deployments.predictions.create", "POST", "/deployments/{deployment_owner}/{deployment_name}/predictions", "deployments.predictions.create", "json", "input,stream,webhook,webhook_events_filter", "", "paid"},
{"files.list", "GET", "/files", "files.list", "none", "", "", "read"},
{"files.create", "POST", "/files", "files.create", "multipart", "content,filename,metadata,type", "", "write"},
{"files.delete", "DELETE", "/files/{file_id}", "files.delete", "none", "", "", "delete"},
{"files.get", "GET", "/files/{file_id}", "files.get", "none", "", "", "read"},
{"files.download", "GET", "/files/{file_id}/download", "files.download", "none", "", "owner,expiry,signature", "read"},
{"hardware.list", "GET", "/hardware", "hardware.list", "none", "", "", "read"},
{"models.list", "GET", "/models", "models.list", "none", "", "sort_by,sort_direction", "read"},
{"models.create", "POST", "/models", "models.create", "json", "cover_image_url,description,github_url,hardware,license_url,name,owner,paper_url,visibility", "", "write"},
{"models.delete", "DELETE", "/models/{model_owner}/{model_name}", "models.delete", "none", "", "", "delete"},
{"models.get", "GET", "/models/{model_owner}/{model_name}", "models.get", "none", "", "", "read"},
{"models.update", "PATCH", "/models/{model_owner}/{model_name}", "models.update", "json", "description,github_url,license_url,paper_url,readme,weights_url", "", "write"},
{"models.examples.list", "GET", "/models/{model_owner}/{model_name}/examples", "models.examples.list", "none", "", "", "read"},
{"models.predictions.create", "POST", "/models/{model_owner}/{model_name}/predictions", "models.predictions.create", "json", "input,stream,webhook,webhook_events_filter", "", "paid"},
{"models.readme.get", "GET", "/models/{model_owner}/{model_name}/readme", "models.readme.get", "none", "", "", "read"},
{"models.versions.list", "GET", "/models/{model_owner}/{model_name}/versions", "models.versions.list", "none", "", "", "read"},
{"models.versions.delete", "DELETE", "/models/{model_owner}/{model_name}/versions/{version_id}", "models.versions.delete", "none", "", "", "delete"},
{"models.versions.get", "GET", "/models/{model_owner}/{model_name}/versions/{version_id}", "models.versions.get", "none", "", "", "read"},
{"trainings.create", "POST", "/models/{model_owner}/{model_name}/versions/{version_id}/trainings", "trainings.create", "json", "destination,input,webhook,webhook_events_filter", "", "paid"},
{"predictions.list", "GET", "/predictions", "predictions.list", "none", "", "created_after,created_before,source", "read"},
{"predictions.create", "POST", "/predictions", "predictions.create", "json", "input,stream,version,webhook,webhook_events_filter", "", "paid"},
{"predictions.get", "GET", "/predictions/{prediction_id}", "predictions.get", "none", "", "", "read"},
{"predictions.cancel", "POST", "/predictions/{prediction_id}/cancel", "predictions.cancel", "none", "", "", "write"},
{"search", "GET", "/search", "search", "none", "", "query,limit", "read"},
{"trainings.list", "GET", "/trainings", "trainings.list", "none", "", "", "read"},
{"trainings.get", "GET", "/trainings/{training_id}", "trainings.get", "none", "", "", "read"},
{"trainings.cancel", "POST", "/trainings/{training_id}/cancel", "trainings.cancel", "none", "", "", "write"},
{"webhooks.default.secret.get", "GET", "/webhooks/default/secret", "webhooks.default.secret.get", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"schemas_prediction_response", "id,model,version,input,output,status,error,logs,metrics,created_at,started_at,completed_at,urls,get,cancel"},
};
// FIELDS-END
const CallTable& ReplicateCalls::table() {
static const CallTable t("replicate", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t ReplicateCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* ReplicateCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct ReplicateCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+278
View File
@@ -0,0 +1,278 @@
#include "vendors/media/replicate/replicate.h"
#include <cstdlib>
#include <sstream>
#include "security/crypto/Base64.h"
#include "security/crypto/Hmac.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/caps.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs args_for(const ReplicateClient::ModelRef& m, const Json& input, const Json& options) {
CallArgs a;
Json body = Json::object();
body.set("input", input.is_null() ? Json::object() : input);
if (options.is_object())
for (const auto& [k, v] : options.as_object()) body.set(k, v); // webhook, webhook_events_filter, stream
if (m.version.empty()) {
a.path["model_owner"] = m.owner;
a.path["model_name"] = m.name;
} else {
body.set("version", Json(m.version));
}
a.body = body;
return a;
}
CallArgs one(const char* name, const std::string& value) {
CallArgs a;
a.path[name] = value;
return a;
}
Json mark_truncated(Json page) {
const Json* next = page.is_object() ? page.find("next") : nullptr;
if (next && next->is_string() && !next->as_string().empty()) page.set("truncated", Json(true));
return page;
}
} // namespace
ReplicateClient::ModelRef ReplicateClient::ModelRef::parse(const std::string& ref) {
ModelRef m;
std::string head = ref;
std::size_t colon = ref.find(':');
if (colon != std::string::npos) {
head = ref.substr(0, colon);
m.version = ref.substr(colon + 1);
}
std::size_t slash = head.find('/');
if (slash == std::string::npos || head.find('/', slash + 1) != std::string::npos)
throw llm::ApiError::invalid("replicate", "model must look like owner/name or owner/name:version");
m.owner = head.substr(0, slash);
m.name = head.substr(slash + 1);
if (!CallTable::valid_segment(m.owner) || !CallTable::valid_segment(m.name) ||
(colon != std::string::npos && !CallTable::valid_segment(m.version)))
throw llm::ApiError::invalid("replicate", "bad model reference");
return m;
}
PreparedCall ReplicateClient::plan_prediction(const ModelRef& m, const Json& input, const Json& options) const {
return prepare(m.version.empty() ? "models.predictions.create" : "predictions.create", args_for(m, input, options));
}
Json ReplicateClient::create_prediction(const ModelRef& m, const Json& input, const Json& options, bool confirm) {
return call_json(m.version.empty() ? "models.predictions.create" : "predictions.create",
args_for(m, input, options), confirm);
}
Json ReplicateClient::create_deployment_prediction(const std::string& owner, const std::string& name,
const Json& input, const Json& options, bool confirm) {
CallArgs a;
a.path["deployment_owner"] = owner;
a.path["deployment_name"] = name;
Json body = Json::object();
body.set("input", input);
if (options.is_object())
for (const auto& [k, v] : options.as_object()) body.set(k, v);
a.body = body;
return call_json("deployments.predictions.create", a, confirm);
}
Json ReplicateClient::get_prediction(const std::string& id) { return call_json("predictions.get", one("prediction_id", id)); }
Json ReplicateClient::cancel_prediction(const std::string& id, bool confirm) {
return call_json("predictions.cancel", one("prediction_id", id), confirm);
}
Json ReplicateClient::list_predictions() { return mark_truncated(call_json("predictions.list")); }
Json ReplicateClient::account() { return call_json("account.get"); }
Json ReplicateClient::hardware() { return call_json("hardware.list"); }
Json ReplicateClient::models_list() { return mark_truncated(call_json("models.list")); }
Json ReplicateClient::model_get(const std::string& owner, const std::string& name) {
CallArgs a;
a.path["model_owner"] = owner;
a.path["model_name"] = name;
return call_json("models.get", a);
}
Json ReplicateClient::versions_list(const std::string& owner, const std::string& name) {
CallArgs a;
a.path["model_owner"] = owner;
a.path["model_name"] = name;
return mark_truncated(call_json("models.versions.list", a));
}
Json ReplicateClient::deployments_list() { return mark_truncated(call_json("deployments.list")); }
Json ReplicateClient::deployment_get(const std::string& owner, const std::string& name) {
CallArgs a;
a.path["deployment_owner"] = owner;
a.path["deployment_name"] = name;
return call_json("deployments.get", a);
}
Json ReplicateClient::files_list() { return mark_truncated(call_json("files.list")); }
Json ReplicateClient::file_get(const std::string& id) { return call_json("files.get", one("file_id", id)); }
Json ReplicateClient::file_delete(const std::string& id, bool confirm) {
return call_json("files.delete", one("file_id", id), confirm);
}
Json ReplicateClient::file_upload(const std::string& filename, const std::string& content_type,
const std::string& data, bool confirm) {
CallArgs a;
a.body = Json::object();
a.body.set("filename", Json(filename));
a.files.push_back(FilePart{"content", filename, content_type, data});
return call_json("files.create", a, confirm);
}
Json ReplicateClient::collections_list() { return mark_truncated(call_json("collections.list")); }
Json ReplicateClient::search(const std::string& query) {
CallArgs a;
a.query["query"] = query;
return call_json("search", a);
}
Json ReplicateClient::webhook_secret() { return call_json("webhooks.default.secret.get"); }
Json ReplicateClient::trainings_list() { return mark_truncated(call_json("trainings.list")); }
// ---------------------------------------------------------------- webhooks
std::string ReplicateWebhook::sign(const std::string& secret, const std::string& id, const std::string& timestamp,
const std::string& body) {
std::string b64 = secret.rfind("whsec_", 0) == 0 ? secret.substr(6) : secret;
std::string key;
if (b64.empty() || !security::crypto::Base64::decode(b64, key) || key.empty()) return "";
if (body.size() > MediaCaps::kMaxWebhookBodyBytes) return "";
auto mac = security::crypto::Hmac::sha256(key, id + "." + timestamp + "." + body);
return security::crypto::Base64::encode(reinterpret_cast<const std::uint8_t*>(mac.data()), mac.size());
}
bool ReplicateWebhook::verify(const std::string& secret, const std::string& id, const std::string& timestamp,
const std::string& signature_header, const std::string& body, long now_seconds) {
if (id.empty() || timestamp.empty() || timestamp.size() > 12 || signature_header.size() > 1024) return false;
for (char c : timestamp)
if (c < '0' || c > '9') return false;
long ts = std::strtol(timestamp.c_str(), nullptr, 10);
long skew = now_seconds > ts ? now_seconds - ts : ts - now_seconds;
if (skew > MediaCaps::kWebhookToleranceSeconds) return false;
std::string expected = sign(secret, id, timestamp, body);
if (expected.empty()) return false;
std::istringstream in(signature_header);
std::string part;
while (in >> part) {
if (part.rfind("v1,", 0) != 0) continue;
std::string got = part.substr(3);
if (got.size() != expected.size()) continue;
unsigned char diff = 0; // constant time over the whole string
for (std::size_t i = 0; i < got.size(); ++i) diff |= static_cast<unsigned char>(got[i] ^ expected[i]);
if (diff == 0) return true;
}
return false;
}
// -------------------------------------------------------------------- jobs
JobTicket ReplicateJobs::ticket_from(const Json& p, const std::string& model, MediaKind kind) const {
JobTicket t;
t.provider = "replicate";
t.model = model;
t.kind = kind;
t.id = AdapterSupport::str(p, "id");
if (t.id.empty()) throw llm::ApiError("replicate", 502, "BAD_RESPONSE", "no id in the prediction");
return t;
}
JobStatus ReplicateJobs::poll(const JobTicket& ticket, MediaKind kind) {
Json p = client_->get_prediction(ticket.id);
JobStatus s;
s.raw_state = AdapterSupport::str(p, "status");
if (s.raw_state == "succeeded") {
s.state = JobState::kSucceeded;
if (const Json* o = p.find("output")) s.assets = OutputScanner::scan(*o, kind);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "succeeded without a downloadable output (non-URL output)";
}
} else if (s.raw_state == "failed") {
s.state = JobState::kFailed;
s.error = AdapterSupport::str(p, "error");
} else if (s.raw_state == "canceled" || s.raw_state == "aborted") {
s.state = JobState::kCancelled;
} else if (s.raw_state == "processing") {
s.state = JobState::kRunning;
} else {
s.state = JobState::kQueued;
}
return s;
}
bool ReplicateJobs::cancel(const JobTicket& ticket, bool confirm) {
AdapterSupport::need_confirm(confirm, "replicate", "cancel");
client_->cancel_prediction(ticket.id, true);
return true;
}
Json ReplicateImageGenerator::input(const ImageRequest& r) const {
r.validate();
Json in = Json::object();
AdapterSupport::put(in, "prompt", r.prompt);
AdapterSupport::put(in, "negative_prompt", r.negative_prompt);
AdapterSupport::put_int(in, "width", r.width);
AdapterSupport::put_int(in, "height", r.height);
AdapterSupport::put(in, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put_seed(in, "seed", r.seed);
if (r.count > 1) in.set("num_outputs", Json(r.count));
AdapterSupport::put(in, "image", r.source_url);
AdapterSupport::merge_extra(in, r.extra);
return in;
}
Json ReplicateImageGenerator::plan(const ImageRequest& r) const {
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
return jobs_.client().plan_prediction(ReplicateClient::ModelRef::parse(r.model), input(r), Json::object()).summary();
}
JobTicket ReplicateImageGenerator::submit(const ImageRequest& r, bool confirm) {
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
auto ref = ReplicateClient::ModelRef::parse(r.model);
Json in = input(r);
AdapterSupport::need_confirm(confirm, "replicate", "create prediction");
return jobs_.ticket_from(jobs_.client().create_prediction(ref, in, Json::object(), true), r.model, MediaKind::kImage);
}
Json ReplicateVideoGenerator::input(const VideoRequest& r) const {
r.validate();
Json in = Json::object();
AdapterSupport::put(in, "prompt", r.prompt);
AdapterSupport::put(in, "negative_prompt", r.negative_prompt);
AdapterSupport::put_int(in, "duration", r.duration_seconds);
AdapterSupport::put(in, "aspect_ratio", r.aspect_ratio);
AdapterSupport::put(in, "resolution", r.resolution);
AdapterSupport::put_seed(in, "seed", r.seed);
AdapterSupport::put(in, "image", r.image_url);
AdapterSupport::put(in, "last_frame_image", r.end_image_url);
AdapterSupport::merge_extra(in, r.extra);
return in;
}
Json ReplicateVideoGenerator::plan(const VideoRequest& r) const {
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
return jobs_.client().plan_prediction(ReplicateClient::ModelRef::parse(r.model), input(r), Json::object()).summary();
}
JobTicket ReplicateVideoGenerator::submit(const VideoRequest& r, bool confirm) {
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
auto ref = ReplicateClient::ModelRef::parse(r.model);
Json in = input(r);
AdapterSupport::need_confirm(confirm, "replicate", "create prediction");
return jobs_.ticket_from(jobs_.client().create_prediction(ref, in, Json::object(), true), r.model, MediaKind::kVideo);
}
} // namespace molo::media
+129
View File
@@ -0,0 +1,129 @@
#pragma once
// Replicate: predictions (official models, versions, deployments), models,
// files, account, hardware, collections, trainings and webhook verification.
// Bearer auth, base https://api.replicate.com/v1.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/replicate/calls.h"
namespace molo::media {
class ReplicateClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.replicate.com/v1"; // the spec server url; table paths are relative to it
static constexpr const char* kKeyEnv = "REPLICATE_API_TOKEN";
explicit ReplicateClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), ReplicateCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::bearer("replicate", kHost, kKeyEnv); }
static std::shared_ptr<ReplicateClient> from_env() {
return std::make_shared<ReplicateClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<ReplicateClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<ReplicateClient>(ClientFactory::with(std::move(t), config(), key));
}
// "owner/name" (official model) or "owner/name:version" (pinned version).
struct ModelRef {
std::string owner, name, version;
static ModelRef parse(const std::string& ref); // throws ApiError::invalid
};
// Paid. Official model -> models.predictions.create; with version -> predictions.create.
Json create_prediction(const ModelRef& model, const Json& input, const Json& options, bool confirm);
PreparedCall plan_prediction(const ModelRef& model, const Json& input, const Json& options) const;
Json create_deployment_prediction(const std::string& owner, const std::string& name, const Json& input,
const Json& options, bool confirm);
Json get_prediction(const std::string& id);
Json cancel_prediction(const std::string& id, bool confirm); // write, no default
// First page only; "truncated" is set when the server has a next page.
Json list_predictions();
Json account();
Json hardware();
Json models_list();
Json model_get(const std::string& owner, const std::string& name);
Json versions_list(const std::string& owner, const std::string& name);
Json deployments_list();
Json deployment_get(const std::string& owner, const std::string& name);
Json files_list();
Json file_get(const std::string& id);
Json file_upload(const std::string& filename, const std::string& content_type, const std::string& data, bool confirm);
Json file_delete(const std::string& id, bool confirm); // destructive, no default
Json collections_list();
Json search(const std::string& query);
Json webhook_secret();
Json trainings_list();
};
// Standard Webhooks verification (id, timestamp, signature headers).
class ReplicateWebhook {
public:
// secret = the "whsec_..." value from webhooks.default.secret.get.
// signature_header = "v1,<base64> v1,<base64>" (space separated).
// now_seconds: injected clock. Returns false on any mismatch, stale or
// malformed input; never throws on attacker-controlled data.
static bool verify(const std::string& secret, const std::string& webhook_id, const std::string& timestamp,
const std::string& signature_header, const std::string& body, long now_seconds);
// base64(HMAC-SHA256(key, id + "." + timestamp + "." + body)); "" on a bad secret.
static std::string sign(const std::string& secret, const std::string& webhook_id, const std::string& timestamp,
const std::string& body);
};
// kind-parametrised adapter core shared by the image and video generators.
class ReplicateJobs {
public:
explicit ReplicateJobs(std::shared_ptr<ReplicateClient> c) : client_(std::move(c)) {}
JobStatus poll(const JobTicket& ticket, MediaKind kind);
bool cancel(const JobTicket& ticket, bool confirm);
JobTicket ticket_from(const Json& prediction, const std::string& model, MediaKind kind) const;
ReplicateClient& client() { return *client_; }
const ReplicateClient& client() const { return *client_; }
private:
std::shared_ptr<ReplicateClient> client_;
};
class ReplicateImageGenerator : public ImageGenerator {
public:
explicit ReplicateImageGenerator(std::shared_ptr<ReplicateClient> c) : jobs_(std::move(c)) {}
std::string provider() const override { return "replicate"; }
bool configured() const override { return jobs_.client().configured(); }
std::vector<std::string> models() const override {
return {"black-forest-labs/flux-schnell", "black-forest-labs/flux-dev", "black-forest-labs/flux-1.1-pro",
"stability-ai/stable-diffusion-3.5-large", "ideogram-ai/ideogram-v3-turbo", "recraft-ai/recraft-v3"};
}
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override { return jobs_.poll(ticket, MediaKind::kImage); }
bool cancel(const JobTicket& ticket, bool confirm) override { return jobs_.cancel(ticket, confirm); }
int suggested_poll_ms() const override { return 1500; }
private:
Json input(const ImageRequest& request) const;
ReplicateJobs jobs_;
};
class ReplicateVideoGenerator : public VideoGenerator {
public:
explicit ReplicateVideoGenerator(std::shared_ptr<ReplicateClient> c) : jobs_(std::move(c)) {}
std::string provider() const override { return "replicate"; }
bool configured() const override { return jobs_.client().configured(); }
std::vector<std::string> models() const override {
return {"minimax/video-01", "luma/ray", "kwaivgi/kling-v1.6-standard", "wan-video/wan-2.1-t2v-480p"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override { return jobs_.poll(ticket, MediaKind::kVideo); }
bool cancel(const JobTicket& ticket, bool confirm) override { return jobs_.cancel(ticket, confirm); }
int suggested_poll_ms() const override { return 3000; }
private:
Json input(const VideoRequest& request) const;
ReplicateJobs jobs_;
};
} // namespace molo::media
+91
View File
@@ -0,0 +1,91 @@
#include "vendors/media/runway/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"text_to_image", "POST", "/v1/text_to_image", "", "json", "model,promptText,ratio,referenceImages,contentModeration,seed,background,outputCount,quality,outputFormat,grounding,edit,maskUri,preciseEdit,thinkingLevel,backgroundColor,colors", "", "paid"},
{"text_to_video", "POST", "/v1/text_to_video", "", "json", "duration,model,promptText,ratio,contentModeration,outputFormat,proresProfile,seed,audio,negativePrompt,providerSettings,referenceAudio,references,referenceVideos,resolution,cfgScale,promptExpansionMode,draft,draftTaskId", "", "paid"},
{"image_to_video", "POST", "/v1/image_to_video", "", "json", "duration,model,promptImage,promptText,ratio,contentModeration,outputFormat,proresProfile,seed,audio,negativePrompt,providerSettings,references,referenceAudio,resolution,cfgScale,referenceVideos,promptExpansionMode,draft", "", "paid"},
{"video_to_video", "POST", "/v1/video_to_video", "", "json", "model,videoUri,contentModeration,keyframes,outputFormat,promptText,proresProfile,ratio,seed,targetAspectRatio,promptVideo,providerSettings,references,duration,referenceAudio,referenceVideos,resolution,audio,mode,draft", "", "paid"},
{"image_upscale", "POST", "/v1/image_upscale", "", "json", "imageUri,model,flavor,scaleFactor,sharpen,smartGrain,ultraDetail", "", "paid"},
{"video_upscale", "POST", "/v1/video_upscale", "", "json", "model,videoUri,creativity,flavor,fpsBoost,resolution,sharpen,smartGrain,targetFramerate", "", "paid"},
{"video_to_hdr", "POST", "/v1/video_to_hdr", "", "json", "model,videoUri,outputFormat,proresProfile", "", "paid"},
{"video_to_audio", "POST", "/v1/video_to_audio", "", "json", "duration,model,videoUri,format,negativePrompt,numVariants,preserveSpeech,promptText,seed", "", "paid"},
{"character_performance", "POST", "/v1/character_performance", "", "json", "type,uri,publicFigureThreshold,character,model,reference,bodyControl,contentModeration,expressionIntensity,ratio,seed", "", "paid"},
{"sound_effect", "POST", "/v1/sound_effect", "", "json", "model,promptText,loudnessRate,outputFormat,pitchRate,referenceAudios,sampleRate,speechRate,duration,format,negativePrompt,numVariants,seed,loop", "", "paid"},
{"speech_to_speech", "POST", "/v1/speech_to_speech", "", "json", "type,uri,presetId,media,model,voice,removeBackgroundNoise", "", "paid"},
{"text_to_speech", "POST", "/v1/text_to_speech", "", "json", "model,promptText,loudnessRate,outputFormat,pitchRate,sampleRate,speechRate,voice,languageCode,nextText,previousText,seed,similarityBoost,speed,stability,style,useSpeakerBoost,applyTextNormalization", "", "paid"},
{"voice_dubbing", "POST", "/v1/voice_dubbing", "", "json", "audioUri,model,targetLang,disableVoiceCloning,dropBackgroundAudio,numSpeakers", "", "paid"},
{"voice_isolation", "POST", "/v1/voice_isolation", "", "json", "audioUri,model", "", "paid"},
{"avatar_videos", "POST", "/v1/avatar_videos", "", "json", "presetId,type,avatarId,audio,text,voice,avatar,model,speech", "", "paid"},
{"generate.image", "POST", "/v1/generate/image", "", "json", "promptText,aspectRatio,contentModeration,outputCount,referenceImages,resolution,seed,configId,input", "", "paid"},
{"generate.video", "POST", "/v1/generate/video", "", "json", "aspectRatio,audio,contentModeration,duration,keyframes,negativePrompt,promptText,referenceAudio,referenceImages,referenceVideos,resolution,seed,configId,input", "", "paid"},
{"generate.audio", "POST", "/v1/generate/audio", "", "json", "promptText,type,duration,loop,referenceAudios,voice,configId,input", "", "paid"},
{"recipes.ad_localization", "POST", "/v1/recipes/ad_localization", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"recipes.marketing_stock_image", "POST", "/v1/recipes/marketing_stock_image", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"recipes.multi_shot_video", "POST", "/v1/recipes/multi_shot_video", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"recipes.product_ad", "POST", "/v1/recipes/product_ad", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"recipes.product_campaign_image", "POST", "/v1/recipes/product_campaign_image", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"recipes.product_swap", "POST", "/v1/recipes/product_swap", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"recipes.product_ugc", "POST", "/v1/recipes/product_ugc", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
{"tasks.get", "GET", "/v1/tasks/{id}", "", "none", "", "", "read"},
{"tasks.delete", "DELETE", "/v1/tasks/{id}", "", "none", "", "", "delete"},
{"organization.get", "GET", "/v1/organization", "", "none", "", "", "read"},
{"organization.usage", "POST", "/v1/organization/usage", "", "json", "beforeDate,startDate", "", "read"},
{"uploads.create", "POST", "/v1/uploads", "", "json", "file,fileMetadata", "", "write"},
{"avatars.create", "POST", "/v1/avatars", "", "json", "presetId,type,id,name,personality,referenceImage,voice,documentIds,imageProcessing,startScript,endDate,startDate", "", "write"},
{"avatars.list", "GET", "/v1/avatars", "", "none", "", "", "read"},
{"avatars.get", "GET", "/v1/avatars/{id}", "", "none", "", "", "read"},
{"avatars.update", "PATCH", "/v1/avatars/{id}", "", "json", "presetId,type,id,name,personality,referenceImage,voice,documentIds,imageProcessing,startScript,endDate,startDate", "", "write"},
{"avatars.delete", "DELETE", "/v1/avatars/{id}", "", "none", "", "", "delete"},
{"avatar_usage", "GET", "/v1/avatar_usage", "", "none", "", "", "read"},
{"avatar_conversations.list", "GET", "/v1/avatar_conversations", "", "none", "", "", "read"},
{"avatar_conversations.get", "GET", "/v1/avatar_conversations/{id}", "", "none", "", "", "read"},
{"avatar_conversations.delete", "DELETE", "/v1/avatar_conversations/{id}", "", "none", "", "", "delete"},
{"documents.create", "POST", "/v1/documents", "", "json", "content,name", "", "write"},
{"documents.list", "GET", "/v1/documents", "", "none", "", "", "read"},
{"documents.get", "GET", "/v1/documents/{id}", "", "none", "", "", "read"},
{"documents.update", "PATCH", "/v1/documents/{id}", "", "json", "content,name", "", "write"},
{"documents.delete", "DELETE", "/v1/documents/{id}", "", "none", "", "", "delete"},
{"voices.create", "POST", "/v1/voices", "", "json", "audio,type,model,prompt,from,name,description", "", "write"},
{"voices.preview", "POST", "/v1/voices/preview", "", "json", "audio,type,model,prompt,from,name,description", "", "paid"},
{"voices.list", "GET", "/v1/voices", "", "none", "", "", "read"},
{"voices.get", "GET", "/v1/voices/{id}", "", "none", "", "", "read"},
{"voices.update", "PATCH", "/v1/voices/{id}", "", "json", "audio,type,model,prompt,from,name,description", "", "write"},
{"voices.delete", "DELETE", "/v1/voices/{id}", "", "none", "", "", "delete"},
{"realtime_sessions.create", "POST", "/v1/realtime_sessions", "", "json", "presetId,type,avatarId,signedUrl,token,roomName,url,agentIdentity,description,name,parameters,timeoutSeconds,avatar,model,integration,livekit,maxDuration,personality,startScript,tools", "", "paid"},
{"realtime_sessions.get", "GET", "/v1/realtime_sessions/{id}", "", "none", "", "", "read"},
{"realtime_sessions.delete", "DELETE", "/v1/realtime_sessions/{id}", "", "none", "", "", "delete"},
{"routers.create", "POST", "/v1/routers", "", "json", "fallback,maxCreditsPerGeneration,models,optimizeFor,schemaVersion,slug,description,name,settings", "", "write"},
{"routers.list", "GET", "/v1/routers", "", "none", "", "", "read"},
{"routers.get", "GET", "/v1/routers/{id}", "", "none", "", "", "read"},
{"routers.update", "PATCH", "/v1/routers/{id}", "", "json", "fallback,maxCreditsPerGeneration,models,optimizeFor,schemaVersion,slug,description,name,settings", "", "write"},
{"routers.delete", "DELETE", "/v1/routers/{id}", "", "none", "", "", "delete"},
{"routers.requests", "GET", "/v1/routers/{id}/requests", "", "none", "", "", "read"},
{"workflows.list", "GET", "/v1/workflows", "", "none", "", "", "read"},
{"workflows.get", "GET", "/v1/workflows/{id}", "", "none", "", "", "read"},
{"workflows.invoke", "POST", "/v1/workflows/{id}", "", "json", "type,value,uri,nodeOutputs", "", "paid"},
{"workflow_invocations.get", "GET", "/v1/workflow_invocations/{id}", "", "none", "", "", "read"},
{"audit_logs.list", "GET", "/v1/organization/webapp/audit_logs", "", "none", "", "", "read"},
{"audit_logs.get", "GET", "/v1/organization/webapp/audit_logs/{id}", "", "none", "", "", "read"},
{"webapp_usage.list", "GET", "/v1/organization/webapp/usage", "", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"TaskRetrieveResponse", "id,status,createdAt,output,failure,failureCode,progress"},
};
// FIELDS-END
const CallTable& RunwayCalls::table() {
static const CallTable t("runway", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t RunwayCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* RunwayCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct RunwayCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+163
View File
@@ -0,0 +1,163 @@
#include "vendors/media/runway/runway.h"
#include "vendors/media/common/adapter_support.h"
#include "vendors/media/common/output_scanner.h"
namespace molo::media {
namespace {
CallArgs with_body(const Json& body) {
CallArgs a;
a.body = body;
return a;
}
CallArgs by_id(const std::string& id) {
CallArgs a;
a.path["id"] = id;
return a;
}
std::string ratio_of(const std::string& aspect, int width, int height) {
if (!aspect.empty()) return aspect;
if (width && height) return std::to_string(width) + ":" + std::to_string(height);
return "";
}
} // namespace
PreparedCall RunwayClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
Json RunwayClient::submit(const std::string& key, const Json& body, bool confirm) {
return call_json(key, with_body(body), confirm);
}
Json RunwayClient::task(const std::string& id) { return call_json("tasks.get", by_id(id)); }
Json RunwayClient::cancel_or_delete_task(const std::string& id, bool confirm) {
return call_json("tasks.delete", by_id(id), confirm);
}
Json RunwayClient::organization() { return call_json("organization.get"); }
Json RunwayClient::organization_usage(const Json& body) {
PreparedCall c = prepare("organization.usage", with_body(body));
c.effect = Effect::kRead; // a usage query: the POST creates nothing and costs nothing
return parse_json("runway", execute(c, false));
}
JobTicket RunwayJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) {
JobTicket t;
t.provider = "runway";
t.model = model;
t.kind = kind;
t.id = AdapterSupport::str(reply, "id");
if (t.id.empty()) throw llm::ApiError("runway", 502, "BAD_RESPONSE", "no id in the task reply");
return t;
}
JobStatus RunwayJobs::status_of(const Json& task, MediaKind kind) {
JobStatus s;
s.raw_state = AdapterSupport::str(task, "status");
if (s.raw_state == "SUCCEEDED") {
s.state = JobState::kSucceeded;
if (const Json* o = task.find("output")) s.assets = OutputScanner::scan(*o, kind);
if (s.assets.empty()) {
s.state = JobState::kFailed;
s.error = "succeeded without output URLs";
}
} else if (s.raw_state == "FAILED") {
s.state = JobState::kFailed;
s.error = AdapterSupport::str(task, "failure");
std::string code = AdapterSupport::str(task, "failureCode");
if (!code.empty()) s.error += " (" + code + ")";
} else if (s.raw_state == "CANCELLED") {
s.state = JobState::kCancelled;
} else if (s.raw_state == "RUNNING") {
s.state = JobState::kRunning;
if (const Json* p = task.find("progress"); p && p->is_number()) s.progress = p->as_number();
} else {
s.state = JobState::kQueued; // PENDING, THROTTLED
}
return s;
}
JobStatus RunwayJobs::poll(RunwayClient& client, const JobTicket& ticket) {
return status_of(client.task(ticket.id), ticket.kind);
}
Json RunwayImageGenerator::body(const ImageRequest& r) const {
r.validate();
if (r.count != 1) throw llm::ApiError::invalid("runway", "one image per task (count must be 1)");
Json b = Json::object();
b.set("model", Json(r.model.empty() ? "gen4_image" : r.model));
AdapterSupport::put(b, "promptText", r.prompt);
AdapterSupport::put(b, "ratio", ratio_of(r.aspect_ratio, r.width, r.height));
AdapterSupport::put_seed(b, "seed", r.seed);
if (!r.source_url.empty()) {
Json refs = Json::array();
Json one = Json::object();
one.set("uri", Json(r.source_url));
one.set("tag", Json("source"));
refs.push(one);
b.set("referenceImages", refs);
}
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json RunwayImageGenerator::plan(const ImageRequest& request) const {
return client_->plan("text_to_image", body(request)).summary();
}
JobTicket RunwayImageGenerator::submit(const ImageRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "runway", "generate");
return RunwayJobs::ticket_from(client_->submit("text_to_image", b, true), b.find("model")->as_string(), MediaKind::kImage);
}
JobStatus RunwayImageGenerator::poll(const JobTicket& t) { return RunwayJobs::poll(*client_, t); }
bool RunwayImageGenerator::cancel(const JobTicket& t, bool confirm) {
AdapterSupport::need_confirm(confirm, "runway", "cancel task");
client_->cancel_or_delete_task(t.id, true);
return true;
}
std::string RunwayVideoGenerator::key_for(const VideoRequest& r) {
return r.image_url.empty() ? "text_to_video" : "image_to_video";
}
Json RunwayVideoGenerator::body(const VideoRequest& r) const {
r.validate();
Json b = Json::object();
b.set("model", Json(r.model.empty() ? (r.image_url.empty() ? "gen4.5" : "gen4_turbo") : r.model));
AdapterSupport::put(b, "promptText", r.prompt);
AdapterSupport::put(b, "ratio", r.aspect_ratio);
AdapterSupport::put_int(b, "duration", r.duration_seconds);
AdapterSupport::put_seed(b, "seed", r.seed);
AdapterSupport::put(b, "negativePrompt", r.negative_prompt);
AdapterSupport::put(b, "resolution", r.resolution);
AdapterSupport::put(b, "promptImage", r.image_url);
if (!r.end_image_url.empty())
throw llm::ApiError::invalid("runway", "end frames use per-model promptImage arrays: pass them in extra");
AdapterSupport::merge_extra(b, r.extra);
return b;
}
Json RunwayVideoGenerator::plan(const VideoRequest& request) const {
return client_->plan(key_for(request), body(request)).summary();
}
JobTicket RunwayVideoGenerator::submit(const VideoRequest& request, bool confirm) {
Json b = body(request);
AdapterSupport::need_confirm(confirm, "runway", "generate");
return RunwayJobs::ticket_from(client_->submit(key_for(request), b, true), b.find("model")->as_string(), MediaKind::kVideo);
}
JobStatus RunwayVideoGenerator::poll(const JobTicket& t) { return RunwayJobs::poll(*client_, t); }
bool RunwayVideoGenerator::cancel(const JobTicket& t, bool confirm) {
AdapterSupport::need_confirm(confirm, "runway", "cancel task");
client_->cancel_or_delete_task(t.id, true);
return true;
}
} // namespace molo::media
+93
View File
@@ -0,0 +1,93 @@
#pragma once
// Runway developer API (https://api.dev.runwayml.com): text / image / video
// generation, tasks, uploads, organization, avatars, voices, documents,
// routers, workflows, realtime sessions. Bearer auth plus the required
// X-Runway-Version header. The key is RUNWAY_API_KEY (the official SDK reads
// RUNWAYML_API_SECRET; both are accepted). Runway publishes no OpenAPI
// document: the table is checked against the official SDK source.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/runway/calls.h"
namespace molo::media {
class RunwayClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.dev.runwayml.com";
static constexpr const char* kKeyEnv = "RUNWAY_API_KEY";
static constexpr const char* kKeyEnvSdk = "RUNWAYML_API_SECRET";
static constexpr const char* kApiVersion = "2024-11-06"; // SDK default X-Runway-Version
explicit RunwayClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), RunwayCalls::table()) {}
static llm::ApiClientConfig config() {
llm::ApiClientConfig c = ClientFactory::bearer("runway", kHost, kKeyEnv);
c.default_headers["X-Runway-Version"] = kApiVersion;
return c;
}
static std::shared_ptr<RunwayClient> from_env() {
return std::make_shared<RunwayClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv, kKeyEnvSdk})));
}
static std::shared_ptr<RunwayClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<RunwayClient>(ClientFactory::with(std::move(t), config(), key));
}
// Paid generation rows (text_to_image, text_to_video, image_to_video, ...). Returns {id, ...}.
Json submit(const std::string& key, const Json& body, bool confirm);
PreparedCall plan(const std::string& key, const Json& body) const;
Json task(const std::string& id);
Json cancel_or_delete_task(const std::string& id, bool confirm); // no default
Json organization();
Json organization_usage(const Json& body); // POST but a query: effect read
};
class RunwayImageGenerator : public ImageGenerator {
public:
explicit RunwayImageGenerator(std::shared_ptr<RunwayClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "runway"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"gen4_image", "gen4_image_turbo", "gemini_2.5_flash", "ideogram_4_5", "gpt_image_2", "seedream5_pro"};
}
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 3000; }
private:
Json body(const ImageRequest& request) const;
std::shared_ptr<RunwayClient> client_;
};
class RunwayVideoGenerator : public VideoGenerator {
public:
explicit RunwayVideoGenerator(std::shared_ptr<RunwayClient> c) : client_(std::move(c)) {}
std::string provider() const override { return "runway"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override {
return {"gen4.5", "gen4_turbo", "veo3.1", "veo3.1_fast", "kling3.0_pro", "seedance2", "hailuo3"};
}
Json plan(const VideoRequest& request) const override;
JobTicket submit(const VideoRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket& ticket, bool confirm) override;
int suggested_poll_ms() const override { return 5000; }
static std::string key_for(const VideoRequest& request);
private:
Json body(const VideoRequest& request) const;
std::shared_ptr<RunwayClient> client_;
};
struct RunwayJobs {
static JobStatus poll(RunwayClient& client, const JobTicket& ticket);
static JobStatus status_of(const Json& task, MediaKind kind);
static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind);
};
} // namespace molo::media
+61
View File
@@ -0,0 +1,61 @@
#include "vendors/media/stability/calls.h"
namespace molo::media {
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
static const CallRow kRows[] = {
{"3d.stable-fast-3d", "POST", "/v2beta/3d/stable-fast-3d", "", "multipart", "image,texture_resolution,foreground_ratio,remesh,vertex_count", "", "paid"},
{"3d.stable-point-aware-3d", "POST", "/v2beta/3d/stable-point-aware-3d", "", "multipart", "image,texture_resolution,foreground_ratio,remesh,target_type,target_count,guidance_scale,seed", "", "paid"},
{"results.id", "GET", "/v2beta/results/{id}", "", "none", "", "", "read"},
{"audio.stable-audio-2.text-to-audio", "POST", "/v2beta/audio/stable-audio-2/text-to-audio", "", "multipart", "prompt,duration,seed,steps,cfg_scale,model,output_format", "", "paid"},
{"audio.stable-audio-2.audio-to-audio", "POST", "/v2beta/audio/stable-audio-2/audio-to-audio", "", "multipart", "prompt,duration,seed,steps,cfg_scale,model,output_format,audio,strength", "", "paid"},
{"audio.stable-audio-2.inpaint", "POST", "/v2beta/audio/stable-audio-2/inpaint", "", "multipart", "prompt,duration,seed,steps,output_format,audio,mask_start,mask_end", "", "paid"},
{"audio.stable-audio.text-to-audio", "POST", "/v2beta/audio/stable-audio/text-to-audio", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format", "", "paid"},
{"audio.stable-audio.audio-to-audio", "POST", "/v2beta/audio/stable-audio/audio-to-audio", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format,strength,audio", "", "paid"},
{"audio.stable-audio.inpaint", "POST", "/v2beta/audio/stable-audio/inpaint", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format,mask_start,mask_end,audio", "", "paid"},
{"audio.results.id", "GET", "/v2beta/audio/results/{id}", "fetchAudioResult", "none", "", "", "read"},
{"stable-image.upscale.conservative", "POST", "/v2beta/stable-image/upscale/conservative", "", "multipart", "image,prompt,negative_prompt,seed,output_format,creativity", "", "paid"},
{"stable-image.upscale.creative", "POST", "/v2beta/stable-image/upscale/creative", "", "multipart", "image,prompt,negative_prompt,output_format,seed,creativity,style_preset", "", "paid"},
{"stable-image.upscale.creative.result.id", "GET", "/v2beta/stable-image/upscale/creative/result/{id}", "", "none", "", "", "read"},
{"stable-image.upscale.fast", "POST", "/v2beta/stable-image/upscale/fast", "", "multipart", "image,output_format", "", "paid"},
{"stable-image.edit.erase", "POST", "/v2beta/stable-image/edit/erase", "", "multipart", "image,mask,grow_mask,seed,output_format", "", "paid"},
{"stable-image.edit.inpaint", "POST", "/v2beta/stable-image/edit/inpaint", "", "multipart", "image,prompt,negative_prompt,mask,grow_mask,seed,output_format,style_preset", "", "paid"},
{"stable-image.edit.outpaint", "POST", "/v2beta/stable-image/edit/outpaint", "", "multipart", "image,left,right,up,down,creativity,prompt,seed,output_format,style_preset", "", "paid"},
{"stable-image.edit.search-and-replace", "POST", "/v2beta/stable-image/edit/search-and-replace", "", "multipart", "image,prompt,search_prompt,negative_prompt,grow_mask,seed,output_format,style_preset", "", "paid"},
{"stable-image.edit.search-and-recolor", "POST", "/v2beta/stable-image/edit/search-and-recolor", "", "multipart", "image,prompt,select_prompt,negative_prompt,grow_mask,seed,output_format,style_preset", "", "paid"},
{"stable-image.edit.remove-background", "POST", "/v2beta/stable-image/edit/remove-background", "", "multipart", "image,output_format", "", "paid"},
{"stable-image.edit.replace-background-and-relight", "POST", "/v2beta/stable-image/edit/replace-background-and-relight", "", "multipart", "subject_image,background_reference,background_prompt,foreground_prompt,negative_prompt,preserve_original_subject,original_background_depth,keep_original_background,light_source_direction,light_reference,light_source_strength,seed,output_format", "", "paid"},
{"stable-image.generate.ultra", "POST", "/v2beta/stable-image/generate/ultra", "", "multipart", "prompt,negative_prompt,aspect_ratio,seed,output_format,image,style_preset,strength", "", "paid"},
{"stable-image.generate.core", "POST", "/v2beta/stable-image/generate/core", "", "multipart", "prompt,aspect_ratio,negative_prompt,seed,style_preset,output_format", "", "paid"},
{"stable-image.generate.sd3", "POST", "/v2beta/stable-image/generate/sd3", "", "multipart", "prompt,mode,image,strength,aspect_ratio,model,seed,output_format,style_preset,negative_prompt,cfg_scale", "", "paid"},
{"stable-image.control.sketch", "POST", "/v2beta/stable-image/control/sketch", "", "multipart", "prompt,image,control_strength,negative_prompt,seed,output_format,style_preset", "", "paid"},
{"stable-image.control.structure", "POST", "/v2beta/stable-image/control/structure", "", "multipart", "prompt,image,control_strength,negative_prompt,seed,output_format,style_preset", "", "paid"},
{"stable-image.control.style", "POST", "/v2beta/stable-image/control/style", "", "multipart", "prompt,image,negative_prompt,aspect_ratio,fidelity,seed,output_format,style_preset", "", "paid"},
{"stable-image.control.style-transfer", "POST", "/v2beta/stable-image/control/style-transfer", "", "multipart", "init_image,style_image,prompt,negative_prompt,seed,style_strength,composition_fidelity,change_strength,output_format", "", "paid"},
{"v1.generation.id.text-to-image", "POST", "/v1/generation/{engine_id}/text-to-image", "textToImage", "json", "height,width,text_prompts,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"},
{"v1.generation.id.image-to-image", "POST", "/v1/generation/{engine_id}/image-to-image", "imageToImage", "multipart", "text_prompts,init_image,init_image_mode,image_strength,step_schedule_start,step_schedule_end,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"},
{"v1.generation.id.image-to-image.masking", "POST", "/v1/generation/{engine_id}/image-to-image/masking", "masking", "multipart", "init_image,mask_source,mask_image,text_prompts,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"},
{"v1.engines.list", "GET", "/v1/engines/list", "listEngines", "none", "", "", "read"},
{"v1.user.account", "GET", "/v1/user/account", "userAccount", "none", "", "", "read"},
{"v1.user.balance", "GET", "/v1/user/balance", "userBalance", "none", "", "", "read"},
};
// CALLS-END
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
static const char* const kFields[][2] = {
{"AsyncGenerationResponse", "id"},
{"ImageJSONResponse", "image,finish_reason,seed"},
{"AudioJSONResponse", "audio,finish_reason,seed"},
};
// FIELDS-END
const CallTable& StabilityCalls::table() {
static const CallTable t("stability", kRows, sizeof(kRows) / sizeof(kRows[0]));
return t;
}
std::size_t StabilityCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
const char* const* StabilityCalls::field_row(std::size_t i) { return kFields[i]; }
} // namespace molo::media
+19
View File
@@ -0,0 +1,19 @@
#pragma once
// The ONE table of REST calls for this provider. tools/check_spec.py checks
// every row against the published spec. Code builds every path from here.
#include <cstddef>
#include "vendors/media/common/call_table.h"
namespace molo::media {
struct StabilityCalls {
static const CallTable& table();
// Response fields our parsers read, per schema (checked by check_spec.py).
static std::size_t field_count();
static const char* const* field_row(std::size_t index);
};
} // namespace molo::media
+141
View File
@@ -0,0 +1,141 @@
#include "vendors/media/stability/stability.h"
#include "vendors/media/common/adapter_support.h"
namespace molo::media {
namespace {
CallArgs form(const Json& fields, const std::vector<FilePart>& files) {
CallArgs a;
a.body = fields;
a.files = files;
a.headers["Accept"] = "application/json";
return a;
}
CallArgs by_id(const std::string& id) {
CallArgs a;
a.path["id"] = id;
a.headers["Accept"] = "application/json";
return a;
}
} // namespace
PreparedCall StabilityClient::plan(const std::string& key, const Json& fields,
const std::vector<FilePart>& files) const {
return prepare(key, form(fields, files));
}
Json StabilityClient::submit(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
bool confirm) {
return call_json(key, form(fields, files), confirm);
}
Json StabilityClient::result(const std::string& id) { return call_json("results.id", by_id(id)); }
Json StabilityClient::audio_result(const std::string& id) { return call_json("audio.results.id", by_id(id)); }
Json StabilityClient::creative_upscale_result(const std::string& id) {
return call_json("stable-image.upscale.creative.result.id", by_id(id));
}
Json StabilityClient::engines() { return call_json("v1.engines.list"); }
Json StabilityClient::account() { return call_json("v1.user.account"); }
Json StabilityClient::balance() { return call_json("v1.user.balance"); }
Json StabilityClient::v1_text_to_image(const std::string& engine_id, const Json& body, bool confirm) {
CallArgs a;
a.path["engine_id"] = engine_id;
a.body = body;
a.headers["Accept"] = "application/json";
return call_json("v1.generation.id.text-to-image", a, confirm);
}
std::vector<std::string> StabilityImageGenerator::models() const {
return {"core", "ultra", "sd3.5-large", "sd3.5-large-turbo", "sd3.5-medium", "sd3.5-flash"};
}
std::string StabilityImageGenerator::key_for(const std::string& model) {
if (model.empty() || model == "core") return "stable-image.generate.core";
if (model == "ultra") return "stable-image.generate.ultra";
if (model.rfind("sd3", 0) == 0) return "stable-image.generate.sd3";
throw llm::ApiError::invalid("stability", "unknown model '" + model + "' (core, ultra, sd3*)");
}
Json StabilityImageGenerator::fields(const ImageRequest& r, std::string& key) const {
r.validate();
key = key_for(r.model);
if (r.count != 1) throw llm::ApiError::invalid("stability", "one image per call (count must be 1)");
if (r.width || r.height) throw llm::ApiError::invalid("stability", "use aspect_ratio, not width / height");
if (!r.source_url.empty())
throw llm::ApiError::invalid("stability", "source images are uploaded as files: use StabilityClient::submit with a FilePart");
Json f = Json::object();
AdapterSupport::put(f, "prompt", r.prompt);
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio);
if (r.seed >= 0) f.set("seed", Json(static_cast<double>(r.seed > 4294967294L ? 4294967294L : r.seed)));
if (key == "stable-image.generate.sd3" && r.model != "sd3") f.set("model", Json(r.model));
f.set("output_format", Json("png"));
AdapterSupport::merge_extra(f, r.extra);
return f;
}
Json StabilityImageGenerator::plan(const ImageRequest& request) const {
std::string key;
Json f = fields(request, key);
return client_->plan(key, f, {}).summary();
}
JobTicket StabilityImageGenerator::submit(const ImageRequest& request, bool confirm) {
std::string key;
Json f = fields(request, key);
AdapterSupport::need_confirm(confirm, "stability", "generate");
Json out = client_->submit(key, f, {}, true);
JobTicket t;
t.provider = "stability";
t.model = request.model.empty() ? "core" : request.model;
t.kind = MediaKind::kImage;
if (const Json* id = out.find("id"); id && id->is_string()) {
t.id = id->as_string();
return t;
}
std::string b64 = AdapterSupport::str(out, "image");
if (b64.empty()) throw llm::ApiError("stability", 502, "BAD_RESPONSE", "no image and no id in the reply");
MediaAsset a;
a.kind = MediaKind::kImage;
a.data_base64 = b64;
a.mime = "image/" + (f.find("output_format") ? f.find("output_format")->as_string() : std::string("png"));
a.label = "seed=" + AdapterSupport::str(out, "seed") + " finish_reason=" + AdapterSupport::str(out, "finish_reason");
t.id = "immediate";
t.immediate = true;
t.assets.push_back(a);
return t;
}
JobStatus StabilityImageGenerator::poll(const JobTicket& ticket) {
JobStatus s;
if (ticket.immediate) {
s.state = JobState::kSucceeded;
s.raw_state = "immediate";
s.assets = ticket.assets;
return s;
}
Json out = client_->result(ticket.id);
std::string b64 = AdapterSupport::str(out, "image");
if (b64.empty()) {
s.state = JobState::kRunning;
s.raw_state = AdapterSupport::str(out, "status");
if (s.raw_state.empty()) s.raw_state = "in-progress";
return s;
}
MediaAsset a;
a.kind = MediaKind::kImage;
a.data_base64 = b64;
a.mime = "image/png";
a.label = "seed=" + AdapterSupport::str(out, "seed") + " finish_reason=" + AdapterSupport::str(out, "finish_reason");
s.state = JobState::kSucceeded;
s.raw_state = "complete";
s.assets.push_back(a);
return s;
}
} // namespace molo::media
+67
View File
@@ -0,0 +1,67 @@
#pragma once
// Stability AI: v2beta Stable Image (generate, edit, upscale, control), 3D,
// Stable Audio, and the v1 engines / account surface. Every v2beta call is
// multipart/form-data; we always ask for JSON (base64 images) so errors and
// results parse the same way.
#include "vendors/media/common/client_factory.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
#include "vendors/media/stability/calls.h"
namespace molo::media {
class StabilityClient : public MediaService {
public:
static constexpr const char* kHost = "https://api.stability.ai";
static constexpr const char* kKeyEnv = "STABILITY_API_KEY";
explicit StabilityClient(std::shared_ptr<llm::ApiClient> client)
: MediaService(std::move(client), StabilityCalls::table()) {}
static llm::ApiClientConfig config() { return ClientFactory::bearer("stability", kHost, kKeyEnv); }
static std::shared_ptr<StabilityClient> from_env() {
return std::make_shared<StabilityClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
}
static std::shared_ptr<StabilityClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
return std::make_shared<StabilityClient>(ClientFactory::with(std::move(t), config(), key));
}
// Any v2beta POST row (stable-image.*, 3d.*, audio.*). Paid: confirm required.
// Returns {image|audio, finish_reason, seed} or, for async rows, {id}.
Json submit(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm);
PreparedCall plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files) const;
// Polls: HTTP 202 {id,status} while running, 200 with the result when done.
Json result(const std::string& id); // /v2beta/results/{id}
Json audio_result(const std::string& id); // /v2beta/audio/results/{id}
Json creative_upscale_result(const std::string& id); // legacy creative upscale path
// v1 reads.
Json engines();
Json account();
Json balance();
// v1 text-to-image (JSON body, artifacts[] with base64).
Json v1_text_to_image(const std::string& engine_id, const Json& body, bool confirm);
};
class StabilityImageGenerator : public ImageGenerator {
public:
explicit StabilityImageGenerator(std::shared_ptr<StabilityClient> client) : client_(std::move(client)) {}
std::string provider() const override { return "stability"; }
bool configured() const override { return client_->configured(); }
std::vector<std::string> models() const override;
Json plan(const ImageRequest& request) const override;
JobTicket submit(const ImageRequest& request, bool confirm) override;
JobStatus poll(const JobTicket& ticket) override;
bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the API
// model -> table key (ultra, core, sd3*), "" model = core.
static std::string key_for(const std::string& model);
private:
Json fields(const ImageRequest& request, std::string& key) const;
std::shared_ptr<StabilityClient> client_;
};
} // namespace molo::media
+578
View File
@@ -0,0 +1,578 @@
#!/usr/bin/env python3
"""Check the media call tables against what each vendor publishes.
For every provider the checker reads the rows between CALLS-BEGIN / CALLS-END
in src/vendors/media/<provider>/calls.cpp (and the response fields between
FIELDS-BEGIN / FIELDS-END) and compares them with the vendor's own source:
openapi Stability (v2beta + v1), Black Forest Labs, Replicate, fal (queue
spec from the docs, Platform API), Ideogram v2, Recraft, Higgsfield
sdk Runway and Luma: the official npm package (resources/*.js for the
paths, resources/*.d.ts for the request fields)
docs Leonardo v2 (the OpenAPI document embedded in three reference
pages), MiniMax (the OpenAPI embedded in the API reference pages)
probe Kling (no machine-readable source exists): the path must answer
401/400/403 unauthenticated, never 404. Reported as UNVERIFIED.
It fails (exit 1) when
row a declared (method, path) is not an operation of the source
field a request body field or query parameter we send is not in the source
kind the declared body kind disagrees with the source content types
schema a response field we read is not reachable from the named schema
coverage a source operation is neither in the table nor in EXCLUSIONS
Python 3 standard library only. No credentials. Network required.
python3 -I src/vendors/media/tools/check_spec.py [provider ...]
"""
import io
import json
import os
import re
import sys
import tarfile
import urllib.error
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.normpath(os.path.join(HERE, ".."))
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
UA = {"User-Agent": "molo-media-spec-check/1"}
# (provider, source operation) -> reason. Everything else in a source must be in the table.
EXCLUSIONS = {
"stability": {
("POST", "/v2alpha/generation/stable-image/upscale"): "alpha endpoint, superseded by v2beta upscale",
("GET", "/v2alpha/generation/stable-image/upscale/result/{id}"): "alpha endpoint",
("POST", "/v2alpha/generation/stable-image/inpaint"): "alpha endpoint, superseded by v2beta edit/inpaint",
},
"fal": {
("GET", "/{owner}/{app}/requests/{request_id}/status/stream"): "server-sent events; ApiClient streams POST only (gap)",
},
}
# fal Platform API operations outside model inference: out of scope for package media.
FAL_PLATFORM_SKIP_PREFIXES = ("/workflows", "/assets", "/storage", "/serverless", "/compute", "/keys", "/account",
"/organization")
# Rows whose path is generic by design (the vendor says its OpenAPI file is supplementary).
GENERIC_ROWS = {"higgsfield": {"model.submit"}}
# Rows checked by documentation prose only (the reference page was removed). Listed, never silent.
PROSE_ONLY = {"leonardo": {"v1.generations.create", "v1.generations.get", "v1.me"}}
problems = []
notes = []
cache = {}
def fail(provider, kind, msg):
problems.append("%s: %s: %s" % (provider, kind, msg))
def fetch(url, binary=False):
if url not in cache:
req = urllib.request.Request(url, headers=UA)
with urllib.request.urlopen(req, timeout=90) as r:
cache[url] = r.read()
data = cache[url]
return data if binary else data.decode("utf-8", "replace")
def rows_of(provider):
text = open(os.path.join(ROOT, provider, "calls.cpp")).read()
a, b = text.index("// CALLS-BEGIN"), text.index("// CALLS-END")
rows = []
for line in text[a:b].splitlines()[2:]:
s = line.strip()
if s.startswith("{") and '"' in s:
f = STR.findall(s)
assert len(f) == 8, (provider, s)
rows.append(dict(zip(("key", "method", "path", "op", "kind", "body", "query", "effect"), f)))
c, d = text.index("// FIELDS-BEGIN"), text.index("// FIELDS-END")
fields = []
for line in text[c:d].splitlines()[1:]:
s = line.strip()
if s.startswith("{") and '"' in s:
fields.append(STR.findall(s))
for r in rows:
r["body"] = [x for x in r["body"].split(",") if x]
r["query"] = [x for x in r["query"].split(",") if x]
return rows, fields
# ---------------------------------------------------------------- OpenAPI ----
class Oas:
def __init__(self, doc):
self.d = doc
def ref(self, node):
n = 0
while isinstance(node, dict) and "$ref" in node and n < 25:
cur = self.d
for part in node["$ref"].lstrip("#/").split("/"):
cur = cur[part.replace("~1", "/").replace("~0", "~")]
node = cur
n += 1
return node
def props(self, schema, depth=0):
schema = self.ref(schema)
out = []
if not isinstance(schema, dict) or depth > 6:
return out
for name in schema.get("properties") or {}:
out.append(name)
for key in ("allOf", "oneOf", "anyOf"):
for sub in schema.get(key) or []:
out.extend(self.props(sub, depth + 1))
return out
def reachable(self, schema, depth=0, seen=None):
"""Every property name reachable from a schema (for response fields)."""
seen = seen if seen is not None else set()
out = set()
if depth > 6:
return out
if isinstance(schema, dict) and "$ref" in schema:
if schema["$ref"] in seen:
return out
seen.add(schema["$ref"])
schema = self.ref(schema)
if not isinstance(schema, dict):
return out
for name, sub in (schema.get("properties") or {}).items():
out.add(name)
out |= self.reachable(sub, depth + 1, seen)
for key in ("allOf", "oneOf", "anyOf"):
for sub in schema.get(key) or []:
out |= self.reachable(sub, depth + 1, seen)
if "items" in schema:
out |= self.reachable(schema["items"], depth + 1, seen)
ap = schema.get("additionalProperties")
if isinstance(ap, dict):
out |= self.reachable(ap, depth + 1, seen)
return out
def ops(self):
for path, item in self.d["paths"].items():
for m, op in item.items():
if m in ("get", "post", "put", "patch", "delete"):
yield m.upper(), path, op
def op(self, method, path):
item = self.d["paths"].get(path)
return item.get(method.lower()) if item else None
def params(self, path, op, where):
out = []
for p in (self.d["paths"][path].get("parameters") or []) + (op.get("parameters") or []):
p = self.ref(p)
if p.get("in") == where:
out.append(p["name"])
return out
def body(self, op):
rb = self.ref(op.get("requestBody") or {})
content = rb.get("content") or {}
kinds, fields = [], []
for ct, k in (("application/json", "json"), ("multipart/form-data", "multipart"),
("application/x-www-form-urlencoded", "form")):
if ct in content:
kinds.append(k)
fields.extend(self.props(content[ct].get("schema") or {}))
return kinds, fields
def check_rows_against_oas(provider, rows, oas, strip="", only=lambda r: True, path_filter=lambda p: True):
"""Compare table rows with one OpenAPI document. Returns the (method, path) set covered."""
covered = set()
for r in rows:
if not only(r):
continue
if r["key"] in GENERIC_ROWS.get(provider, ()):
notes.append("%s: row %s is generic by design (not in the spec)" % (provider, r["key"]))
continue
path = r["path"][len(strip):] if strip and r["path"].startswith(strip) else r["path"]
op = oas.op(r["method"], path)
if op is None:
fail(provider, "row", "%s %s (%s) is not in the spec" % (r["method"], r["path"], r["key"]))
continue
covered.add((r["method"], path))
if r["op"] and op.get("operationId") != r["op"]:
fail(provider, "row", "%s declares operation id %s, spec says %s" % (r["key"], r["op"], op.get("operationId")))
kinds, fields = oas.body(op)
want = r["kind"]
have = "none" if not kinds else (kinds[0] if len(kinds) == 1 else "either")
if want != have:
fail(provider, "kind", "%s declares body kind %s, spec has %s" % (r["key"], want, have))
for f in r["body"]:
if f != "*" and f not in fields:
fail(provider, "field", "%s sends body field %s, absent from the spec" % (r["key"], f))
q = oas.params(path, op, "query")
for f in r["query"]:
if f not in q:
fail(provider, "field", "%s sends query parameter %s, absent from the spec" % (r["key"], f))
return covered
def check_coverage(provider, oas, covered, strip="", path_filter=lambda p: True):
excl = EXCLUSIONS.get(provider, {})
n = 0
for m, p, _op in oas.ops():
if not path_filter(p):
continue
n += 1
if (m, p) in covered or (m, p) in excl:
continue
fail(provider, "coverage", "spec operation %s %s is neither in the table nor excluded" % (m, p))
return n
def check_fields(provider, fields, oas_list):
for schema, flist in fields:
want = [x for x in flist.split(",") if x]
ok = False
for oas in oas_list:
schemas = oas.d.get("components", {}).get("schemas", {})
if schema in schemas:
have = oas.reachable({"$ref": "#/components/schemas/" + schema})
missing = [f for f in want if f not in have]
for f in missing:
fail(provider, "schema", "response field %s is not reachable from schema %s" % (f, schema))
ok = True
break
if not ok:
fail(provider, "schema", "schema %s not found in the spec" % schema)
def summarize(provider, rows, ops, kind):
nb = sum(len(r["body"]) + len(r["query"]) for r in rows)
print(" %-11s %-8s rows=%-3d fields=%-4d source ops=%s" % (provider, kind, len(rows), nb, ops))
# ------------------------------------------------------------- providers ----
def stability():
rows, fields = rows_of("stability")
v2 = Oas(json.loads(fetch("https://docs.stability.ai/openapi.json")))
v1 = Oas(json.loads(fetch("https://docs.stability.ai/openapi-v1.json")))
c2 = check_rows_against_oas("stability", rows, v2, only=lambda r: r["path"].startswith("/v2beta"))
c1 = check_rows_against_oas("stability", rows, v1, only=lambda r: r["path"].startswith("/v1"))
n = check_coverage("stability", v2, c2) + check_coverage("stability", v1, c1)
check_fields("stability", fields, [v2, v1])
summarize("stability", rows, n, "openapi")
def bfl():
rows, fields = rows_of("bfl")
oas = Oas(json.loads(fetch("https://api.bfl.ai/openapi.json")))
cov = check_rows_against_oas("bfl", rows, oas)
n = check_coverage("bfl", oas, cov)
check_fields("bfl", fields, [oas])
summarize("bfl", rows, n, "openapi")
def replicate():
rows, fields = rows_of("replicate")
oas = Oas(json.loads(fetch("https://api.replicate.com/openapi.json")))
servers = [s.get("url") for s in oas.d.get("servers", [])]
if servers != ["https://api.replicate.com/v1"]:
fail("replicate", "row", "spec server url changed: %s (kHost assumes https://api.replicate.com/v1)" % servers)
cov = check_rows_against_oas("replicate", rows, oas)
n = check_coverage("replicate", oas, cov)
check_fields("replicate", fields, [oas])
summarize("replicate", rows, n, "openapi")
def fal():
rows, fields = rows_of("fal")
queue = Oas(json.loads(fetch("https://fal.ai/docs/api-reference/inference-api/openapi/queue.json")))
plat = Oas(json.loads(fetch("https://api.fal.ai/v1/openapi.json")))
cq = check_rows_against_oas("fal", rows, queue, only=lambda r: r["key"].startswith("queue."))
cp = check_rows_against_oas("fal", rows, plat, strip="/v1", only=lambda r: r["key"].startswith("platform."))
n = check_coverage("fal", queue, cq) + check_coverage(
"fal", plat, cp, path_filter=lambda p: not p.startswith(FAL_PLATFORM_SKIP_PREFIXES))
skipped = [p for _m, p, _o in plat.ops() if p.startswith(FAL_PLATFORM_SKIP_PREFIXES)]
notes.append("fal: %d Platform API operations outside model inference are out of scope (assets, storage, "
"serverless, compute, keys, account, organization, workflows)" % len(skipped))
check_fields("fal", fields, [queue, plat])
summarize("fal", rows, n, "openapi")
def ideogram():
rows, fields = rows_of("ideogram")
oas = Oas(json.loads(fetch("https://developer.ideogram.ai/v2/openapi.json")))
cov = check_rows_against_oas("ideogram", rows, oas)
n = check_coverage("ideogram", oas, cov)
check_fields("ideogram", fields, [oas])
summarize("ideogram", rows, n, "openapi")
def recraft():
rows, fields = rows_of("recraft")
oas = Oas(json.loads(fetch("https://www.recraft.ai/docs/api-reference/openapi.json")))
cov = check_rows_against_oas("recraft", rows, oas)
n = check_coverage("recraft", oas, cov)
check_fields("recraft", fields, [oas])
summarize("recraft", rows, n, "openapi")
def higgsfield():
rows, fields = rows_of("higgsfield")
oas = Oas(json.loads(fetch("https://docs.higgsfield.ai/docs/openapi.json")))
cov = check_rows_against_oas("higgsfield", rows, oas)
n = check_coverage("higgsfield", oas, cov)
check_fields("higgsfield", fields, [oas])
summarize("higgsfield", rows, n, "openapi")
# ------------------------------------------------------- docs-embedded ----
def md_openapi(url):
text = fetch(url)
m = re.search(r"```json\n(.*?)\n```", text, re.S)
return Oas(json.loads(m.group(1)))
def leonardo():
rows, fields = rows_of("leonardo")
pages = {"createGeneration": "creategeneration", "createSyncGeneration": "createsyncgeneration", "getModels": "getmodels"}
docs = [md_openapi("https://docs.leonardo.ai/reference/%s.md" % p) for p in pages.values()]
covered = set()
n = 0
for oas in docs:
for m, p, _op in oas.ops():
n += 1
covered.add((m, "/v2" + p))
prose = PROSE_ONLY["leonardo"]
for r in rows:
if r["key"] in prose:
continue
hit = None
for oas in docs:
op = oas.op(r["method"], r["path"][len("/v2"):])
if op is not None:
hit = (oas, op)
if hit is None:
fail("leonardo", "row", "%s %s (%s) is not in the v2 reference" % (r["method"], r["path"], r["key"]))
continue
oas, op = hit
if r["op"] and op.get("operationId") != r["op"]:
fail("leonardo", "row", "%s operation id %s != %s" % (r["key"], r["op"], op.get("operationId")))
kinds, have = oas.body(op)
if r["body"] and not (set(r["body"]) <= set(have) | {"model", "parameters", "public"}):
fail("leonardo", "field", "%s body fields %s" % (r["key"], r["body"]))
for key in sorted(prose):
notes.append("leonardo: row %s is UNVERIFIED against a spec (v1 reference pages were removed); it is "
"documented in the quick start (POST /v1/generations) and FAQ prose (GET generation by id)" % key)
quick = fetch("https://docs.leonardo.ai/docs/getting-started.md")
if "api/rest/v1/generations" not in quick:
fail("leonardo", "row", "quick start no longer shows POST /api/rest/v1/generations")
summarize("leonardo", rows, n, "docs")
MINIMAX_PAGES = ["video-generation-t2v", "video-generation-i2v", "video-generation-fl2v", "video-generation-s2v",
"video-generation-query", "video-generation-v2-create", "video-generation-v2-query",
"video-generation-v2-list", "video-generation-v2-delete"]
MINIMAX_FILE_PAGE = "file-management-retrieve"
MINIMAX_UNSUPPORTED_PAGES = {
"video-generation-v2-h3-context-ir": "H3-Context-IR task (gap)",
"video-generation-v2-regeneration": "video regeneration task (gap)",
"video-agent-create": "Video Agent templates (gap)",
"video-agent-query": "Video Agent templates (gap)",
"video-generation-download": "download helper page, covered by files.retrieve",
}
def minimax():
rows, _fields = rows_of("minimax")
base = "https://platform.minimax.io/docs/api-reference/"
texts = {p: fetch(base + p + ".md") for p in MINIMAX_PAGES + [MINIMAX_FILE_PAGE]}
heads = set()
page_text = {}
for p, t in texts.items():
for m in re.finditer(r"^````yaml \S+ (get|post|put|patch|delete) (/\S+)$", t, re.M | re.I):
heads.add((m.group(1).upper(), m.group(2)))
page_text.setdefault((m.group(1).upper(), m.group(2)), "")
page_text[(m.group(1).upper(), m.group(2))] += t
for r in rows:
key = (r["method"], r["path"])
if key not in heads:
fail("minimax", "row", "%s %s (%s) is not an operation in the API reference pages" % (r["method"], r["path"], r["key"]))
continue
t = page_text[key]
for f in r["body"]:
if not re.search(r"^\s+%s:\s*$" % re.escape(f), t, re.M):
fail("minimax", "field", "%s body field %s not found in the reference" % (r["key"], f))
for f in r["query"]:
if not re.search(r"- name: %s\b" % re.escape(f), t):
fail("minimax", "field", "%s query parameter %s not found in the reference" % (r["key"], f))
declared = {(r["method"], r["path"]) for r in rows}
for op in sorted(heads):
if op not in declared and op not in EXCLUSIONS["minimax"]:
fail("minimax", "coverage", "reference operation %s %s is neither in the table nor excluded" % op)
index = fetch("https://platform.minimax.io/docs/llms.txt")
for page in re.findall(r"api-reference/((?:video-generation|video-agent)[a-z0-9-]*)\.md", index):
if page in texts or page in MINIMAX_UNSUPPORTED_PAGES:
continue
fail("minimax", "coverage", "index lists reference page %s that is neither checked nor excluded" % page)
for page, why in sorted(MINIMAX_UNSUPPORTED_PAGES.items()):
notes.append("minimax: reference page %s not implemented: %s" % (page, why))
summarize("minimax", rows, len(heads), "docs")
# -------------------------------------------------------------------- SDK ----
def npm_tarball(pkg):
meta = json.loads(fetch("https://registry.npmjs.org/%s/latest" % pkg))
data = fetch(meta["dist"]["tarball"], binary=True)
files = {}
with tarfile.open(fileobj=io.BytesIO(data)) as tf:
for m in tf.getmembers():
if m.isfile() and m.name.startswith("package/") and m.size < 4_000_000:
files[m.name[len("package/"):]] = tf.extractfile(m).read().decode("utf-8", "replace")
return meta["version"], files
def sdk_ops(files, prefix, js_ext=".js"):
"""(METHOD, normalized path) -> resource file, from `_client.<verb>(...)` calls."""
out = {}
pat = re.compile(r"_client\.(get|post|put|patch|delete|getAPIList)\(\s*(?:\(0,\s*path_1\.path\)\s*`([^`]*)`|'([^']*)'|`([^`]*)`)")
for name, text in files.items():
if not name.startswith(prefix) or not name.endswith(js_ext) or name.endswith(".d.ts"):
continue
for m in pat.finditer(text):
verb = m.group(1)
p = m.group(2) or m.group(3) or m.group(4)
p = re.sub(r"\$\{[^}]*\}", "{id}", p)
out[("GET" if verb == "getAPIList" else verb.upper(), p)] = name
return out
def norm(path):
return re.sub(r"\{[^}]*\}", "{id}", path)
def runway_fields(text):
out = []
for m in re.finditer(r"export declare namespace (\w+Params) \{\n(.*?)\n\}\n", text, re.S):
out += re.findall(r"^ (\w+)\??:", m.group(2), re.M)
for m in re.finditer(r"export interface (\w+Params) \{\n(.*?)\n\}\n", text, re.S):
out += re.findall(r"^ (\w+)\??:", m.group(2), re.M)
return set(out)
def luma_fields(text):
out = []
for m in re.finditer(r"export interface (\w+Params) \{\n(.*?)\n\}\n", text, re.S):
out += re.findall(r"^ (\w+)\??:", m.group(2), re.M)
return set(out)
def sdk_check(provider, pkg, rows, prefix, base_strip, field_fn):
version, files = npm_tarball(pkg)
ops = sdk_ops(files, prefix)
notes.append("%s: official SDK %s %s (%d operations found)" % (provider, pkg, version, len(ops)))
declared = set()
for r in rows:
path = r["path"]
key = (r["method"], norm(path))
declared.add(key)
if key not in ops:
fail(provider, "row", "%s %s (%s) is not called by the SDK" % (r["method"], path, r["key"]))
continue
if r["body"] and r["body"] != ["*"]:
js = ops[key]
dts = files.get(js[:-3] + ".d.ts", "")
have = field_fn(dts)
for f in r["body"]:
if f not in have:
fail(provider, "field", "%s sends body field %s, absent from %s" % (r["key"], f, js[:-3] + ".d.ts"))
for key in sorted(ops):
if key not in declared and key not in EXCLUSIONS.get(provider, {}):
fail(provider, "coverage", "SDK operation %s %s is neither in the table nor excluded" % key)
return version, ops, files
def runway():
rows, fields = rows_of("runway")
version, ops, files = sdk_check("runway", "@runwayml/sdk", rows, "resources/", "", runway_fields)
tasks = files.get("resources/tasks.d.ts", "")
for schema, flist in fields:
for f in [x for x in flist.split(",") if x]:
if not re.search(r"\b%s\??:" % re.escape(f), tasks):
fail("runway", "schema", "task field %s not found in resources/tasks.d.ts" % f)
for line in ("X-Runway-Version",):
if not any(line in t for n, t in files.items() if n.endswith("client.js")):
fail("runway", "row", "SDK no longer sends %s" % line)
m = re.search(r"runwayVersion = '([^']+)'", files.get("client.js", ""))
notes.append("runway: SDK default X-Runway-Version %s (table assumes 2024-11-06)" % (m.group(1) if m else "?"))
if m and m.group(1) != "2024-11-06":
fail("runway", "row", "SDK default X-Runway-Version is %s, RunwayClient::kApiVersion says 2024-11-06" % m.group(1))
summarize("runway", rows, len(ops), "sdk")
def luma():
rows, fields = rows_of("luma")
version, ops, files = sdk_check("luma", "lumaai", rows, "resources/", "", luma_fields)
gen = files.get("resources/generations/generations.d.ts", "")
for schema, flist in fields:
for f in [x for x in flist.split(",") if x]:
if not any(re.search(r"\b%s\??:" % re.escape(f), t) for n, t in files.items() if n.endswith(".d.ts")):
fail("luma", "schema", "field %s not found in the SDK types" % f)
if "dream-machine/v1" not in files.get("index.js", ""):
fail("luma", "row", "SDK base URL is no longer .../dream-machine/v1")
if "'queued' | 'dreaming' | 'completed' | 'failed'" not in gen:
fail("luma", "schema", "generation states changed in the SDK types")
summarize("luma", rows, len(ops), "sdk")
# ------------------------------------------------------------------ probe ----
def kling():
rows, _f = rows_of("kling")
bad = 0
for r in rows:
url = "https://api-singapore.klingai.com" + r["path"].replace("{id}", "0")
data = b"{}" if r["method"] == "POST" else None
req = urllib.request.Request(url, data=data, method=r["method"], headers=dict(UA, **{"Content-Type": "application/json"}))
try:
urllib.request.urlopen(req, timeout=30)
code = 200
except urllib.error.HTTPError as e:
code = e.code
except Exception as e: # network down: do not pretend
fail("kling", "row", "probe of %s failed: %s" % (r["path"], e))
continue
if code == 404:
fail("kling", "row", "%s %s answered 404 on the documented host" % (r["method"], r["path"]))
else:
bad += 0
notes.append("kling: UNVERIFIED. No spec or SDK source exists; %d paths only probed (not 404 unauthenticated). "
"Request and response fields come from community sources." % len(rows))
summarize("kling", rows, "-", "probe")
CHECKS = {"stability": stability, "bfl": bfl, "replicate": replicate, "fal": fal, "ideogram": ideogram,
"recraft": recraft, "leonardo": leonardo, "runway": runway, "luma": luma, "minimax": minimax,
"kling": kling, "higgsfield": higgsfield}
def main():
wanted = sys.argv[1:] or list(CHECKS)
print("media call tables against vendor sources")
for name in wanted:
if name not in CHECKS:
print("unknown provider", name)
return 2
try:
CHECKS[name]()
except Exception as e: # a vendor source that cannot be read is a failure, loudly
fail(name, "source", "%s: %s" % (type(e).__name__, e))
for n in notes:
print(" note:", n)
if problems:
print("\nFAILED (%d)" % len(problems))
for p in problems:
print(" ", p)
return 1
print("\nOK: every declared row, field and schema matches its source; every source operation is covered or excluded")
return 0
if __name__ == "__main__":
sys.exit(main())
+47
View File
@@ -0,0 +1,47 @@
// Routes: GET /ai/media (page), /status, /calls (no credential needed)
#include <cstdlib>
#include "support/Assert.hpp"
using namespace molo::example::test;
int main() {
setenv("MOLO_ENV_FILES", "off", 1);
for (const char* v : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "RUNWAYML_API_SECRET", "LUMA_API_KEY",
"REPLICATE_API_TOKEN", "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY",
"MINIMAX_API_KEY", "KLING_ACCESS_KEY", "KLING_SECRET_KEY", "HF_API_KEY_ID", "HF_API_KEY_SECRET"})
unsetenv(v);
Fixture f;
auto page = f.client->get("/ai/media");
check(page.status == 200, "page 200");
check(contains(page.headers.get("Content-Type"), "text/html"), "html");
check(contains(page.body, "Media generation") && contains(page.body, "viewport"), "title and mobile viewport");
check(contains(page.body, "/embedded/ai_media.js"), "script wired");
check(f.client->post("/ai/media", "{}", "application/json").status == 405, "post page is 405");
auto st = f.client->get("/ai/media/status");
check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without keys");
check(!contains(st.body, "\"configured\":true"), "no provider is configured");
for (const char* env : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "LUMA_API_KEY", "REPLICATE_API_TOKEN",
"FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY"})
check(contains(st.body, env), std::string("status names ") + env);
check(contains(st.body, "Midjourney") && contains(st.body, "No public API"), "Midjourney is listed as unavailable with the reason");
check(contains(st.body, "Pika"), "Pika is listed as unavailable");
check(contains(st.body, "\"evidence\":\"probe-only\"") && contains(st.body, "\"evidence\":\"openapi\""), "evidence levels shown");
check(contains(st.body, "flux-2-pro") && contains(st.body, "fal-ai/flux/dev"), "models are listed");
check(contains(st.body, "\"max_poll_attempts\":240"), "caps are shown");
check(f.client->post("/ai/media/status", "{}", "application/json").status == 405, "post status is 405");
auto all = f.client->get("/ai/media/calls");
check(all.status == 200 && contains(all.body, "\"provider\":\"stability\"") && contains(all.body, "\"provider\":\"higgsfield\""),
"calls lists every provider");
auto bfl = f.client->get("/ai/media/calls?provider=bfl");
check(bfl.status == 200 && contains(bfl.body, "/v1/flux-2-pro") && contains(bfl.body, "\"effect\":\"paid\"") &&
contains(bfl.body, "/v1/credits"),
"bfl call table");
auto fal = f.client->get("/ai/media/calls?provider=fal");
check(fal.status == 200 && contains(fal.body, "queue.fal.run") == false && contains(fal.body, "/{owner}/{app}/requests/{request_id}/status"),
"fal call table");
check(f.client->get("/ai/media/calls?provider=midjourney").status == 422, "unknown provider is 422");
return finish("ai/media/page");
}
+128
View File
@@ -0,0 +1,128 @@
// Routes: POST /ai/media/* (405, 422, dry runs, 503 without a credential)
#include <cstdlib>
#include "support/Assert.hpp"
using namespace molo::example::test;
int main() {
setenv("MOLO_ENV_FILES", "off", 1);
for (const char* v : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "RUNWAYML_API_SECRET", "LUMA_API_KEY",
"REPLICATE_API_TOKEN", "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY",
"MINIMAX_API_KEY", "KLING_ACCESS_KEY", "KLING_SECRET_KEY", "HF_API_KEY_ID", "HF_API_KEY_SECRET"})
unsetenv(v);
Fixture f;
auto post = [&](const std::string& route, const std::string& body) {
return f.client->post("/ai/media/" + route, body, "application/json");
};
const char* posts[] = {"image", "video", "poll", "wait", "call", "account", "fetch", "webhook"};
for (const char* r : posts) {
std::string route = std::string("/ai/media/") + r;
check(f.client->get(route).status == 405, route + " get is 405");
check(post(r, "not json").status == 422, route + " bad json is 422");
check(post(r, "[1]").status == 422, route + " non-object is 422");
check(post(r, "{}").status == 422, route + " empty body is 422");
}
check(f.client->get("/ai/media/job").status == 405, "job get is 405");
check(f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "not json").status == 422, "job bad json is 422");
// 422: input rules run before anything else
check(post("image", "{\"provider\":\"midjourney\",\"prompt\":\"x\"}").status == 422, "midjourney has no API");
check(post("image", "{\"provider\":\"minimax\",\"prompt\":\"x\"}").status == 422, "minimax has no image generation");
check(post("video", "{\"provider\":\"stability\",\"prompt\":\"x\"}").status == 422, "stability has no video generation");
check(post("image", "{\"provider\":\"bfl\"}").status == 422, "prompt is required");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":0}").status == 422, "count 0");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":9}").status == 422, "count over 8");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"width\":512}").status == 422, "width without height");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"width\":9000,\"height\":9000}").status == 422, "dimension over the cap");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"extra\":{\"bogus\":1}}").status == 422, "field not in the call table");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"model\":\"flux-9000\"}").status == 422, "model not in the call table");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":2}").status == 422, "bfl takes one image per call");
check(post("image", "{\"provider\":\"stability\",\"prompt\":\"x\",\"width\":512,\"height\":512}").status == 422, "stability sizes by aspect ratio");
check(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\"}").status == 422, "fal needs an endpoint id");
check(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\",\"model\":\"a/../b\"}").status == 422, "bad endpoint id");
check(post("image", "{\"provider\":\"replicate\",\"prompt\":\"x\",\"model\":\"nope\"}").status == 422, "replicate needs owner/name");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"seed\":\"a\"}").status == 422, "seed must be a number");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"confirm\":\"yes\"}").status == 422, "confirm must be a boolean");
check(post("video", "{\"provider\":\"runway\"}").status == 422, "video needs a prompt or an image");
check(post("video", "{\"provider\":\"runway\",\"prompt\":\"x\",\"duration_seconds\":999}").status == 422, "duration over the cap");
check(post("video", "{\"provider\":\"minimax\",\"prompt\":\"x\",\"aspect_ratio\":\"16:9\"}").status == 422, "minimax v1 has no aspect ratio");
check(post("poll", "{\"ticket\":{\"provider\":\"bfl\"}}").status == 422, "ticket needs an id");
check(post("poll", "{\"ticket\":{\"provider\":\"nope\",\"id\":\"1\"}}").status == 422, "ticket with an unknown provider");
check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_attempts\":31}").status == 422, "demo attempt cap");
check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_wait_seconds\":61}").status == 422, "demo wait cap");
check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_attempts\":0}").status == 422, "attempts under 1");
check(post("call", "{\"provider\":\"bfl\",\"key\":\"nope\"}").status == 422, "unknown call key");
check(post("call", "{\"provider\":\"bfl\",\"key\":\"get_result\",\"query\":{\"nope\":\"1\"}}").status == 422, "unknown query parameter");
check(post("call", "{\"provider\":\"replicate\",\"key\":\"predictions.get\",\"path\":{\"prediction_id\":\"../x\"}}").status == 422, "path traversal");
check(post("call", "{\"provider\":\"replicate\",\"key\":\"predictions.get\"}").status == 422, "missing path argument");
check(post("call", "{\"provider\":\"bfl\",\"key\":\"credits\",\"body\":{\"x\":1}}").status == 422, "a GET takes no body");
check(post("call", "{\"provider\":\"stability\",\"key\":\"stable-image.upscale.fast\",\"files\":[{\"field\":\"image\",\"data_base64\":\"@@\"}]}").status == 422,
"bad base64 file");
check(post("call", "{\"provider\":\"stability\",\"key\":\"stable-image.upscale.fast\",\"files\":[{\"field\":\"picture\",\"data_base64\":\"aGk=\"}]}").status == 422,
"file field not in the table");
check(post("fetch", "{\"url\":\"http://example.com/a.png\"}").status == 422, "fetch needs https");
check(post("fetch", "{\"url\":\"https://user:pw@example.com/a.png\"}").status == 422, "fetch refuses embedded credentials");
check(post("webhook", "{\"secret\":\"s\",\"webhook_id\":\"i\",\"timestamp\":\"1\",\"signature\":\"v1,x\"}").status == 422, "webhook needs a payload");
// dry runs: paid or resource-changing routes show the exact request
auto dry = post("image", "{\"provider\":\"bfl\",\"prompt\":\"a cat\",\"width\":1024,\"height\":768}");
check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "/v1/flux-2-pro") &&
contains(dry.body, "\"effect\":\"paid\"") && contains(dry.body, "\"prompt\":\"a cat\""),
"image is a dry run without confirm");
check(contains(post("image", "{\"provider\":\"stability\",\"prompt\":\"a cat\",\"model\":\"ultra\",\"aspect_ratio\":\"16:9\"}").body,
"/v2beta/stable-image/generate/ultra"),
"stability dry run shows the endpoint");
check(contains(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\",\"model\":\"fal-ai/flux/dev\"}").body, "\"target\":\"/fal-ai/flux/dev\""),
"fal dry run shows the queue target");
check(contains(post("image", "{\"provider\":\"replicate\",\"prompt\":\"x\",\"model\":\"black-forest-labs/flux-schnell\"}").body,
"/models/black-forest-labs/flux-schnell/predictions"),
"replicate dry run shows the prediction target");
check(contains(post("image", "{\"provider\":\"leonardo\",\"prompt\":\"x\",\"model\":\"flux-dev\"}").body, "\"public\":false"),
"leonardo never publishes by default");
check(contains(post("image", "{\"provider\":\"ideogram\",\"prompt\":\"x\",\"model\":\"ideogram-4\"}").body, "/v2/image/generate/ideogram-4"),
"ideogram dry run");
check(contains(post("image", "{\"provider\":\"recraft\",\"prompt\":\"x\",\"width\":1024,\"height\":1024}").body, "\"size\":\"1024x1024\""),
"recraft dry run");
check(contains(post("video", "{\"provider\":\"runway\",\"prompt\":\"waves\",\"duration_seconds\":5}").body, "/v1/text_to_video"),
"runway video dry run");
check(contains(post("video", "{\"provider\":\"luma\",\"prompt\":\"waves\",\"duration_seconds\":5}").body, "\"duration\":\"5s\""),
"luma video dry run");
check(contains(post("video", "{\"provider\":\"kling\",\"prompt\":\"waves\"}").body, "/v1/videos/text2video"), "kling video dry run");
check(contains(post("video", "{\"provider\":\"higgsfield\",\"prompt\":\"waves\",\"model\":\"minimax/hailuo-2.3/standard/text-to-video\"}").body,
"/minimax/hailuo-2.3/standard/text-to-video"),
"higgsfield video dry run");
check(contains(post("call", "{\"provider\":\"bfl\",\"key\":\"flux-dev\",\"body\":{\"prompt\":\"x\"}}").body, "\"dry_run\":true"),
"a paid table call is a dry run");
check(contains(post("call", "{\"provider\":\"replicate\",\"key\":\"files.delete\",\"path\":{\"file_id\":\"f1\"}}").body, "\"effect\":\"delete\""),
"a destructive table call is a dry run");
check(contains(post("call", "{\"provider\":\"bfl\",\"key\":\"delete_finetune\",\"body\":{\"finetune_id\":\"my-ft\"}}").body, "\"dry_run\":true"),
"delete_finetune is a dry run");
auto cancel = f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "{\"ticket\":{\"provider\":\"luma\",\"id\":\"g1\",\"kind\":\"video\"}}");
check(cancel.status == 200 && contains(cancel.body, "\"dry_run\":true"), "cancel is a dry run");
// pure routes
auto hook = post("webhook",
"{\"secret\":\"whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw\",\"webhook_id\":\"msg_p5jXN8AQM9LWM0D4loKWxJek\","
"\"timestamp\":\"1614265330\",\"signature\":\"v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=\","
"\"payload\":\"{\\\"test\\\": 2432232314}\"}");
check(hook.status == 200 && contains(hook.body, "\"valid\":false"), "a vector from 2021 is stale now, so it is rejected");
// 503: reads and confirmed calls need the credential
auto acct = post("account", "{\"provider\":\"bfl\"}");
check(acct.status == 503 && contains(acct.body, "media_not_configured"), "account without key is 503");
check(post("account", "{\"provider\":\"minimax\"}").status == 422, "minimax has no free account read");
check(post("account", "{\"provider\":\"midjourney\"}").status == 422, "unknown provider account");
check(post("call", "{\"provider\":\"bfl\",\"key\":\"credits\"}").status == 503, "read call without key is 503");
check(post("call", "{\"provider\":\"fal\",\"key\":\"platform.meta\"}").status == 503, "fal platform read without key is 503");
check(post("call", "{\"provider\":\"bfl\",\"key\":\"flux-dev\",\"body\":{\"prompt\":\"x\"},\"confirm\":true}").status == 503,
"confirmed paid call without key is 503");
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed image without key is 503");
check(post("video", "{\"provider\":\"runway\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed video without key is 503");
check(post("video", "{\"provider\":\"kling\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "kling without keys is 503");
check(post("poll", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"}}").status == 503, "poll without key is 503");
check(post("wait", "{\"ticket\":{\"provider\":\"fal\",\"id\":\"1\",\"meta\":{\"endpoint\":\"fal-ai/flux/dev\"}}}").status == 503, "wait without key is 503");
check(f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "{\"ticket\":{\"provider\":\"luma\",\"id\":\"g1\"},\"confirm\":true}").status == 503,
"confirmed cancel without key is 503");
return finish("ai/media/routes");
}
+57
View File
@@ -0,0 +1,57 @@
#pragma once
// Shared rig for the media tests. A MockTransport only RECORDS the requests
// the real clients build; the few replies it serves are labelled FIXTURE
// (verbatim vendor documentation) or EMPTY (no content, the assertion is on
// our request). Nothing here verifies vendor behavior: see
// src/vendors/media/SPEC.md "Unverified".
#include <memory>
#include <string>
#include "clients/transports/mock_transport.h"
#include "framework.h"
#include "llm/error.h"
#include "vendors/media/common/job.h"
#include "vendors/media/common/media_service.h"
namespace media_test {
using molo::clients::ClientResponse;
using molo::clients::HeaderMap;
using molo::clients::Method;
using molo::clients::transports::MockExpectation;
using molo::clients::transports::MockTransport;
using molo::llm::ApiError;
using molo::media::Json;
inline ClientResponse reply(const std::string& body, int status = 200,
const std::string& type = "application/json") {
HeaderMap h;
h.set("Content-Type", type);
return ClientResponse(status, h, body);
}
// EMPTY: a 200 with `{}`; used only to let the real client finish a request
// whose OUTGOING shape the test inspects.
inline ClientResponse empty_ok() { return reply("{}"); }
inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; }
// Allows a path to answer several times.
inline void serve(MockTransport* t, Method m, const std::string& path, ClientResponse r, int times = 8) {
MockExpectation e;
e.method = m;
e.path = path;
e.response = std::move(r);
e.times = times;
t->expect(e);
}
struct Mock {
std::unique_ptr<MockTransport> owned = std::make_unique<MockTransport>();
MockTransport* raw = owned.get();
std::unique_ptr<molo::clients::Transport> take() { return std::move(owned); }
};
} // namespace media_test
+361
View File
@@ -0,0 +1,361 @@
// ImageGenerator / VideoGenerator adapters: the exact request each one would
// send (plan), loud refusal of options a provider cannot honour, and the
// rule that submit() without confirm makes no network call.
#include "support.h"
#include "vendors/media/bfl/bfl.h"
#include "vendors/media/common/registry.h"
#include "vendors/media/fal/fal.h"
#include "vendors/media/higgsfield/higgsfield.h"
#include "vendors/media/ideogram/ideogram.h"
#include "vendors/media/kling/kling.h"
#include "vendors/media/leonardo/leonardo.h"
#include "vendors/media/luma/luma.h"
#include "vendors/media/minimax/minimax.h"
#include "vendors/media/recraft/recraft.h"
#include "vendors/media/replicate/replicate.h"
#include "vendors/media/runway/runway.h"
#include "vendors/media/stability/stability.h"
using namespace molo::media;
using namespace media_test;
namespace {
ImageRequest image(const std::string& model = "") {
ImageRequest r;
r.prompt = "a lighthouse at dusk";
r.model = model;
return r;
}
VideoRequest video(const std::string& model = "") {
VideoRequest r;
r.prompt = "waves rolling in";
r.model = model;
return r;
}
std::string target(const Json& plan) { return plan.find("target")->as_string(); }
std::string body_of(const Json& plan) { return plan.find("body") ? plan.find("body")->dump() : ""; }
} // namespace
TEST(media_adapter, stability_picks_the_endpoint_from_the_model) {
ASSERT_EQ(StabilityImageGenerator::key_for(""), std::string("stable-image.generate.core"));
ASSERT_EQ(StabilityImageGenerator::key_for("ultra"), std::string("stable-image.generate.ultra"));
ASSERT_EQ(StabilityImageGenerator::key_for("sd3.5-large"), std::string("stable-image.generate.sd3"));
ASSERT_THROW(StabilityImageGenerator::key_for("dall-e"), ApiError);
Mock m;
StabilityImageGenerator g(StabilityClient::with(m.take(), "KEY"));
ImageRequest r = image("sd3.5-large");
r.aspect_ratio = "16:9";
r.seed = 42;
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/v2beta/stable-image/generate/sd3"));
ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"16:9\""));
ASSERT_TRUE(has(body_of(p), "\"model\":\"sd3.5-large\""));
ASSERT_TRUE(has(body_of(p), "\"seed\":42"));
ImageRequest wh = image();
wh.width = 512;
wh.height = 512;
ASSERT_THROW(g.plan(wh), ApiError); // stability sizes by aspect ratio only
ImageRequest many = image();
many.count = 2;
ASSERT_THROW(g.plan(many), ApiError);
ImageRequest src = image();
src.source_url = "https://example.com/in.png";
ASSERT_THROW(g.plan(src), ApiError);
ASSERT_THROW(g.submit(image(), false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, bfl_models_and_options) {
Mock m;
BflImageGenerator g(BflClient::with(m.take(), "KEY"));
ImageRequest r = image("flux-kontext-pro");
r.aspect_ratio = "1:1";
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/v1/flux-kontext-pro"));
ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"1:1\""));
ImageRequest wh = image("flux-kontext-pro"); // kontext takes aspect_ratio, not width / height
wh.width = 1024;
wh.height = 1024;
ASSERT_THROW(g.plan(wh), ApiError);
ImageRequest def = image();
def.width = 1024;
def.height = 768;
ASSERT_EQ(target(g.plan(def)), std::string("/v1/flux-2-pro")); // default model
ImageRequest many = image();
many.count = 3;
ASSERT_THROW(g.plan(many), ApiError);
ImageRequest unknown = image("flux-9000");
ASSERT_THROW(g.plan(unknown), ApiError); // not in the table
ASSERT_THROW(g.submit(def, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
BflVideoGenerator v(BflClient::with(std::make_unique<MockTransport>(), "KEY"));
VideoRequest vr = video();
vr.duration_seconds = 5;
vr.aspect_ratio = "16:9";
Json vp = v.plan(vr);
ASSERT_EQ(target(vp), std::string("/v1/flux-3-video"));
ASSERT_TRUE(has(body_of(vp), "\"duration\":5"));
}
TEST(media_adapter, replicate_needs_owner_name_and_maps_common_fields_into_input) {
Mock m;
ReplicateImageGenerator g(ReplicateClient::with(m.take(), "KEY"));
ImageRequest r = image("black-forest-labs/flux-schnell");
r.aspect_ratio = "3:2";
r.count = 2;
r.seed = 9;
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/models/black-forest-labs/flux-schnell/predictions"));
ASSERT_TRUE(has(body_of(p), "\"num_outputs\":2"));
ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"3:2\""));
ASSERT_THROW(g.plan(image()), ApiError);
ASSERT_THROW(g.plan(image("flux-schnell")), ApiError);
ASSERT_THROW(g.submit(r, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
ReplicateVideoGenerator v(ReplicateClient::with(std::make_unique<MockTransport>(), "KEY"));
VideoRequest vr = video("minimax/video-01");
vr.image_url = "https://example.com/a.png";
Json vp = v.plan(vr);
ASSERT_TRUE(has(body_of(vp), "\"image\":\"https://example.com/a.png\""));
}
TEST(media_adapter, fal_takes_any_endpoint_id) {
Mock m;
FalImageGenerator g(FalQueue::with(m.take(), "id:secret"));
ImageRequest r = image("fal-ai/flux/dev");
r.width = 1024;
r.height = 576;
r.count = 2;
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/fal-ai/flux/dev"));
ASSERT_TRUE(has(body_of(p), "\"image_size\":{\"height\":576,\"width\":1024}"));
ASSERT_TRUE(has(body_of(p), "\"num_images\":2"));
ASSERT_THROW(g.plan(image()), ApiError);
ASSERT_THROW(g.submit(r, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
FalVideoGenerator v(FalQueue::with(std::make_unique<MockTransport>(), "id:secret"));
VideoRequest vr = video("fal-ai/kling-video/v2.1/master/text-to-video");
vr.duration_seconds = 5;
Json vp = v.plan(vr);
ASSERT_EQ(target(vp), std::string("/fal-ai/kling-video/v2.1/master/text-to-video"));
ASSERT_TRUE(has(body_of(vp), "\"duration\":\"5\""));
}
TEST(media_adapter, ideogram_models_map_to_rows_and_edit_inputs_are_files) {
Mock m;
IdeogramImageGenerator g(IdeogramClient::with(m.take(), "KEY"));
ASSERT_EQ(IdeogramImageGenerator::key_for(""), std::string("image.generate.ideogram-3"));
ImageRequest r = image("ideogram-4");
r.count = 2;
r.width = 1024;
r.height = 1024;
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/v2/image/generate/ideogram-4"));
ASSERT_TRUE(has(body_of(p), "\"resolution\":\"1024x1024\""));
ImageRequest src = image();
src.source_url = "https://example.com/in.png";
ASSERT_THROW(g.plan(src), ApiError);
ASSERT_THROW(g.plan(image("ideogram-99")), ApiError);
ASSERT_THROW(g.submit(image(), false), ApiError);
IdeogramVideoGenerator v(IdeogramClient::with(std::make_unique<MockTransport>(), "KEY"));
ASSERT_THROW(v.plan(video()), ApiError); // a video model is required
Json vp = v.plan(video("kling-3-standard-text-to-video"));
ASSERT_EQ(target(vp), std::string("/v2/video/generate/kling-3-standard-text-to-video"));
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, recraft_sizes_by_wxh) {
Mock m;
RecraftImageGenerator g(RecraftClient::with(m.take(), "KEY"));
ImageRequest r = image("recraftv3");
r.width = 1024;
r.height = 1024;
r.count = 2;
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/v1/images/generations"));
ASSERT_TRUE(has(body_of(p), "\"size\":\"1024x1024\""));
ASSERT_TRUE(has(body_of(p), "\"n\":2"));
ImageRequest ar = image();
ar.aspect_ratio = "16:9";
ASSERT_THROW(g.plan(ar), ApiError);
ASSERT_THROW(g.submit(r, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, leonardo_wraps_parameters_and_never_publishes) {
Mock m;
LeonardoImageGenerator g(LeonardoClient::with(m.take(), "KEY"));
ImageRequest r = image("flux-dev");
r.width = 1024;
r.height = 1024;
r.count = 4;
Json p = g.plan(r);
ASSERT_EQ(target(p), std::string("/v2/generations"));
ASSERT_TRUE(has(body_of(p), "\"model\":\"flux-dev\""));
ASSERT_TRUE(has(body_of(p), "\"public\":false"));
ASSERT_TRUE(has(body_of(p), "\"quantity\":4"));
ASSERT_THROW(g.plan(image()), ApiError); // model required
ImageRequest ar = image("flux-dev");
ar.aspect_ratio = "1:1";
ASSERT_THROW(g.plan(ar), ApiError);
ASSERT_THROW(g.submit(r, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, runway_chooses_text_or_image_to_video) {
Mock m;
RunwayVideoGenerator g(RunwayClient::with(m.take(), "KEY"));
VideoRequest t = video();
t.duration_seconds = 5;
t.aspect_ratio = "1280:720";
ASSERT_EQ(target(g.plan(t)), std::string("/v1/text_to_video"));
ASSERT_TRUE(has(body_of(g.plan(t)), "\"model\":\"gen4.5\""));
VideoRequest i = video();
i.image_url = "https://example.com/a.png";
ASSERT_EQ(target(g.plan(i)), std::string("/v1/image_to_video"));
ASSERT_TRUE(has(body_of(g.plan(i)), "\"model\":\"gen4_turbo\""));
ASSERT_TRUE(has(body_of(g.plan(i)), "\"promptImage\""));
RunwayImageGenerator ig(RunwayClient::with(std::make_unique<MockTransport>(), "KEY"));
ImageRequest r = image();
r.width = 1280;
r.height = 720;
Json p = ig.plan(r);
ASSERT_EQ(target(p), std::string("/v1/text_to_image"));
ASSERT_TRUE(has(body_of(p), "\"ratio\":\"1280:720\""));
ASSERT_TRUE(has(body_of(p), "\"promptText\""));
ASSERT_THROW(g.submit(t, false), ApiError);
ASSERT_THROW(ig.cancel(JobTicket{}, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, luma_durations_are_strings_and_keyframes_nest_frames) {
Mock m;
LumaVideoGenerator g(LumaClient::with(m.take(), "KEY"));
VideoRequest v = video();
v.duration_seconds = 5;
v.image_url = "https://example.com/start.png";
v.end_image_url = "https://example.com/end.png";
Json p = g.plan(v);
ASSERT_EQ(target(p), std::string("/generations/video"));
ASSERT_TRUE(has(body_of(p), "\"duration\":\"5s\""));
ASSERT_TRUE(has(body_of(p), "\"frame0\":{\"type\":\"image\",\"url\":\"https://example.com/start.png\"}"));
ASSERT_TRUE(has(body_of(p), "\"frame1\""));
ASSERT_TRUE(has(body_of(p), "\"model\":\"ray-2\""));
LumaImageGenerator ig(LumaClient::with(std::make_unique<MockTransport>(), "KEY"));
ImageRequest wh = image();
wh.width = 512;
wh.height = 512;
ASSERT_THROW(ig.plan(wh), ApiError);
ASSERT_EQ(target(ig.plan(image())), std::string("/generations/image"));
ASSERT_THROW(g.cancel(JobTicket{}, false), ApiError); // cancel deletes: confirm required
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, minimax_refuses_options_v1_does_not_have) {
Mock m;
MinimaxVideoGenerator g(MinimaxClient::with(m.take(), "KEY"));
VideoRequest v = video();
v.duration_seconds = 6;
v.resolution = "768p";
Json p = g.plan(v);
ASSERT_EQ(target(p), std::string("/v1/video_generation"));
ASSERT_TRUE(has(body_of(p), "\"resolution\":\"768P\""));
ASSERT_TRUE(has(body_of(p), "\"model\":\"MiniMax-Hailuo-2.3\""));
VideoRequest ar = video();
ar.aspect_ratio = "16:9";
ASSERT_THROW(g.plan(ar), ApiError);
VideoRequest sd = video();
sd.seed = 5;
ASSERT_THROW(g.plan(sd), ApiError);
ASSERT_THROW(g.submit(v, false), ApiError);
ASSERT_FALSE(g.cancel(JobTicket{}, true)); // not supported, says so without a call
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, kling_and_higgsfield) {
auto kc = std::make_shared<KlingClient>("AK", "SK", [] { return std::make_unique<MockTransport>(); });
KlingVideoGenerator k(kc);
VideoRequest v = video();
v.duration_seconds = 5;
Json p = k.plan(v);
ASSERT_EQ(target(p), std::string("/v1/videos/text2video"));
ASSERT_TRUE(has(body_of(p), "\"model_name\":\"kling-v1-6\""));
ASSERT_TRUE(has(body_of(p), "\"duration\":\"5\""));
VideoRequest i = video();
i.image_url = "https://example.com/a.png";
ASSERT_EQ(target(k.plan(i)), std::string("/v1/videos/image2video"));
VideoRequest sd = video();
sd.seed = 1;
ASSERT_THROW(k.plan(sd), ApiError);
ASSERT_THROW(k.submit(v, false), ApiError);
Mock m;
HiggsfieldImageGenerator h(HiggsfieldClient::with(m.take(), "id", "secret"));
Json hp = h.plan(image());
ASSERT_EQ(target(hp), std::string("/higgsfield-ai/soul/standard"));
ImageRequest wh = image();
wh.width = 512;
wh.height = 512;
ASSERT_THROW(h.plan(wh), ApiError);
HiggsfieldVideoGenerator hv(HiggsfieldClient::with(std::make_unique<MockTransport>(), "id", "secret"));
ASSERT_THROW(hv.plan(video()), ApiError); // model path required
ASSERT_EQ(target(hv.plan(video("kling-video/v2.5-turbo/pro/text-to-video"))),
std::string("/kling-video/v2.5-turbo/pro/text-to-video"));
ASSERT_THROW(h.cancel(JobTicket{}, false), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_adapter, registry_builds_a_generator_for_every_declared_capability) {
MediaRegistry reg;
for (const ProviderInfo& p : MediaRegistry::providers()) {
if (p.image) {
auto g = reg.image(p.id);
ASSERT_EQ(g->provider(), p.id);
ASSERT_FALSE(g->models().empty());
} else {
ASSERT_THROW(reg.image(p.id), ApiError);
}
if (p.video) {
auto g = reg.video(p.id);
ASSERT_EQ(g->provider(), p.id);
ASSERT_FALSE(g->models().empty());
} else {
ASSERT_THROW(reg.video(p.id), ApiError);
}
ASSERT_TRUE(reg.service(p.id) != nullptr);
}
ASSERT_THROW(reg.image("midjourney"), ApiError);
ASSERT_THROW(reg.service("pika"), ApiError);
}
TEST(media_adapter, every_default_model_resolves_to_a_row_in_its_table) {
// A model named by models() must plan cleanly where the adapter derives the row from it.
MediaRegistry reg;
ImageRequest r = image();
for (const std::string& model : reg.image("bfl")->models()) {
r.model = model;
r.width = 0;
r.height = 0;
if (model.rfind("flux-kontext", 0) == 0) r.aspect_ratio = "1:1";
else r.aspect_ratio = "";
if (model == "flux-3-image") continue; // takes images[], aspect_ratio: covered by the table test
r.aspect_ratio = "";
reg.image("bfl")->plan(r);
}
for (const std::string& model : reg.image("ideogram")->models()) {
r.model = model;
r.aspect_ratio = "";
r.width = 0;
r.height = 0;
reg.image("ideogram")->plan(r);
}
for (const std::string& model : reg.video("ideogram")->models()) {
VideoRequest v = video(model);
reg.video("ideogram")->plan(v);
}
for (const std::string& model : reg.video("higgsfield")->models()) reg.video("higgsfield")->plan(video(model));
}
+154
View File
@@ -0,0 +1,154 @@
// Call tables: structure of every provider table, CallTable rules, registry.
#include <set>
#include "support.h"
#include "vendors/media/common/registry.h"
using namespace molo::media;
using namespace media_test;
namespace {
struct Expect {
const char* id;
std::size_t rows; // rows declared from the published spec / SDK / docs (see SPEC.md)
};
const Expect kExpected[] = {
{"stability", 34}, {"bfl", 30}, {"replicate", 36}, {"fal", 16}, {"ideogram", 70}, {"recraft", 22},
{"leonardo", 6}, {"runway", 66}, {"luma", 13}, {"minimax", 7}, {"kling", 6}, {"higgsfield", 10},
};
} // namespace
TEST(media_tables, every_provider_has_the_declared_row_count) {
for (const Expect& e : kExpected) {
const std::string id = e.id;
const CallTable& t = MediaRegistry::table(id);
ASSERT_EQ(t.size(), e.rows);
ASSERT_EQ(t.vendor(), std::string(e.id));
}
ASSERT_EQ(MediaRegistry::providers().size(), sizeof(kExpected) / sizeof(kExpected[0]));
}
TEST(media_tables, rows_are_well_formed) {
for (const Expect& e : kExpected) {
const std::string id = e.id;
const CallTable& t = MediaRegistry::table(id);
std::set<std::string> seen;
for (std::size_t i = 0; i < t.size(); ++i) {
const CallRow& r = t.at(i);
ASSERT_TRUE(seen.insert(r.key).second);
std::string method = r.http_method;
std::string kind = r.body_kind;
// a body-less method never declares body fields
if (method == "GET" || method == "DELETE") ASSERT_TRUE(kind == "none");
if (kind == "none") ASSERT_EQ(std::string(r.body_fields), std::string(""));
if (kind != "none") ASSERT_TRUE(method != "GET");
// destructive rows are DELETE or an explicit delete_* call
if (CallTable::effect_of(r) == Effect::kDelete)
ASSERT_TRUE(method == "DELETE" || std::string(r.key).find("delete") != std::string::npos);
// every GET reads
if (method == "GET") ASSERT_TRUE(CallTable::effect_of(r) == Effect::kRead);
// placeholders are non-empty and unique within a path
std::set<std::string> ph;
for (const std::string& p : CallTable::placeholders(r.path)) {
ASSERT_FALSE(p.empty());
ASSERT_TRUE(ph.insert(p).second);
}
}
}
}
TEST(media_tables, paid_rows_exist_for_every_generating_provider) {
for (const Expect& e : kExpected) {
const std::string id = e.id;
const CallTable& t = MediaRegistry::table(id);
std::size_t paid = 0;
for (std::size_t i = 0; i < t.size(); ++i)
if (CallTable::effect_of(t.at(i)) == Effect::kPaid) ++paid;
ASSERT_TRUE(paid >= 1);
}
}
TEST(media_tables, rejects_a_malformed_table) {
static const CallRow dup[] = {{"a", "GET", "/x", "", "none", "", "", "read"}, {"a", "GET", "/y", "", "none", "", "", "read"}};
ASSERT_THROW(CallTable("t", dup, 2), std::logic_error);
static const CallRow badm[] = {{"a", "FETCH", "/x", "", "none", "", "", "read"}};
ASSERT_THROW(CallTable("t", badm, 1), std::logic_error);
static const CallRow badk[] = {{"a", "GET", "/x", "", "blob", "", "", "read"}};
ASSERT_THROW(CallTable("t", badk, 1), std::logic_error);
static const CallRow bade[] = {{"a", "GET", "/x", "", "none", "", "", "free"}};
ASSERT_THROW(CallTable("t", bade, 1), std::logic_error);
static const CallRow badp[] = {{"a", "GET", "x", "", "none", "", "", "read"}};
ASSERT_THROW(CallTable("t", badp, 1), std::logic_error);
}
TEST(media_tables, path_rendering_validates_every_argument) {
static const CallRow rows[] = {
{"one", "GET", "/a/{id}/b/{name}", "", "none", "", "", "read"},
{"multi", "GET", "/{endpoint_id}/x", "", "none", "", "", "read"},
};
CallTable t("t", rows, 2);
ASSERT_EQ(t.render_path(t.get("one"), {{"id", "abc-1"}, {"name", "n.v"}}), std::string("/a/abc-1/b/n.v"));
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "abc"}}), ApiError); // missing
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a"}, {"name", "b"}, {"x", "c"}}), ApiError); // extra
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "../x"}, {"name", "b"}}), ApiError); // traversal
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a/b"}, {"name", "b"}}), ApiError); // slash in a segment
ASSERT_THROW(t.render_path(t.get("one"), {{"id", ".."}, {"name", "b"}}), ApiError);
ASSERT_THROW(t.render_path(t.get("one"), {{"id", ""}, {"name", "b"}}), ApiError);
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a b"}, {"name", "b"}}), ApiError);
ASSERT_THROW(t.render_path(t.get("one"), {{"id", std::string(257, 'a')}, {"name", "b"}}), ApiError);
ASSERT_EQ(t.render_path(t.get("multi"), {{"endpoint_id", "fal-ai/flux/dev"}}), std::string("/fal-ai/flux/dev/x"));
ASSERT_THROW(t.render_path(t.get("multi"), {{"endpoint_id", "a//b"}}), ApiError);
ASSERT_THROW(t.render_path(t.get("multi"), {{"endpoint_id", "a/b/c/d/e/f/g"}}), ApiError); // 7 segments
ASSERT_THROW(t.get("nope"), ApiError);
}
TEST(media_tables, segment_alphabet_matches_the_documented_id_shapes) {
ASSERT_TRUE(CallTable::valid_segment("cneqzikepnug6xezperrr4z55o")); // Replicate file id
ASSERT_TRUE(CallTable::valid_segment("d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff")); // Higgsfield uuid
ASSERT_TRUE(CallTable::valid_segment("abc_DEF-123==")); // URL-safe base64 with padding
ASSERT_FALSE(CallTable::valid_segment("a?b"));
ASSERT_FALSE(CallTable::valid_segment("a#b"));
ASSERT_FALSE(CallTable::valid_segment("a%2fb"));
}
TEST(media_registry, lists_unsupported_products_with_a_reason) {
const auto& u = MediaRegistry::unsupported();
ASSERT_EQ(u.size(), static_cast<std::size_t>(2));
ASSERT_EQ(u[0].name, std::string("Midjourney"));
ASSERT_TRUE(has(u[0].reason, "No public API"));
ASSERT_EQ(u[1].name, std::string("Pika"));
for (const auto& x : u) ASSERT_TRUE(x.reason.size() > 40);
}
TEST(media_registry, every_provider_declares_credentials_evidence_and_a_console) {
for (const ProviderInfo& p : MediaRegistry::providers()) {
ASSERT_FALSE(p.env.empty());
ASSERT_TRUE(p.image || p.video);
ASSERT_TRUE(p.console_url.rfind("https://", 0) == 0);
ASSERT_TRUE(p.note.size() > 10);
if (!p.account_key.empty()) {
const CallRow& r = MediaRegistry::table(p.id).get(p.account_key);
ASSERT_TRUE(CallTable::effect_of(r) == Effect::kRead); // the credential proof never costs money
}
}
ASSERT_EQ(std::string(MediaRegistry::evidence_name(MediaRegistry::find("kling")->evidence)), std::string("probe-only"));
ASSERT_EQ(std::string(MediaRegistry::evidence_name(MediaRegistry::find("bfl")->evidence)), std::string("openapi"));
ASSERT_TRUE(MediaRegistry::find("midjourney") == nullptr);
ASSERT_THROW(MediaRegistry::table("midjourney"), ApiError);
}
TEST(media_registry, required_environment_variables_are_the_documented_ones) {
auto env = [](const char* id) { return MediaRegistry::find(id)->env; };
ASSERT_EQ(env("stability")[0], std::string("STABILITY_API_KEY"));
ASSERT_EQ(env("bfl")[0], std::string("BFL_API_KEY"));
ASSERT_EQ(env("runway")[0], std::string("RUNWAY_API_KEY"));
ASSERT_EQ(env("luma")[0], std::string("LUMA_API_KEY"));
ASSERT_EQ(env("replicate")[0], std::string("REPLICATE_API_TOKEN"));
ASSERT_EQ(env("fal")[0], std::string("FAL_KEY"));
ASSERT_EQ(env("ideogram")[0], std::string("IDEOGRAM_API_KEY"));
ASSERT_EQ(env("recraft")[0], std::string("RECRAFT_API_TOKEN"));
ASSERT_EQ(env("leonardo")[0], std::string("LEONARDO_API_KEY"));
}
+414
View File
@@ -0,0 +1,414 @@
// The neutral job seam: request validation, JobRunner caps (with a fake
// generator: local logic only), output scanning, status mappers, downloader
// rules. Inputs to the mappers are hand-built (LOCAL-LOGIC): they exercise our
// code, they are not claims about what a vendor returns.
#include <vector>
#include "support.h"
#include "vendors/media/bfl/bfl.h"
#include "vendors/media/common/caps.h"
#include "vendors/media/common/downloader.h"
#include "vendors/media/common/job_runner.h"
#include "vendors/media/common/output_scanner.h"
#include "vendors/media/higgsfield/higgsfield.h"
#include "vendors/media/kling/kling.h"
#include "vendors/media/leonardo/leonardo.h"
#include "vendors/media/luma/luma.h"
#include "vendors/media/minimax/minimax.h"
#include "vendors/media/runway/runway.h"
using namespace molo::media;
using namespace media_test;
namespace {
Json obj(std::initializer_list<std::pair<const char*, Json>> items) {
Json o = Json::object();
for (const auto& [k, v] : items) o.set(k, v);
return o;
}
Json arr(std::initializer_list<Json> items) {
Json a = Json::array();
for (const Json& v : items) a.push(v);
return a;
}
// Local fake: finishes after `finish_after` polls (0 = never).
class FakeGenerator : public ImageGenerator {
public:
explicit FakeGenerator(int finish_after, JobState end = JobState::kSucceeded) : finish_after_(finish_after), end_(end) {}
std::string provider() const override { return "fake"; }
bool configured() const override { return true; }
std::vector<std::string> models() const override { return {"m"}; }
Json plan(const ImageRequest&) const override { return Json::object(); }
JobTicket submit(const ImageRequest&, bool) override { return JobTicket{}; }
JobStatus poll(const JobTicket&) override {
++polls;
JobStatus s;
if (finish_after_ > 0 && polls >= finish_after_) {
s.state = end_;
if (end_ == JobState::kFailed) s.error = "boom";
MediaAsset a;
a.url = "https://example.com/out.png";
if (end_ == JobState::kSucceeded) s.assets.push_back(a);
} else {
s.state = JobState::kRunning;
}
return s;
}
bool cancel(const JobTicket&, bool) override { return false; }
int suggested_poll_ms() const override { return 100; } // below the floor on purpose
int polls = 0;
private:
int finish_after_;
JobState end_;
};
struct Clockwork {
long now = 0;
std::vector<int> slept;
JobRunner::Sleeper sleeper() {
return [this](int ms) {
slept.push_back(ms);
now += ms;
};
}
JobRunner::Clock clock() {
return [this] { return now; };
}
};
JobTicket ticket(const std::string& id = "j1") {
JobTicket t;
t.provider = "fake";
t.id = id;
return t;
}
} // namespace
// ------------------------------------------------------------------ JobRunner
TEST(media_runner, polls_until_terminal_and_applies_the_interval_floor_and_backoff) {
Clockwork w;
FakeGenerator g(5);
WaitPolicy p;
p.backoff = 2.0;
p.max_interval_ms = 3000;
JobRunner r(p, w.sleeper(), w.clock());
JobStatus s = r.wait(g, ticket());
ASSERT_TRUE(s.state == JobState::kSucceeded);
ASSERT_EQ(g.polls, 5);
// suggestion 100 ms is lifted to the 500 ms floor, then doubles, capped at 3000
ASSERT_EQ(w.slept.size(), static_cast<std::size_t>(4));
ASSERT_EQ(w.slept[0], 500);
ASSERT_EQ(w.slept[1], 1000);
ASSERT_EQ(w.slept[2], 2000);
ASSERT_EQ(w.slept[3], 3000);
}
TEST(media_runner, a_failed_job_is_a_status_not_an_exception) {
Clockwork w;
FakeGenerator g(2, JobState::kFailed);
JobRunner r({}, w.sleeper(), w.clock());
JobStatus s = r.wait(g, ticket());
ASSERT_TRUE(s.state == JobState::kFailed);
ASSERT_EQ(s.error, std::string("boom"));
}
TEST(media_runner, attempt_cap_throws_loudly_and_does_not_cancel) {
Clockwork w;
FakeGenerator g(0);
WaitPolicy p;
p.max_attempts = 3;
JobRunner r(p, w.sleeper(), w.clock());
try {
r.wait(g, ticket("job-77"));
FAIL("expected WAIT_EXPIRED");
} catch (const ApiError& e) {
ASSERT_EQ(e.type(), std::string("WAIT_EXPIRED"));
ASSERT_EQ(e.http_status(), 504);
ASSERT_TRUE(has(e.message(), "job-77"));
ASSERT_TRUE(has(e.message(), "NOT cancelled"));
}
ASSERT_EQ(g.polls, 3);
}
TEST(media_runner, total_wait_cap_stops_before_sleeping_past_the_deadline) {
Clockwork w;
FakeGenerator g(0);
WaitPolicy p;
p.max_wait_seconds = 5;
p.max_attempts = 200;
p.first_interval_ms = 2000;
p.backoff = 1.0;
JobRunner r(p, w.sleeper(), w.clock());
ASSERT_THROW(r.wait(g, ticket()), ApiError);
long total = 0;
for (int s : w.slept) total += s;
ASSERT_TRUE(total <= 5000);
ASSERT_TRUE(g.polls <= 4);
}
TEST(media_runner, immediate_tickets_need_no_poll) {
Clockwork w;
FakeGenerator g(0);
JobTicket t = ticket();
t.immediate = true;
MediaAsset a;
a.data_base64 = "AAAA";
t.assets.push_back(a);
JobStatus s = JobRunner({}, w.sleeper(), w.clock()).wait(g, t);
ASSERT_TRUE(s.state == JobState::kSucceeded);
ASSERT_EQ(g.polls, 0);
ASSERT_EQ(s.assets.size(), static_cast<std::size_t>(1));
}
TEST(media_runner, policy_values_outside_the_caps_are_rejected) {
auto bad = [](auto edit) {
WaitPolicy p;
edit(p);
return p;
};
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_attempts = 0; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_attempts = MediaCaps::kMaxPollAttempts + 1; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_wait_seconds = MediaCaps::kMaxWaitSeconds + 1; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_wait_seconds = 0; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.backoff = 0.9; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.backoff = 3.5; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_interval_ms = MediaCaps::kMinPollIntervalMs - 1; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_interval_ms = MediaCaps::kMaxPollIntervalMs + 1; })), ApiError);
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.first_interval_ms = -1; })), ApiError);
JobRunner ok{WaitPolicy{}};
ASSERT_EQ(ok.policy().max_attempts, 120);
}
// -------------------------------------------------------------------- requests
TEST(media_requests, image_validation) {
ImageRequest r;
ASSERT_THROW(r.validate(), ApiError); // no prompt
r.prompt = "a cat";
r.validate();
r.count = 0;
ASSERT_THROW(r.validate(), ApiError);
r.count = MediaCaps::kMaxImagesPerRequest + 1;
ASSERT_THROW(r.validate(), ApiError);
r.count = 1;
r.width = 512;
ASSERT_THROW(r.validate(), ApiError); // width without height
r.height = MediaCaps::kMaxDimension + 1;
ASSERT_THROW(r.validate(), ApiError);
r.height = 512;
r.validate();
r.seed = -2;
ASSERT_THROW(r.validate(), ApiError);
r.seed = 4294967296L;
ASSERT_THROW(r.validate(), ApiError);
r.seed = 7;
r.extra = Json(5);
ASSERT_THROW(r.validate(), ApiError);
r.extra = Json::object();
r.prompt = std::string(MediaCaps::kMaxPromptChars + 1, 'x');
ASSERT_THROW(r.validate(), ApiError);
}
TEST(media_requests, video_validation) {
VideoRequest v;
ASSERT_THROW(v.validate(), ApiError);
v.image_url = "https://example.com/a.png"; // image-to-video needs no prompt
v.validate();
v.duration_seconds = MediaCaps::kMaxVideoSeconds + 1;
ASSERT_THROW(v.validate(), ApiError);
v.duration_seconds = 5;
v.validate();
v.duration_seconds = -1;
ASSERT_THROW(v.validate(), ApiError);
}
TEST(media_requests, ticket_and_status_json) {
JobTicket t;
t.provider = "fal";
t.id = "r1";
t.model = "fal-ai/flux/dev";
t.kind = MediaKind::kVideo;
t.meta = obj({{"endpoint", Json("fal-ai/flux/dev")}});
JobTicket back = JobTicket::from_json(t.to_json());
ASSERT_EQ(back.provider, std::string("fal"));
ASSERT_EQ(back.id, std::string("r1"));
ASSERT_TRUE(back.kind == MediaKind::kVideo);
ASSERT_EQ(back.meta.find("endpoint")->as_string(), std::string("fal-ai/flux/dev"));
ASSERT_THROW(JobTicket::from_json(Json(1)), ApiError);
ASSERT_THROW(JobTicket::from_json(obj({{"provider", Json("x")}})), ApiError);
JobStatus s;
s.state = JobState::kRunning;
s.progress = 0.5;
ASSERT_TRUE(has(s.to_json().dump(), "\"state\":\"running\""));
ASSERT_FALSE(s.terminal());
s.state = JobState::kCancelled;
ASSERT_TRUE(s.terminal());
}
// -------------------------------------------------------------- OutputScanner
TEST(media_scanner, finds_urls_in_the_shapes_providers_use) {
// fal style: images[].url and video.url
auto fal = OutputScanner::scan(obj({{"images", arr({obj({{"url", Json("https://v3.fal.media/a.png")}})})},
{"video", obj({{"url", Json("https://v3.fal.media/b.mp4")}})}}),
MediaKind::kImage);
ASSERT_EQ(fal.size(), static_cast<std::size_t>(2));
ASSERT_TRUE(fal[0].kind == MediaKind::kImage);
ASSERT_EQ(fal[0].mime, std::string("image/png"));
ASSERT_TRUE(fal[1].kind == MediaKind::kVideo);
// Replicate style: a bare string, or an array
ASSERT_EQ(OutputScanner::scan(Json("https://replicate.delivery/x/out.webp"), MediaKind::kImage).size(), static_cast<std::size_t>(1));
ASSERT_EQ(OutputScanner::scan(arr({Json("https://a/1.png"), Json("https://a/2.png")}), MediaKind::kImage).size(), static_cast<std::size_t>(2));
// non-URL strings are ignored
ASSERT_EQ(OutputScanner::scan(arr({Json("hello"), Json("ftp://x/y.png")}), MediaKind::kImage).size(), static_cast<std::size_t>(0));
}
TEST(media_scanner, skips_request_echoes_and_control_links_and_dedupes) {
Json doc = obj({{"input", obj({{"image", Json("https://in/source.png")}})},
{"urls", obj({{"get", Json("https://api.replicate.com/v1/predictions/p1")}})},
{"status_url", Json("https://api.higgsfield.ai/requests/x/status")},
{"output", arr({Json("https://out/a.png"), Json("https://out/a.png")})}});
auto found = OutputScanner::scan(doc, MediaKind::kImage);
ASSERT_EQ(found.size(), static_cast<std::size_t>(1));
ASSERT_EQ(found[0].url, std::string("https://out/a.png"));
}
TEST(media_scanner, data_urls_become_inline_assets_and_caps_hold) {
auto d = OutputScanner::scan(Json("data:image/png;base64,iVBORw0KGgo="), MediaKind::kOther);
ASSERT_EQ(d.size(), static_cast<std::size_t>(1));
ASSERT_EQ(d[0].mime, std::string("image/png"));
ASSERT_EQ(d[0].data_base64, std::string("iVBORw0KGgo="));
ASSERT_TRUE(d[0].kind == MediaKind::kImage);
Json many = Json::array();
for (int i = 0; i < 100; ++i) many.push(Json("https://x/" + std::to_string(i) + ".png"));
ASSERT_EQ(OutputScanner::scan(many, MediaKind::kImage).size(), MediaCaps::kMaxAssetsPerJob);
Json deep = Json("https://deep/out.png");
for (int i = 0; i < 20; ++i) deep = obj({{"k", deep}});
ASSERT_EQ(OutputScanner::scan(deep, MediaKind::kImage).size(), static_cast<std::size_t>(0)); // depth cap
ASSERT_TRUE(OutputScanner::kind_from_name("https://x/a.GLB?sig=1", MediaKind::kOther) == MediaKind::kModel3d);
ASSERT_TRUE(OutputScanner::kind_from_name("https://x/a.wav", MediaKind::kOther) == MediaKind::kAudio);
}
// ------------------------------------------------------------ status mappers
TEST(media_status, bfl_words) {
ASSERT_TRUE(BflImageGenerator::state_of("Ready") == JobState::kSucceeded);
ASSERT_TRUE(BflImageGenerator::state_of("Pending") == JobState::kQueued);
ASSERT_TRUE(BflImageGenerator::state_of("Generating") == JobState::kRunning);
for (const char* w : {"Error", "Request Moderated", "Content Moderated", "Task not found", "No Card Generated"})
ASSERT_TRUE(BflImageGenerator::state_of(w) == JobState::kFailed);
ASSERT_TRUE(BflImageGenerator::state_of("something new") == JobState::kRunning); // unknown = keep polling, bounded by caps
}
TEST(media_status, runway_tasks) {
JobStatus ok = RunwayJobs::status_of(obj({{"status", Json("SUCCEEDED")}, {"output", arr({Json("https://r/v.mp4")})}}), MediaKind::kVideo);
ASSERT_TRUE(ok.state == JobState::kSucceeded);
ASSERT_TRUE(ok.assets[0].kind == MediaKind::kVideo);
JobStatus empty = RunwayJobs::status_of(obj({{"status", Json("SUCCEEDED")}}), MediaKind::kVideo);
ASSERT_TRUE(empty.state == JobState::kFailed);
JobStatus run = RunwayJobs::status_of(obj({{"status", Json("RUNNING")}, {"progress", Json(0.4)}}), MediaKind::kVideo);
ASSERT_TRUE(run.state == JobState::kRunning);
ASSERT_TRUE(run.progress > 0.39 && run.progress < 0.41);
JobStatus fail = RunwayJobs::status_of(obj({{"status", Json("FAILED")}, {"failure", Json("moderation")}, {"failureCode", Json("SAFETY")}}), MediaKind::kVideo);
ASSERT_EQ(fail.error, std::string("moderation (SAFETY)"));
ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("CANCELLED")}}), MediaKind::kVideo).state == JobState::kCancelled);
ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("THROTTLED")}}), MediaKind::kVideo).state == JobState::kQueued);
ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("PENDING")}}), MediaKind::kVideo).state == JobState::kQueued);
}
TEST(media_status, luma_generations_ignore_the_progress_preview) {
Json done = obj({{"state", Json("completed")},
{"assets", obj({{"video", Json("https://l/final.mp4")}, {"progress_video", Json("https://l/progress.mp4")}})}});
JobStatus s = LumaJobs::status_of(done, MediaKind::kVideo);
ASSERT_TRUE(s.state == JobState::kSucceeded);
ASSERT_EQ(s.assets.size(), static_cast<std::size_t>(1));
ASSERT_EQ(s.assets[0].url, std::string("https://l/final.mp4"));
ASSERT_TRUE(LumaJobs::status_of(obj({{"state", Json("dreaming")}}), MediaKind::kVideo).state == JobState::kRunning);
ASSERT_TRUE(LumaJobs::status_of(obj({{"state", Json("queued")}}), MediaKind::kVideo).state == JobState::kQueued);
JobStatus f = LumaJobs::status_of(obj({{"state", Json("failed")}, {"failure_reason", Json("nope")}}), MediaKind::kVideo);
ASSERT_EQ(f.error, std::string("nope"));
}
TEST(media_status, word_maps_of_minimax_higgsfield_kling) {
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Success") == JobState::kSucceeded);
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Fail") == JobState::kFailed);
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Processing") == JobState::kRunning);
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Queueing") == JobState::kQueued);
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Preparing") == JobState::kQueued);
for (const char* w : {"completed"}) ASSERT_TRUE(HiggsfieldJobs::state_of(w) == JobState::kSucceeded);
ASSERT_TRUE(HiggsfieldJobs::state_of("nsfw") == JobState::kFailed);
ASSERT_TRUE(HiggsfieldJobs::state_of("failed") == JobState::kFailed);
ASSERT_TRUE(HiggsfieldJobs::state_of("canceled") == JobState::kCancelled);
ASSERT_TRUE(HiggsfieldJobs::state_of("in_progress") == JobState::kRunning);
ASSERT_TRUE(HiggsfieldJobs::state_of("queued") == JobState::kQueued);
ASSERT_TRUE(KlingVideoGenerator::state_of("succeed") == JobState::kSucceeded);
ASSERT_TRUE(KlingVideoGenerator::state_of("failed") == JobState::kFailed);
ASSERT_TRUE(KlingVideoGenerator::state_of("processing") == JobState::kRunning);
ASSERT_TRUE(KlingVideoGenerator::state_of("submitted") == JobState::kQueued);
}
TEST(media_status, leonardo_generation_shape_is_read_from_generations_by_pk) {
Json done = obj({{"generations_by_pk", obj({{"status", Json("COMPLETE")},
{"generated_images", arr({obj({{"url", Json("https://cdn.leonardo.ai/a.jpg")}, {"id", Json("i1")}})})}})}});
JobStatus s = LeonardoJobs::status_of(done, MediaKind::kImage);
ASSERT_TRUE(s.state == JobState::kSucceeded);
ASSERT_EQ(s.assets.size(), static_cast<std::size_t>(1));
ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("PENDING")}})}}), MediaKind::kImage).state == JobState::kRunning);
ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("FAILED")}})}}), MediaKind::kImage).state == JobState::kFailed);
ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("COMPLETE")}})}}), MediaKind::kImage).state == JobState::kFailed);
}
// ----------------------------------------------------------------- downloader
TEST(media_downloader, url_rules) {
Downloader::validate_url("https://cdn.example.com/a.png?sig=abc");
for (const char* bad : {"", "http://example.com/a.png", "ftp://x/y", "https://", "https:///path", "https://u:p@example.com/a",
"https://exa mple.com/a", "javascript:alert(1)", "file:///etc/passwd", "data:image/png;base64,AA=="})
ASSERT_THROW(Downloader::validate_url(bad), ApiError);
ASSERT_THROW(Downloader::validate_url("https://x/" + std::string(MediaCaps::kMaxUrlChars, 'a')), ApiError);
ASSERT_THROW(Downloader::validate_url("https://x/a\nb"), ApiError);
}
TEST(media_downloader, inline_assets_decode_without_network) {
Mock m;
MockTransport* t = m.raw;
Downloader d(m.take());
MediaAsset a;
a.data_base64 = "aGVsbG8=";
a.mime = "image/png";
FetchedAsset f = d.fetch(a);
ASSERT_EQ(f.bytes, std::string("hello"));
ASSERT_EQ(f.content_type, std::string("image/png"));
MediaAsset bad;
bad.data_base64 = "@@@";
ASSERT_THROW(d.fetch(bad), ApiError);
ASSERT_THROW(d.fetch(MediaAsset{}), ApiError);
MediaAsset insecure;
insecure.url = "http://example.com/x.png";
ASSERT_THROW(d.fetch(insecure), ApiError);
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
}
TEST(media_downloader, sends_no_credential_and_maps_failures) {
Mock m;
serve(m.raw, Method::kGet, "/missing.png", reply("not found", 404, "text/plain"));
MockTransport* t = m.raw;
Downloader d(m.take());
try {
d.fetch_url("https://cdn.example.com/missing.png");
FAIL("expected a download failure");
} catch (const ApiError& e) {
ASSERT_EQ(e.type(), std::string("DOWNLOAD_FAILED"));
ASSERT_EQ(e.http_status(), 404);
}
auto seen = t->seen();
ASSERT_EQ(seen.size(), static_cast<std::size_t>(1));
ASSERT_FALSE(seen[0].headers.get("Authorization").has_value());
ASSERT_FALSE(seen[0].headers.get("x-key").has_value());
}
+106
View File
@@ -0,0 +1,106 @@
// Opt-in live checks. MOLO_LIVE_NET=1 enables the network; a provider's key
// (via llm::Config) enables its read-only calls (account, balance, models).
// Anything that spends money additionally needs MOLO_LIVE_PAID=1 and is
// limited to a free price quote where the provider offers one (Ideogram
// dry_run) - this file never submits a generation. Without the variables each
// test prints a loud SKIP. Nothing is created or deleted.
#include <cstdlib>
#include <iostream>
#include "clients/transports/http_transport.h"
#include "support.h"
#include "vendors/media/common/downloader.h"
#include "vendors/media/common/registry.h"
#include "vendors/media/bfl/bfl.h"
#include "vendors/media/fal/fal.h"
#include "vendors/media/higgsfield/higgsfield.h"
#include "vendors/media/ideogram/ideogram.h"
#include "vendors/media/kling/kling.h"
#include "vendors/media/leonardo/leonardo.h"
#include "vendors/media/luma/luma.h"
#include "vendors/media/minimax/minimax.h"
#include "vendors/media/recraft/recraft.h"
#include "vendors/media/replicate/replicate.h"
#include "vendors/media/runway/runway.h"
#include "vendors/media/stability/stability.h"
using namespace molo::media;
using namespace media_test;
namespace {
// Read once at startup: other suites in this binary unset MOLO_LIVE_NET while they run.
const bool kLiveNet = std::getenv("MOLO_LIVE_NET") != nullptr;
const bool kLivePaid = std::getenv("MOLO_LIVE_PAID") != nullptr;
bool net() { return kLiveNet; }
void skip(const std::string& name, const std::string& why) { std::cout << "SKIP " << name << " (" << why << ")\n"; }
// Unauthenticated probe: a deliberately invalid key must be refused (the host
// resolves, TLS works, the documented path exists). `accept` lists the HTTP
// statuses seen on 2026-10-25: BFL answers 422 for a malformed key, MiniMax
// answers 200 with base_resp 1004 (turned into an ApiError by the client).
template <class Fn>
void expect_refused(const std::string& name, Fn call, std::initializer_list<int> accept = {401, 403}) {
try {
call();
FAIL(name << ": an invalid key was accepted");
} catch (const ApiError& e) {
std::cout << " " << name << ": " << e.http_status() << " " << e.type() << "\n";
bool ok = false;
for (int a : accept) ok = ok || e.http_status() == a;
ASSERT_TRUE(ok);
}
}
std::unique_ptr<molo::clients::Transport> real() { return std::make_unique<molo::clients::transports::HttpTransport>(); }
} // namespace
TEST(media_live, invalid_keys_are_refused_by_every_provider) {
if (!net()) return skip("media_live.invalid_keys_are_refused", "set MOLO_LIVE_NET=1");
const std::string bad = "molo-invalid-probe";
expect_refused("stability", [&] { StabilityClient::with(real(), bad)->balance(); });
expect_refused("bfl", [&] { BflClient::with(real(), bad)->credits(); }, {401, 403, 422});
expect_refused("replicate", [&] { ReplicateClient::with(real(), bad)->account(); });
expect_refused("fal.platform", [&] { FalPlatform::with(real(), bad)->pricing("fal-ai/flux/dev"); });
expect_refused("fal.queue", [&] { FalQueue::with(real(), bad)->status(FalEndpoint::parse("fal-ai/flux/dev"), "00000000-0000-0000-0000-000000000000"); });
expect_refused("ideogram", [&] { IdeogramClient::with(real(), bad)->api_keys(); });
expect_refused("recraft", [&] { RecraftClient::with(real(), bad)->me(); });
expect_refused("leonardo", [&] { LeonardoClient::with(real(), bad)->models(); });
expect_refused("runway", [&] { RunwayClient::with(real(), bad)->organization(); });
expect_refused("luma", [&] { LumaClient::with(real(), bad)->credits(); });
expect_refused("minimax", [&] { MinimaxClient::with(real(), bad)->query_video("1"); }, {400});
expect_refused("kling", [&] { std::make_shared<KlingClient>("AK-invalid-probe", "SK-invalid-probe")->get("text2video.get", "0"); });
expect_refused("higgsfield", [&] { HiggsfieldClient::with(real(), "a", "b")->status("00000000-0000-0000-0000-000000000000"); });
}
TEST(media_live, read_only_account_calls_for_every_configured_provider) {
MediaRegistry reg;
int ran = 0;
for (const ProviderInfo& p : MediaRegistry::providers()) {
const std::string name = "media_live.account." + p.id;
if (!net()) {
skip(name, "set MOLO_LIVE_NET=1");
continue;
}
if (!MediaRegistry::configured(p.id)) {
skip(name, "set " + p.env[0]);
continue;
}
if (p.account_key.empty()) {
skip(name, p.id + " has no free account read");
continue;
}
Json out = reg.account(p.id);
std::cout << " " << p.id << " " << p.account_key << " -> " << out.dump().substr(0, 120) << "\n";
ASSERT_TRUE(out.is_object());
++ran;
}
std::cout << " account reads run: " << ran << "\n";
}
TEST(media_live, paid_calls_are_not_exercised_here) {
if (!kLivePaid) return skip("media_live.paid", "set MOLO_LIVE_PAID=1 (and read this file: nothing paid runs even then)");
std::cout << " MOLO_LIVE_PAID is set; this file still submits no generation. Use the demo with confirm:true by hand.\n";
}
+430
View File
@@ -0,0 +1,430 @@
// Requests built from the call tables: exact targets, bodies, headers, field
// validation and the confirmation gate. No reply is scripted except where a
// fixture is labelled; prepare() itself never touches a transport.
#include "support.h"
#include "vendors/media/common/caps.h"
#include "vendors/media/bfl/bfl.h"
#include "vendors/media/fal/fal.h"
#include "vendors/media/higgsfield/higgsfield.h"
#include "vendors/media/ideogram/ideogram.h"
#include "vendors/media/kling/kling.h"
#include "vendors/media/leonardo/leonardo.h"
#include "vendors/media/luma/luma.h"
#include "vendors/media/minimax/minimax.h"
#include "vendors/media/recraft/recraft.h"
#include "vendors/media/replicate/replicate.h"
#include "vendors/media/runway/runway.h"
#include "vendors/media/stability/stability.h"
using namespace molo::media;
using namespace media_test;
namespace {
Json obj(std::initializer_list<std::pair<const char*, Json>> items) {
Json o = Json::object();
for (const auto& [k, v] : items) o.set(k, v);
return o;
}
} // namespace
// ------------------------------------------------------------------ Stability
TEST(media_prepare, stability_generate_is_multipart_with_json_accept) {
Mock m;
auto c = StabilityClient::with(m.take(), "KEY");
PreparedCall p = c->plan("stable-image.generate.core", obj({{"prompt", Json("a cat")}, {"output_format", Json("png")}}), {});
ASSERT_EQ(p.method, std::string("POST"));
ASSERT_EQ(p.target, std::string("/v2beta/stable-image/generate/core"));
ASSERT_TRUE(has(p.content_type, "multipart/form-data; boundary="));
ASSERT_TRUE(has(p.body, "name=\"prompt\""));
ASSERT_TRUE(has(p.body, "a cat"));
ASSERT_EQ(p.headers.at("Accept"), std::string("application/json"));
ASSERT_TRUE(p.effect == Effect::kPaid);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_prepare, stability_files_must_use_a_declared_field_and_fit_the_cap) {
Mock m;
auto c = StabilityClient::with(m.take(), "KEY");
FilePart img{"image", "in.png", "image/png", "\x89PNG-bytes"};
PreparedCall p = c->plan("stable-image.upscale.fast", obj({{"output_format", Json("webp")}}), {img});
ASSERT_TRUE(has(p.body, "filename=\"in.png\""));
ASSERT_TRUE(has(p.body, "Content-Type: image/png"));
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"picture", "a.png", "image/png", "x"}}), ApiError);
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"image", "a.png", "image/png", ""}}), ApiError);
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(),
{FilePart{"image", "a.png", "image/png", std::string(MediaCaps::kMaxFileBytes + 1, 'x')}}), ApiError);
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"image", "a\".png", "image/png", "x"}}), ApiError);
}
TEST(media_prepare, stability_rejects_unknown_fields_and_unknown_calls) {
Mock m;
auto c = StabilityClient::with(m.take(), "KEY");
ASSERT_THROW(c->plan("stable-image.generate.core", obj({{"prompt", Json("x")}, {"bogus", Json(1)}}), {}), ApiError);
ASSERT_THROW(c->plan("stable-image.generate.nothing", Json::object(), {}), ApiError);
}
TEST(media_prepare, stability_result_ids_are_validated_before_any_traffic) {
Mock m;
auto c = StabilityClient::with(m.take(), "KEY");
ASSERT_THROW(c->result("../etc"), ApiError);
ASSERT_THROW(c->result("a/b"), ApiError);
ASSERT_THROW(c->audio_result(""), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_prepare, stability_v1_text_to_image_is_json_with_an_engine_in_the_path) {
Mock m;
auto c = StabilityClient::with(m.take(), "KEY");
CallArgs a;
a.path["engine_id"] = "stable-diffusion-xl-1024-v1-0";
a.body = obj({{"samples", Json(1)}, {"steps", Json(30)}});
PreparedCall p = c->prepare("v1.generation.id.text-to-image", a);
ASSERT_EQ(p.target, std::string("/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image"));
ASSERT_EQ(p.content_type, std::string("application/json"));
ASSERT_EQ(p.body, std::string("{\"samples\":1,\"steps\":30}"));
}
// ------------------------------------------------------------------------ BFL
TEST(media_prepare, bfl_body_is_json_and_query_is_encoded) {
Mock m;
auto c = BflClient::with(m.take(), "KEY");
PreparedCall p = c->plan("flux-2-pro", obj({{"prompt", Json("x")}, {"width", Json(1024)}, {"height", Json(768)}, {"seed", Json(7)}}));
ASSERT_EQ(p.target, std::string("/v1/flux-2-pro"));
ASSERT_EQ(p.body, std::string("{\"height\":768,\"prompt\":\"x\",\"seed\":7,\"width\":1024}"));
CallArgs a;
a.query["id"] = "a b&c";
PreparedCall g = c->prepare("get_result", a);
ASSERT_EQ(g.target, std::string("/v1/get_result?id=a%20b%26c"));
ASSERT_EQ(g.body, std::string(""));
CallArgs bad;
bad.query["nope"] = "1";
ASSERT_THROW(c->prepare("get_result", bad), ApiError);
}
TEST(media_prepare, bfl_sends_the_x_key_header_and_never_a_bearer) {
Mock m;
serve(m.raw, Method::kGet, "/v1/credits", empty_ok());
MockTransport* t = m.raw;
auto c = BflClient::with(m.take(), "SECRET-KEY");
c->credits();
auto seen = t->seen();
ASSERT_EQ(seen.size(), static_cast<std::size_t>(1));
ASSERT_EQ(*seen[0].headers.get("x-key"), std::string("SECRET-KEY"));
ASSERT_FALSE(seen[0].headers.get("Authorization").has_value());
}
// ------------------------------------------------------------------ Replicate
TEST(media_prepare, replicate_official_model_and_pinned_version_use_different_rows) {
Mock m;
auto c = ReplicateClient::with(m.take(), "KEY");
Json in = obj({{"prompt", Json("a cat")}});
PreparedCall a = c->plan_prediction(ReplicateClient::ModelRef::parse("black-forest-labs/flux-schnell"), in, Json::object());
ASSERT_EQ(a.target, std::string("/models/black-forest-labs/flux-schnell/predictions"));
ASSERT_EQ(a.body, std::string("{\"input\":{\"prompt\":\"a cat\"}}"));
PreparedCall b = c->plan_prediction(ReplicateClient::ModelRef::parse("owner/name:abc123"), in, Json::object());
ASSERT_EQ(b.target, std::string("/predictions"));
ASSERT_EQ(b.body, std::string("{\"input\":{\"prompt\":\"a cat\"},\"version\":\"abc123\"}"));
Json hook = obj({{"webhook", Json("https://example.com/h")}, {"webhook_events_filter", Json::array()}});
PreparedCall w = c->plan_prediction(ReplicateClient::ModelRef::parse("o/n"), in, hook);
ASSERT_TRUE(has(w.body, "\"webhook\":\"https://example.com/h\""));
ASSERT_THROW(c->plan_prediction(ReplicateClient::ModelRef::parse("o/n"), in, obj({{"nope", Json(1)}})), ApiError);
}
TEST(media_prepare, replicate_model_references_are_validated) {
using R = ReplicateClient::ModelRef;
ASSERT_THROW(R::parse("justaname"), ApiError);
ASSERT_THROW(R::parse("a/b/c"), ApiError);
ASSERT_THROW(R::parse("a/../b"), ApiError);
ASSERT_THROW(R::parse("a/b:v/x"), ApiError);
ASSERT_THROW(R::parse("/b"), ApiError);
R ok = R::parse("o/n:v1");
ASSERT_EQ(ok.owner, std::string("o"));
ASSERT_EQ(ok.version, std::string("v1"));
}
TEST(media_prepare, replicate_uses_the_v1_base_and_bearer_auth) {
Mock m;
serve(m.raw, Method::kGet, "/v1/account", empty_ok());
MockTransport* t = m.raw;
auto c = ReplicateClient::with(m.take(), "tok");
c->account();
auto seen = t->seen();
ASSERT_EQ(seen[0].target, std::string("/v1/account"));
ASSERT_EQ(*seen[0].headers.get("Authorization"), std::string("Bearer tok"));
}
// ------------------------------------------------------------------------ fal
TEST(media_prepare, fal_endpoint_ids_split_into_owner_app_path) {
FalEndpoint e = FalEndpoint::parse("fal-ai/flux/dev");
ASSERT_EQ(e.owner, std::string("fal-ai"));
ASSERT_EQ(e.app, std::string("flux"));
ASSERT_EQ(e.path, std::string("dev"));
ASSERT_EQ(e.app_id(), std::string("fal-ai/flux"));
FalEndpoint two = FalEndpoint::parse("fal-ai/veo3");
ASSERT_EQ(two.path, std::string(""));
FalEndpoint deep = FalEndpoint::parse("fal-ai/kling-video/v2.1/master/text-to-video");
ASSERT_EQ(deep.path, std::string("v2.1/master/text-to-video"));
ASSERT_THROW(FalEndpoint::parse("only"), ApiError);
ASSERT_THROW(FalEndpoint::parse("a/../b"), ApiError);
ASSERT_THROW(FalEndpoint::parse("a//b"), ApiError);
ASSERT_THROW(FalEndpoint::parse("a/b/c/d/e/f"), ApiError);
}
TEST(media_prepare, fal_submit_carries_priority_header_and_webhook_query) {
Mock m;
auto q = FalQueue::with(m.take(), "id:secret");
FalSubmitOptions o;
o.priority = "high";
o.webhook = "https://example.com/h";
o.max_queue_length = 3;
PreparedCall p = q->plan(FalEndpoint::parse("fal-ai/flux/dev"), obj({{"prompt", Json("x")}}), o);
ASSERT_EQ(p.method, std::string("POST"));
ASSERT_EQ(p.target, std::string("/fal-ai/flux/dev?fal_max_queue_length=3&fal_webhook=https%3A%2F%2Fexample.com%2Fh"));
ASSERT_EQ(p.headers.at("X-Fal-Queue-Priority"), std::string("high"));
ASSERT_EQ(p.body, std::string("{\"prompt\":\"x\"}"));
PreparedCall two = q->plan(FalEndpoint::parse("fal-ai/veo3"), Json::object(), FalSubmitOptions{});
ASSERT_EQ(two.target, std::string("/fal-ai/veo3"));
FalSubmitOptions bad;
bad.priority = "urgent";
ASSERT_THROW(q->plan(FalEndpoint::parse("fal-ai/veo3"), Json::object(), bad), ApiError);
}
TEST(media_prepare, fal_status_result_and_cancel_use_owner_app_only) {
Mock m;
auto q = FalQueue::with(m.take(), "k");
ASSERT_THROW(q->status(FalEndpoint::parse("fal-ai/flux/dev"), "../x"), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
// FIXTURE: queue.json cancelQueueRequest example 202 (verbatim, https://fal.ai/docs/api-reference/inference-api/openapi/queue.json)
serve(m.raw, Method::kPut, "/fal-ai/flux/requests/019a1b2c/cancel", reply("{\"status\": \"CANCELLATION_REQUESTED\"}", 202));
serve(m.raw, Method::kGet, "/fal-ai/flux/requests/019a1b2c/status", empty_ok());
Json r = q->cancel(FalEndpoint::parse("fal-ai/flux/dev"), "019a1b2c", true);
ASSERT_EQ(r.find("status")->as_string(), std::string("CANCELLATION_REQUESTED"));
q->status(FalEndpoint::parse("fal-ai/flux/dev"), "019a1b2c", true);
auto seen = m.raw->seen();
ASSERT_EQ(seen[0].method, std::string("PUT"));
ASSERT_TRUE(has(seen[1].target, "/status?logs=1"));
}
TEST(media_prepare, fal_sends_the_key_prefix_on_both_hosts) {
Mock m;
serve(m.raw, Method::kGet, "/v1/meta", empty_ok());
MockTransport* t = m.raw;
auto p = FalPlatform::with(m.take(), "id:secret");
p->meta();
ASSERT_EQ(*t->seen()[0].headers.get("Authorization"), std::string("Key id:secret"));
ASSERT_EQ(FalQueue::header_value(""), std::string(""));
}
// ------------------------------------------------------------------- Ideogram
TEST(media_prepare, ideogram_quote_is_a_free_dry_run_and_generate_is_paid) {
Mock m;
auto c = IdeogramClient::with(m.take(), "KEY");
Json f = obj({{"prompt", Json("a cat")}, {"num_images", Json(1)}});
PreparedCall q = c->plan("image.generate.ideogram-3", f, {}, true);
ASSERT_EQ(q.target, std::string("/v2/image/generate/ideogram-3?dry_run=true"));
ASSERT_TRUE(has(q.content_type, "multipart/form-data")); // ideogram-3 declares multipart only
ASSERT_TRUE(q.effect == Effect::kPaid);
PreparedCall j = c->plan("image.generate.ideogram-4", f, {}, false);
ASSERT_EQ(j.content_type, std::string("application/json"));
ASSERT_EQ(j.target, std::string("/v2/image/generate/ideogram-4"));
ASSERT_THROW(c->plan("image.generate.ideogram-3", obj({{"nope", Json(1)}}), {}), ApiError);
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
}
TEST(media_prepare, ideogram_uses_the_api_key_header) {
Mock m;
serve(m.raw, Method::kGet, "/v2/generations/abc", empty_ok());
MockTransport* t = m.raw;
auto c = IdeogramClient::with(m.take(), "KEY");
c->generation("abc");
ASSERT_EQ(*t->seen()[0].headers.get("Api-Key"), std::string("KEY"));
}
// --------------------------------------------------------------------- Recraft
TEST(media_prepare, recraft_json_by_default_multipart_with_files) {
Mock m;
auto c = RecraftClient::with(m.take(), "KEY");
PreparedCall j = c->plan("images.generations", obj({{"prompt", Json("x")}, {"n", Json(2)}}), {});
ASSERT_EQ(j.target, std::string("/v1/images/generations"));
ASSERT_EQ(j.body, std::string("{\"n\":2,\"prompt\":\"x\"}"));
ASSERT_THROW(c->plan("images.generations", obj({{"prompt", Json("x")}}), {FilePart{"image", "a.png", "image/png", "x"}}),
ApiError); // generations is JSON-only
ASSERT_THROW(c->style("../x"), ApiError);
ASSERT_THROW(c->delete_style("s1", false), ApiError); // gate
}
// --------------------------------------------------------------------- Leonardo
TEST(media_prepare, leonardo_v2_create_wraps_model_and_parameters) {
Mock m;
auto c = LeonardoClient::with(m.take(), "KEY");
Json b = LeonardoJobs::body("flux-dev", obj({{"prompt", Json("x")}}), false);
PreparedCall p = c->plan(b);
ASSERT_EQ(p.target, std::string("/v2/generations"));
ASSERT_EQ(p.body, std::string("{\"model\":\"flux-dev\",\"parameters\":{\"prompt\":\"x\"},\"public\":false}"));
ASSERT_THROW(c->generation("../x"), ApiError);
}
// ---------------------------------------------------------------------- Runway
TEST(media_prepare, runway_sends_the_version_header_and_accepts_both_key_names) {
Mock m;
serve(m.raw, Method::kGet, "/v1/organization", empty_ok());
MockTransport* t = m.raw;
auto c = RunwayClient::with(m.take(), "KEY");
c->organization();
ASSERT_EQ(*t->seen()[0].headers.get("X-Runway-Version"), std::string(RunwayClient::kApiVersion));
ASSERT_EQ(*t->seen()[0].headers.get("Authorization"), std::string("Bearer KEY"));
ASSERT_EQ(std::string(RunwayClient::kKeyEnv), std::string("RUNWAY_API_KEY"));
ASSERT_EQ(std::string(RunwayClient::kKeyEnvSdk), std::string("RUNWAYML_API_SECRET"));
}
TEST(media_prepare, runway_task_bodies_are_checked_against_the_sdk_field_union) {
Mock m;
auto c = RunwayClient::with(m.take(), "KEY");
PreparedCall p = c->plan("text_to_video", obj({{"model", Json("gen4.5")}, {"promptText", Json("x")}, {"duration", Json(5)}}));
ASSERT_EQ(p.target, std::string("/v1/text_to_video"));
ASSERT_THROW(c->plan("text_to_video", obj({{"promt", Json("typo")}})), ApiError);
ASSERT_THROW(c->task("a/b"), ApiError);
ASSERT_THROW(c->cancel_or_delete_task("t1", false), ApiError);
}
// ------------------------------------------------------------------------ Luma
TEST(media_prepare, luma_paths_and_the_dream_machine_base) {
Mock m;
serve(m.raw, Method::kGet, "/dream-machine/v1/credits", empty_ok());
MockTransport* t = m.raw;
auto c = LumaClient::with(m.take(), "KEY");
PreparedCall p = c->plan("generations.create_video", obj({{"model", Json("ray-2")}, {"prompt", Json("x")}}));
ASSERT_EQ(p.target, std::string("/generations/video"));
c->credits();
ASSERT_EQ(t->seen()[0].target, std::string("/dream-machine/v1/credits"));
CallArgs a;
a.path["id"] = "g1";
ASSERT_EQ(c->prepare("generations.upscale", a).target, std::string("/generations/g1/upscale"));
ASSERT_THROW(c->remove("g1", false), ApiError);
}
// --------------------------------------------------------------------- MiniMax
TEST(media_prepare, minimax_v1_and_v2_rows) {
Mock m;
auto c = MinimaxClient::with(m.take(), "KEY");
PreparedCall p = c->plan_video(obj({{"model", Json("MiniMax-Hailuo-2.3")}, {"prompt", Json("x")}, {"duration", Json(6)}}));
ASSERT_EQ(p.target, std::string("/v1/video_generation"));
CallArgs q;
q.query["task_id"] = "t1";
ASSERT_EQ(c->prepare("video.query", q).target, std::string("/v1/query/video_generation?task_id=t1"));
CallArgs d;
d.path["task_id"] = "t1";
ASSERT_EQ(c->prepare("v2.video.delete", d).target, std::string("/v2/video_generation/t1"));
ASSERT_THROW(c->delete_video_v2("t1", false), ApiError);
Json ok = obj({{"base_resp", obj({{"status_code", Json(0)}, {"status_msg", Json("success")}})}});
MinimaxClient::check_base_resp(ok); // no throw
Json bad = obj({{"base_resp", obj({{"status_code", Json(1004)}, {"status_msg", Json("login fail")}})}});
ASSERT_THROW(MinimaxClient::check_base_resp(bad), ApiError);
}
// ------------------------------------------------------------------ Higgsfield
TEST(media_prepare, higgsfield_typed_rows_for_known_paths_and_a_generic_row_for_the_rest) {
ASSERT_EQ(HiggsfieldClient::row_for("higgsfield-ai/soul/standard"), std::string("higgsfield-ai.soul.standard"));
ASSERT_EQ(HiggsfieldClient::row_for("vendor/new-model/v9"), std::string("model.submit"));
Mock m;
auto c = HiggsfieldClient::with(m.take(), "id", "secret");
PreparedCall typed = c->plan("higgsfield-ai/soul/standard", obj({{"prompt", Json("x")}}));
ASSERT_EQ(typed.target, std::string("/higgsfield-ai/soul/standard"));
ASSERT_THROW(c->plan("higgsfield-ai/soul/standard", obj({{"nope", Json(1)}})), ApiError); // typed row validates fields
PreparedCall generic = c->plan("vendor/new-model/v9", obj({{"anything", Json(1)}}));
ASSERT_EQ(generic.target, std::string("/vendor/new-model/v9"));
ASSERT_THROW(c->plan("vendor/../x", Json::object()), ApiError);
ASSERT_EQ(HiggsfieldClient::header_value("id", "secret"), std::string("Key id:secret"));
ASSERT_EQ(HiggsfieldClient::header_value("id", ""), std::string(""));
}
TEST(media_prepare, higgsfield_submit_and_status_over_the_wire) {
Mock m;
// FIXTURE: docs.higgsfield.ai quickstart, submit reply (verbatim)
const char* kSubmit =
"{\"status\":\"queued\",\"request_id\":\"d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff\","
"\"status_url\":\"https://api.higgsfield.ai/requests/d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff/status\","
"\"cancel_url\":\"https://api.higgsfield.ai/requests/d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff/cancel\"}";
serve(m.raw, Method::kPost, "/higgsfield-ai/soul/standard", reply(kSubmit));
MockTransport* t = m.raw;
auto c = HiggsfieldClient::with(m.take(), "id", "secret");
Json out = c->submit("higgsfield-ai/soul/standard", obj({{"prompt", Json("Editorial portrait in soft daylight")}}), true);
JobTicket ticket = HiggsfieldJobs::ticket_from(out, "higgsfield-ai/soul/standard", MediaKind::kImage);
ASSERT_EQ(ticket.id, std::string("d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff"));
auto seen = t->seen();
ASSERT_EQ(*seen[0].headers.get("Authorization"), std::string("Key id:secret"));
ASSERT_EQ(seen[0].body, std::string("{\"prompt\":\"Editorial portrait in soft daylight\"}"));
}
// ----------------------------------------------------------------------- Kling
TEST(media_prepare, kling_rows_and_envelope) {
Mock m;
auto c = std::make_shared<KlingClient>("AK", "SK", [] { return std::make_unique<MockTransport>(); });
PreparedCall p = c->plan("text2video.create", obj({{"model_name", Json("kling-v1-6")}, {"prompt", Json("x")}}));
ASSERT_EQ(p.target, std::string("/v1/videos/text2video"));
ASSERT_TRUE(c->configured());
KlingClient::check_envelope(obj({{"code", Json(0)}}));
ASSERT_THROW(KlingClient::check_envelope(obj({{"code", Json(1001)}, {"message", Json("Authorization is empty")}})), ApiError);
auto none = std::make_shared<KlingClient>("", "", [] { return std::make_unique<MockTransport>(); });
ASSERT_FALSE(none->configured());
}
// ------------------------------------------------------- confirmation gate
TEST(media_gate, non_read_calls_need_confirm_and_make_no_traffic_without_it) {
Mock m;
MockTransport* t = m.raw;
auto bfl = BflClient::with(m.take(), "KEY");
ASSERT_THROW(bfl->submit("flux-2-pro", obj({{"prompt", Json("x")}}), false), ApiError);
ASSERT_THROW(bfl->delete_finetune("my-ft", false), ApiError);
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
Mock m2;
MockTransport* t2 = m2.raw;
auto rep = ReplicateClient::with(m2.take(), "KEY");
ASSERT_THROW(rep->cancel_prediction("p1", false), ApiError);
ASSERT_THROW(rep->file_delete("f1", false), ApiError);
ASSERT_THROW(rep->file_upload("a.png", "image/png", "x", false), ApiError);
ASSERT_THROW(rep->create_prediction(ReplicateClient::ModelRef::parse("o/n"), Json::object(), Json::object(), false), ApiError);
ASSERT_EQ(t2->send_count(), static_cast<std::size_t>(0));
}
TEST(media_gate, missing_credentials_fail_before_any_traffic) {
Mock m;
MockTransport* t = m.raw;
auto c = BflClient::with(m.take(), "");
ASSERT_FALSE(c->configured());
ASSERT_THROW(c->credits(), molo::llm::NotConfiguredError);
ASSERT_THROW(c->submit("flux-2-pro", obj({{"prompt", Json("x")}}), true), molo::llm::NotConfiguredError);
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
ASSERT_FALSE(FalQueue::with(std::make_unique<MockTransport>(), "")->configured());
ASSERT_FALSE(HiggsfieldClient::with(std::make_unique<MockTransport>(), "id", "")->configured());
}
TEST(media_gate, headers_and_query_are_bounded) {
Mock m;
auto c = BflClient::with(m.take(), "KEY");
CallArgs a;
a.headers["Bad Name"] = "v";
ASSERT_THROW(c->prepare("credits", a), ApiError);
CallArgs b;
b.headers["X-A"] = "line\r\nbreak";
ASSERT_THROW(c->prepare("credits", b), ApiError);
CallArgs d;
d.query["id"] = std::string(2000, 'x');
ASSERT_THROW(c->prepare("get_result", d), ApiError);
CallArgs e;
e.body = obj({{"x", Json(1)}});
ASSERT_THROW(c->prepare("credits", e), ApiError); // a GET takes no body
}
+61
View File
@@ -0,0 +1,61 @@
// Webhook signature verification (Replicate, Standard Webhooks scheme) and the
// Kling JWT. Both use vectors produced outside this code base.
#include "support.h"
#include "vendors/media/kling/kling.h"
#include "vendors/media/replicate/replicate.h"
using namespace molo::media;
using namespace media_test;
namespace {
// Standard Webhooks published test vector (github.com/standard-webhooks): secret, id, timestamp, payload.
const char* kSecret = "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw";
const char* kId = "msg_p5jXN8AQM9LWM0D4loKWxJek";
const char* kTimestamp = "1614265330";
const char* kPayload = "{\"test\": 2432232314}";
const char* kSignature = "g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=";
} // namespace
TEST(media_signing, webhook_signature_matches_the_standard_webhooks_vector) {
ASSERT_EQ(ReplicateWebhook::sign(kSecret, kId, kTimestamp, kPayload), std::string(kSignature));
ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, std::string("v1,") + kSignature, kPayload, 1614265330 + 10));
}
TEST(media_signing, webhook_rejects_tampering_staleness_and_malformed_input) {
const std::string good = std::string("v1,") + kSignature;
const long now = 1614265330 + 10;
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, "{\"test\": 2432232315}", now)); // body changed
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, "msg_other", kTimestamp, good, kPayload, now)); // id changed
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, "1614265331", good, kPayload, now)); // timestamp changed
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 + 301)); // too old
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 - 301)); // too new
ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 + 300)); // boundary
ASSERT_FALSE(ReplicateWebhook::verify("whsec_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", kId, kTimestamp, good, kPayload, now)); // wrong secret
ASSERT_FALSE(ReplicateWebhook::verify("whsec_!!notbase64!!", kId, kTimestamp, good, kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify("", kId, kTimestamp, good, kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, "", kTimestamp, good, kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, "12ab", good, kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "", kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, kSignature, kPayload, now)); // no v1, prefix
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "v2," + std::string(kSignature), kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, std::string(2000, 'x'), kPayload, now)); // header cap
}
TEST(media_signing, webhook_accepts_any_of_several_signatures) {
const long now = 1614265330;
std::string rotated = "v1,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= v1," + std::string(kSignature);
ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, rotated, kPayload, now));
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "v1,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", kPayload, now));
}
TEST(media_signing, kling_jwt_matches_an_independent_hs256_implementation) {
// Computed with python3 hmac / base64 (see SPEC "Verified against spec"): header {"alg":"HS256","typ":"JWT"},
// payload {"exp":1700001800,"iss":"AK-test","nbf":1699999995}, secret "SK-test".
const char* expected =
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MDAwMDE4MDAsImlzcyI6IkFLLXRlc3QiLCJuYmYiOjE2OTk5OTk5OTV9."
"f3MPnq8kVzVYtGGChN2U_Rd1oYs3-a_FFYC4PCVGdSc";
ASSERT_EQ(KlingJwt::make("AK-test", "SK-test", 1700000000), std::string(expected));
ASSERT_TRUE(KlingJwt::make("AK-test", "SK-other", 1700000000) != std::string(expected));
ASSERT_EQ(KlingJwt::kTtlSeconds, 1800L);
ASSERT_TRUE(KlingJwt::kRenewAfterSeconds < KlingJwt::kTtlSeconds);
}