diff --git a/example/app/ai/MediaDemo.cpp b/example/app/ai/MediaDemo.cpp index 398386b..72d8a07 100644 --- a/example/app/ai/MediaDemo.cpp +++ b/example/app/ai/MediaDemo.cpp @@ -1,13 +1,589 @@ #include "../AiDemos.hpp" +#include +#include + #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "security/crypto/Base64.h" +#include "security/crypto/Sha256.h" +#include "vendors/media/common/caps.h" +#include "vendors/media/common/downloader.h" +#include "vendors/media/common/job_runner.h" +#include "vendors/media/common/registry.h" +#include "vendors/media/replicate/replicate.h" namespace molo::example::app { -// media. Stub: the package agent fills this in. +// Package media over the real src/vendors/media: image and video generation +// across twelve providers behind one seam (submit, poll, fetch), plus a +// generic call route over every provider's call table. GET routes need no +// credential. Every route that calls a provider is POST or DELETE: 422 on bad +// input, 503 media_not_configured without the provider's credential. Anything +// that costs money or changes a resource is a dry run showing the exact +// request unless the body has "confirm":true. + +namespace { + +namespace md = media; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 24 * 1024 * 1024; // uploads travel as base64 +constexpr std::size_t kInlineAssetCap = 4 * 1024 * 1024; // base64 echoed back for inline results +constexpr std::size_t kTextPreviewCap = 64 * 1024; +constexpr int kDemoMaxWaitSeconds = 60; // below the package cap: a request thread is held +constexpr int kDemoMaxAttempts = 30; + +const char kPage[] = R"MEDIA({% extends "base.html" %} +{% block content %} +

Media generation

+

Images and video from Stability, Black Forest Labs, Runway, Luma, Replicate, fal, Ideogram, +Recraft, Leonardo, MiniMax, Kling and Higgsfield behind one submit, poll, fetch seam. Every call that costs +money or changes a resource is a dry run until the body says "confirm":true.

+

Checking credentials...

+
+ + + +
+ + +
+ +

+
Not available
    +
    + + +{% endblock %} +)MEDIA"; + +[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string text(const Json& b, const char* key, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + return v->as_string(); +} + +int number(const Json& b, const char* key, int fallback = 0) { + const Json* v = b.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + return static_cast(v->as_int()); +} + +long seed_of(const Json& b) { + const Json* v = b.find("seed"); + if (!v || v->is_null()) return -1; + if (!v->is_number()) bad("seed must be a number"); + return v->as_int(); +} + +bool flag(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +Json object_at(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return Json::object(); + if (!v->is_object()) bad(std::string(key) + " must be a JSON object"); + return *v; +} + +std::map strings_at(const Json& b, const char* key) { + std::map out; + const Json* v = b.find(key); + if (!v || v->is_null()) return out; + if (!v->is_object()) bad(std::string(key) + " must be a JSON object of strings"); + if (v->as_object().size() > md::MediaCaps::kMaxQueryArgs) bad(std::string(key) + " has too many entries"); + for (const auto& [k, x] : v->as_object()) { + if (x.is_string()) out[k] = x.as_string(); + else if (x.is_number() || x.is_bool()) out[k] = x.dump(); + else bad(std::string(key) + "." + k + " must be a string"); + } + return out; +} + +Json dry_run(const Json& plan, const char* what) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set("what", Json(what)); + o.set("request", plan); + o.set("hint", Json("send confirm:true to run it; this request costs money or changes a resource")); + return o; +} + +Json asset_json(const md::MediaAsset& a) { + Json o = a.to_json(); + if (!a.data_base64.empty()) { + if (a.data_base64.size() <= kInlineAssetCap) o.set("data_base64", Json(a.data_base64)); + else o.set("note", Json("inline data over the echo cap; fetch it with the provider client")); + } + return o; +} + +Json status_json(const md::JobStatus& s) { + Json o = s.to_json(); + Json arr = Json::array(); + for (const md::MediaAsset& a : s.assets) arr.push(asset_json(a)); + o.set("assets", arr); + return o; +} + +Json ticket_reply(const md::JobTicket& t) { + Json o = Json::object(); + Json tj = t.to_json(); + Json arr = Json::array(); + for (const md::MediaAsset& a : t.assets) arr.push(asset_json(a)); + tj.set("assets", arr); + o.set("ticket", tj); + o.set("next", Json(t.immediate ? "the result is in ticket.assets" : "POST /ai/media/poll or /ai/media/wait with the ticket")); + return o; +} + +class MediaRoutes { + public: + // ---- read-only GET data --------------------------------------------------- + Json status() { + std::lock_guard lock(mu_); + Json o = Json::object(); + Json arr = Json::array(); + for (const md::ProviderInfo& p : md::MediaRegistry::providers()) arr.push(provider_json(p)); + o.set("providers", arr); + Json un = Json::array(); + for (const md::UnsupportedProduct& u : md::MediaRegistry::unsupported()) { + Json x = Json::object(); + x.set("name", Json(u.name)); + x.set("reason", Json(u.reason)); + un.push(x); + } + o.set("unsupported", un); + Json caps = Json::object(); + caps.set("max_poll_attempts", Json(md::MediaCaps::kMaxPollAttempts)); + caps.set("min_poll_interval_ms", Json(md::MediaCaps::kMinPollIntervalMs)); + caps.set("max_wait_seconds", Json(md::MediaCaps::kMaxWaitSeconds)); + caps.set("demo_max_wait_seconds", Json(kDemoMaxWaitSeconds)); + caps.set("max_images_per_request", Json(md::MediaCaps::kMaxImagesPerRequest)); + caps.set("max_video_seconds", Json(md::MediaCaps::kMaxVideoSeconds)); + caps.set("max_file_bytes", Json(static_cast(md::MediaCaps::kMaxFileBytes))); + caps.set("max_download_bytes", Json(static_cast(md::MediaCaps::kMaxDownloadBytes))); + o.set("caps", caps); + return o; + } + + Json calls(const std::string& provider) { + if (provider.empty()) { + Json o = Json::object(); + Json arr = Json::array(); + for (const md::ProviderInfo& p : md::MediaRegistry::providers()) { + Json x = Json::object(); + x.set("provider", Json(p.id)); + x.set("calls", Json(static_cast(md::MediaRegistry::table(p.id).size()))); + arr.push(x); + } + o.set("providers", arr); + return o; + } + if (!md::MediaRegistry::find(provider)) bad("unknown provider '" + provider + "'"); + const md::CallTable& t = md::MediaRegistry::table(provider); + Json arr = Json::array(); + for (std::size_t i = 0; i < t.size(); ++i) { + const md::CallRow& r = t.at(i); + Json x = Json::object(); + x.set("key", Json(r.key)); + x.set("http", Json(r.http_method)); + x.set("path", Json(r.path)); + x.set("effect", Json(r.effect)); + x.set("body", Json(r.body_kind)); + arr.push(x); + } + Json o = Json::object(); + o.set("provider", Json(provider)); + o.set("calls", arr); + return o; + } + + // ---- generation seam ------------------------------------------------------ + fw::Response image(const fw::Request& req) { + Json b = parse_body(req); + std::string provider = provider_of(b, true); + md::ImageRequest r; + r.prompt = text(b, "prompt", true); + r.negative_prompt = text(b, "negative_prompt"); + r.model = text(b, "model"); + r.width = number(b, "width"); + r.height = number(b, "height"); + r.aspect_ratio = text(b, "aspect_ratio"); + r.count = number(b, "count", 1); + r.seed = seed_of(b); + r.source_url = text(b, "source_url"); + r.extra = object_at(b, "extra"); + std::shared_ptr gen; + Json plan; + try { + std::lock_guard lock(mu_); + gen = reg_.image(provider); + r.validate(); + plan = gen->plan(r); + } catch (const llm::ApiError& e) { + map(e); + } + if (!flag(b, "confirm")) return json_out(dry_run(plan, "image generation")); + try { + return json_out(ticket_reply(gen->submit(r, true))); + } catch (const llm::ApiError& e) { + map(e); + } + } + + fw::Response video(const fw::Request& req) { + Json b = parse_body(req); + std::string provider = provider_of(b, false); + md::VideoRequest r; + r.prompt = text(b, "prompt"); + r.negative_prompt = text(b, "negative_prompt"); + r.model = text(b, "model"); + r.duration_seconds = number(b, "duration_seconds"); + r.aspect_ratio = text(b, "aspect_ratio"); + r.resolution = text(b, "resolution"); + r.seed = seed_of(b); + r.image_url = text(b, "image_url"); + r.end_image_url = text(b, "end_image_url"); + r.extra = object_at(b, "extra"); + std::shared_ptr gen; + Json plan; + try { + std::lock_guard lock(mu_); + gen = reg_.video(provider); + r.validate(); + plan = gen->plan(r); + } catch (const llm::ApiError& e) { + map(e); + } + if (!flag(b, "confirm")) return json_out(dry_run(plan, "video generation")); + try { + return json_out(ticket_reply(gen->submit(r, true))); + } catch (const llm::ApiError& e) { + map(e); + } + } + + fw::Response poll(const fw::Request& req) { + Json b = parse_body(req); + md::JobTicket t = ticket_of(b); + try { + require_configured(t.provider); + return json_out(status_json(generator_poll(t))); + } catch (const llm::ApiError& e) { + map(e); + } + } + + fw::Response wait(const fw::Request& req) { + Json b = parse_body(req); + md::JobTicket t = ticket_of(b); + md::WaitPolicy policy; + policy.max_attempts = number(b, "max_attempts", 15); + policy.max_wait_seconds = number(b, "max_wait_seconds", 45); + policy.first_interval_ms = number(b, "first_interval_ms", 0); + if (policy.max_attempts > kDemoMaxAttempts) bad("max_attempts is limited to " + std::to_string(kDemoMaxAttempts) + " in the demo"); + if (policy.max_wait_seconds > kDemoMaxWaitSeconds) + bad("max_wait_seconds is limited to " + std::to_string(kDemoMaxWaitSeconds) + " in the demo"); + try { + md::JobRunner runner(policy); // validates against the package caps + require_configured(t.provider); + std::shared_ptr gen = generator_for(t); + return json_out(status_json(runner.wait(*gen, t))); + } catch (const llm::ApiError& e) { + map(e); + } + } + + fw::Response cancel(const fw::Request& req) { + Json b = parse_body(req); + md::JobTicket t = ticket_of(b); + Json plan = Json::object(); + plan.set("provider", Json(t.provider)); + plan.set("job", Json(t.id)); + plan.set("action", Json("cancel (some providers delete the generation)")); + if (!flag(b, "confirm")) return json_out(dry_run(plan, "cancel a job")); + try { + require_configured(t.provider); + bool done = generator_for(t)->cancel(t, true); + Json o = Json::object(); + o.set("cancelled", Json(done)); + if (!done) o.set("note", Json(t.provider + " has no cancel endpoint; nothing was sent")); + return json_out(o); + } catch (const llm::ApiError& e) { + map(e); + } + } + + // ---- generic table call --------------------------------------------------- + fw::Response call(const fw::Request& req) { + Json b = parse_body(req); + std::string provider = provider_of(b, false, false); + std::string key = text(b, "key", true); + md::CallArgs args; + args.path = strings_at(b, "path"); + args.query = strings_at(b, "query"); + args.headers = strings_at(b, "headers"); + args.body = object_at(b, "body"); + if (const Json* files = b.find("files"); files && !files->is_null()) { + if (!files->is_array() || files->as_array().size() > md::MediaCaps::kMaxFiles) bad("files must be an array of at most 16 objects"); + for (const Json& f : files->as_array()) { + if (!f.is_object()) bad("each file must be an object"); + md::FilePart part; + part.field = text(f, "field", true); + part.filename = text(f, "filename"); + part.content_type = text(f, "content_type"); + if (!security::crypto::Base64::decode(text(f, "data_base64", true), part.data)) bad("data_base64 is not valid base64"); + args.files.push_back(part); + } + } + md::PreparedCall prepared; + std::shared_ptr service; + try { + std::lock_guard lock(mu_); + service = reg_.service(provider, key); + prepared = service->prepare(key, args); + } catch (const llm::ApiError& e) { + map(e); + } + if (prepared.effect != md::Effect::kRead && !flag(b, "confirm")) return json_out(dry_run(prepared.summary(), "provider call")); + try { + llm::RawResponse raw = service->execute(prepared, true); + Json o = Json::object(); + o.set("request", prepared.summary()); + o.set("status", Json(raw.status)); + o.set("content_type", Json(raw.content_type)); + o.set("bytes", Json(static_cast(raw.body.size()))); + if (raw.content_type.find("json") != std::string::npos) { + try { + o.set("json", Json::parse(raw.body)); + } catch (const std::exception&) { + o.set("text", Json(raw.body.substr(0, kTextPreviewCap))); + } + } else if (raw.content_type.rfind("text/", 0) == 0) { + o.set("text", Json(raw.body.substr(0, kTextPreviewCap))); + } else { + o.set("note", Json("binary reply; use the generation seam or the provider client to fetch it")); + } + return json_out(o); + } catch (const llm::ApiError& e) { + map(e); + } + } + + fw::Response account(const fw::Request& req) { + Json b = parse_body(req); + std::string provider = provider_of(b, false, false); + if (md::MediaRegistry::find(provider)->account_key.empty()) bad(provider + " has no free account read"); + try { + std::lock_guard lock(mu_); + require_configured_locked(provider); + Json o = Json::object(); + o.set("provider", Json(provider)); + o.set("read", Json(md::MediaRegistry::find(provider)->account_key)); + o.set("result", reg_.account(provider)); + return json_out(o); + } catch (const llm::ApiError& e) { + map(e); + } + } + + // Downloads a result URL (public or pre-signed) and reports its size and + // digest; no provider credential is involved. + fw::Response fetch(const fw::Request& req) { + Json b = parse_body(req); + std::string url = text(b, "url", true); + try { + md::Downloader::validate_url(url); + md::Downloader dl; + std::string type; + std::string bytes = dl.fetch_url(url, &type); + Json o = Json::object(); + o.set("bytes", Json(static_cast(bytes.size()))); + o.set("content_type", Json(type)); + o.set("sha256", Json(security::crypto::Sha256::hex(security::crypto::Sha256::hash(bytes)))); + if (bytes.size() <= 128 * 1024) o.set("data_base64", Json(security::crypto::Base64::encode(bytes))); + return json_out(o); + } catch (const llm::ApiError& e) { + map(e); + } + } + + fw::Response webhook(const fw::Request& req) { + Json b = parse_body(req); + std::string secret = text(b, "secret", true); + std::string id = text(b, "webhook_id", true); + std::string ts = text(b, "timestamp", true); + std::string sig = text(b, "signature", true); + std::string body = text(b, "payload", true); + if (body.size() > md::MediaCaps::kMaxWebhookBodyBytes) bad("payload is over the webhook body cap"); + bool ok = md::ReplicateWebhook::verify(secret, id, ts, sig, body, static_cast(std::time(nullptr))); + Json o = Json::object(); + o.set("valid", Json(ok)); + o.set("scheme", Json("Standard Webhooks (HMAC-SHA256 over id.timestamp.payload)")); + o.set("tolerance_seconds", Json(static_cast(md::MediaCaps::kWebhookToleranceSeconds))); + return json_out(o); + } + + private: + Json provider_json(const md::ProviderInfo& p) { + Json x = Json::object(); + x.set("id", Json(p.id)); + x.set("name", Json(p.name)); + x.set("configured", Json(md::MediaRegistry::configured(p.id))); + Json env = Json::array(); + for (const std::string& e : p.env) env.push(Json(e)); + x.set("env", env); + x.set("image", Json(p.image)); + x.set("video", Json(p.video)); + x.set("evidence", Json(md::MediaRegistry::evidence_name(p.evidence))); + x.set("console", Json(p.console_url)); + x.set("account_read", Json(p.account_key)); + x.set("note", Json(p.note)); + x.set("calls", Json(static_cast(md::MediaRegistry::table(p.id).size()))); + Json models = Json::array(); + if (p.image) + for (const std::string& m : reg_.image(p.id)->models()) models.push(Json(m)); + x.set("image_models", models); + Json vmodels = Json::array(); + if (p.video) + for (const std::string& m : reg_.video(p.id)->models()) vmodels.push(Json(m)); + x.set("video_models", vmodels); + return x; + } + + std::string provider_of(const Json& b, bool image, bool generating = true) { + std::string id = text(b, "provider", true); + const md::ProviderInfo* p = md::MediaRegistry::find(id); + if (!p) bad("unknown provider '" + id + "'"); + if (generating && image && !p->image) bad(id + " has no image generation"); + if (generating && !image && !p->video) bad(id + " has no video generation"); + return id; + } + + md::JobTicket ticket_of(const Json& b) { + const Json* t = b.find("ticket"); + if (!t || !t->is_object()) bad("ticket is required (the object returned by image or video)"); + try { + md::JobTicket ticket = md::JobTicket::from_json(*t); + if (!md::MediaRegistry::find(ticket.provider)) bad("unknown provider '" + ticket.provider + "'"); + return ticket; + } catch (const llm::ApiError& e) { + map(e); + } + } + + std::shared_ptr generator_for(const md::JobTicket& t) { + std::lock_guard lock(mu_); + if (t.kind == md::MediaKind::kVideo) return reg_.video(t.provider); + return reg_.image(t.provider); + } + + md::JobStatus generator_poll(const md::JobTicket& t) { + if (t.immediate) { + md::JobStatus s; + s.state = md::JobState::kSucceeded; + s.raw_state = "immediate"; + s.assets = t.assets; + return s; + } + return generator_for(t)->poll(t); + } + + void require_configured(const std::string& provider) { + std::lock_guard lock(mu_); + require_configured_locked(provider); + } + void require_configured_locked(const std::string& provider) { + if (!md::MediaRegistry::configured(provider)) { + const md::ProviderInfo* p = md::MediaRegistry::find(provider); + throw llm::NotConfiguredError(provider, p->env[0]); + } + } + + [[noreturn]] static void map(const llm::ApiError& e) { + if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") + throw fw::HttpError(503, "media_not_configured", e.message()); + if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) + throw fw::HttpError(422, "validation_error", e.message()); + if (e.type() == "WAIT_EXPIRED") throw fw::HttpError(504, "media_wait_expired", e.message()); + throw fw::HttpError(502, "media_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message()); + } + + std::mutex mu_; + md::MediaRegistry reg_; +}; + +} // namespace + void register_ai_media_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + auto routes = std::make_shared(); + fw::Environment* env = services.templates.get(); + services.templates->add_template("ai_media.html", kPage); + services.add_feature("ai-media", "/ai/media"); + + router.get("/ai/media", "ai-media-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Media generation"); + return fw::Response(200, env->render("ai_media.html", ctx), "text/html; charset=utf-8"); + }); + router.get("/ai/media/status", "ai-media-status", [routes](fw::Request req) { + (void)req; + return json_out(routes->status()); + }); + router.get("/ai/media/calls", "ai-media-calls", [routes](fw::Request req) { + return json_out(routes->calls(req.query_or("provider", ""))); + }); + + using Handler = fw::Response (MediaRoutes::*)(const fw::Request&); + auto bind = [routes](Handler h) { return [routes, h](fw::Request req) { return ((*routes).*h)(req); }; }; + router.post("/ai/media/image", "ai-media-image", bind(&MediaRoutes::image)); + router.post("/ai/media/video", "ai-media-video", bind(&MediaRoutes::video)); + router.post("/ai/media/poll", "ai-media-poll", bind(&MediaRoutes::poll)); + router.post("/ai/media/wait", "ai-media-wait", bind(&MediaRoutes::wait)); + router.post("/ai/media/call", "ai-media-call", bind(&MediaRoutes::call)); + router.post("/ai/media/account", "ai-media-account", bind(&MediaRoutes::account)); + router.post("/ai/media/fetch", "ai-media-fetch", bind(&MediaRoutes::fetch)); + router.post("/ai/media/webhook", "ai-media-webhook", bind(&MediaRoutes::webhook)); + router.del("/ai/media/job", "ai-media-cancel", bind(&MediaRoutes::cancel)); } } // namespace molo::example::app diff --git a/example/assets/ai_media.js b/example/assets/ai_media.js new file mode 100644 index 0000000..aab7709 --- /dev/null +++ b/example/assets/ai_media.js @@ -0,0 +1,102 @@ +// Media page: thin client over /ai/media/* (all limits are enforced server side). +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("md-out"), err = $("md-error"); + var providers = []; + var last = null; // last ticket returned by image / video + + var routes = { + "image": function (p) { return { provider: p, prompt: "A lighthouse at dusk, oil painting", model: "", aspect_ratio: "", count: 1 }; }, + "video": function (p) { return { provider: p, prompt: "Waves rolling into a harbour", model: "", duration_seconds: 5 }; }, + "poll": function () { return { ticket: last || {} }; }, + "wait": function () { return { ticket: last || {}, max_attempts: 10, max_wait_seconds: 40 }; }, + "job": function () { return { ticket: last || {} }; }, + "account": function (p) { return { provider: p }; }, + "call": function (p) { return { provider: p, key: "", path: {}, query: {}, body: {} }; }, + "fetch": function () { return { url: "https://example.com/result.png" }; }, + "webhook": function () { return { secret: "whsec_...", webhook_id: "msg_...", timestamp: "0", signature: "v1,...", payload: "{}" }; } + }; + var labels = { + "image": "image: submit a generation", + "video": "video: submit a generation", + "poll": "poll: one status check", + "wait": "wait: poll under caps", + "job": "job: cancel (DELETE)", + "account": "account: free credential check", + "call": "call: any table row", + "fetch": "fetch: download a result URL", + "webhook": "webhook: verify a Replicate signature" + }; + + var route = $("md-route"), prov = $("md-provider"); + Object.keys(routes).forEach(function (k) { + var o = document.createElement("option"); + o.value = k; o.textContent = labels[k]; + route.appendChild(o); + }); + + function fill() { + var p = prov.value || "fal"; + $("md-body").value = JSON.stringify(routes[route.value](p), null, 1); + } + route.onchange = fill; + prov.onchange = fill; + + function send(confirm) { + var body; + try { body = JSON.parse($("md-body").value || "{}"); } + catch (e) { err.textContent = "Body is not valid JSON"; return; } + if (confirm) body.confirm = true; + err.textContent = ""; + out.textContent = "..."; + var key = route.value; + var url = "/ai/media/" + (key === "job" ? "job" : key); + fetch(url, { + method: key === "job" ? "DELETE" : "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; return; } + if (j.ticket) last = j.ticket; + out.textContent = JSON.stringify(j, null, 2); + var imgs = (j.ticket && j.ticket.assets) || j.assets || []; + imgs.forEach(function (a) { + if (a.data_base64 && /^image\//.test(a.mime || "")) { + var im = document.createElement("img"); + im.src = "data:" + a.mime + ";base64," + a.data_base64; + im.style.maxWidth = "100%"; + out.appendChild(document.createElement("br")); + out.appendChild(im); + } + }); + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + $("md-send").onclick = function () { send(false); }; + $("md-confirm").onclick = function () { send(true); }; + + fetch("/ai/media/status").then(function (r) { return r.json(); }).then(function (s) { + providers = s.providers || []; + providers.forEach(function (p) { + var o = document.createElement("option"); + o.value = p.id; + o.textContent = p.name + (p.configured ? "" : " (no key)"); + prov.appendChild(o); + }); + var ready = providers.filter(function (p) { return p.configured; }).length; + $("md-config").textContent = ready + ? ready + " of " + providers.length + " providers have a credential. Paid calls need confirm." + : "No provider credential found: calls answer 503 until one is configured; dry runs still work."; + var ul = $("md-unsupported"); + (s.unsupported || []).forEach(function (u) { + var li = document.createElement("li"); + li.textContent = u.name + ": " + u.reason; + ul.appendChild(li); + }); + fill(); + }); +})(); diff --git a/src/vendors/media/SPEC.md b/src/vendors/media/SPEC.md new file mode 100644 index 0000000..7da9821 --- /dev/null +++ b/src/vendors/media/SPEC.md @@ -0,0 +1,254 @@ +# Package media - Specification + +Owned directory: `src/vendors/media/`. Namespace `molo::media`. One subdirectory +per provider, one shared `common/`. Depends on `molo::llm` (`ApiClient`, +`Service`, `ApiError`, `Config`), `molo::clients` and `security/crypto` +(Base64, Sha256, Hmac). Plan: `plans/18-breadth.md` section 2, method of +`plans/17-llm-vendor-platforms.md`. Demo: `example/app/ai/MediaDemo.cpp`, +routes `/ai/media`. Nothing in `src/llm` was edited. + +## 1. Scope and module map + +Twelve providers, one call table each (`

    /calls.cpp`, checked by +`tools/check_spec.py`), one client class per host, generator adapters on a +neutral seam. + +| Directory | Classes | Covers | +|---|---|---| +| `common/` | `CallTable`, `MediaService`, `PreparedCall`, `ImageGenerator`, `VideoGenerator`, `JobTicket`, `JobStatus`, `JobRunner`, `Downloader`, `OutputScanner`, `MediaRegistry`, `ClientFactory`, `AdapterSupport`, `MediaCaps` | the seam, request building, polling, downloads, provider list | +| `stability/` | `StabilityClient`, `StabilityImageGenerator` | v2beta Stable Image (generate ultra / core / sd3, edit x7, upscale x3, control x4), 3D (2), Stable Audio (6 + results), v1 engines list, account, balance, v1 text-to-image / image-to-image / masking | +| `bfl/` | `BflClient`, `BflImageGenerator`, `BflVideoGenerator` | all 25 FLUX model and tool endpoints, get_result polling, credits, fine-tune list / details / delete | +| `runway/` | `RunwayClient`, `RunwayImageGenerator`, `RunwayVideoGenerator` | every SDK route: generation (18), recipes (7), tasks, uploads, organization, avatars, voices, documents, routers, workflows, realtime sessions | +| `luma/` | `LumaClient`, `LumaVideoGenerator`, `LumaImageGenerator` | Ray video, Photon image, reframe, modify, upscale, audio, list, delete, concepts, credits | +| `replicate/` | `ReplicateClient`, `ReplicateWebhook`, `ReplicateJobs`, `ReplicateImageGenerator`, `ReplicateVideoGenerator` | all 36 operations: predictions (model, version, deployment), models, versions, deployments, files, trainings, hardware, collections, search, account, webhook secret; Standard Webhooks verification | +| `fal/` | `FalQueue`, `FalPlatform`, `FalImageGenerator`, `FalVideoGenerator` | queue API (submit, submit with path, status, result, cancel) and the model-related Platform API (search, pricing, estimate, usage, analytics, billing events, request listing, payload deletion, meta) | +| `ideogram/` | `IdeogramClient`, `IdeogramImageGenerator`, `IdeogramVideoGenerator` | all 70 v2 operations (image, SVG, video, ad tools, generations, account); free price quote through `dry_run=true` | +| `recraft/` | `RecraftClient`, `RecraftImageGenerator` | all 22 operations: generation (3 routes), image tools, styles, prompt enhancement, user | +| `leonardo/` | `LeonardoClient`, `LeonardoImageGenerator`, `LeonardoVideoGenerator` | v2 async and sync generations, models; v1 generation get and `me` (prose only) | +| `minimax/` | `MinimaxClient`, `MinimaxVideoGenerator` | Hailuo video v1 (create, query, file retrieve) and v2 (create, query, list, delete) | +| `kling/` | `KlingJwt`, `KlingClient`, `KlingVideoGenerator` | text2video, image2video, image generation, task queries; JWT auth | +| `higgsfield/` | `HiggsfieldClient`, `HiggsfieldJobs`, `HiggsfieldImageGenerator`, `HiggsfieldVideoGenerator` | the shared lifecycle (submit by model path, status, cancel, delete) for every model | + +Products without a public API (not implemented, shown in +`MediaRegistry::unsupported()` and on the demo page): + +- **Midjourney**: no public API. Its docs describe the web and Discord products + only; third-party relays are unofficial and break its terms. +- **Pika**: `pika.art/api` advertises a membership API that resells other + vendors' models, but no endpoint, schema or auth document exists + (2026-10-25). Pika models are reachable through fal. + +## 2. The seam (`common/job.h`) + +``` +ImageRequest / VideoRequest neutral request, validate() with caps +ImageGenerator::plan(req) exact request as JSON, no network, no credential +ImageGenerator::submit(req, confirm) JobTicket (confirm has no default) +MediaGenerator::poll(ticket) JobStatus {queued|running|succeeded|failed|cancelled} +MediaGenerator::cancel(ticket, confirm) false when the API has no cancel +JobRunner::wait(generator, ticket) polls under hard caps +Downloader::fetch(asset) bytes of a URL or an inline base64 output +``` + +Synchronous providers (Stability, Recraft, Ideogram by default) return an +`immediate` ticket that already carries its assets; `JobRunner` answers those +without a poll. A failed job is a `JobStatus`, a transport or auth failure is an +`ApiError`. + +## 3. Static caps (see `common/caps.h`) + +| Knob | Value | +|---|---| +| Prompt | 32000 chars | +| Body fields / one text field | 128 / 1 MiB | +| Uploaded file / files per call | 16 MiB / 16 | +| Query args / value | 16 / 1024 chars | +| Path argument / segments of a multi-segment id | 256 chars / 6 | +| Extra headers | 8, 512 chars each | +| Images per request / video seconds / dimension | 8 / 60 / 8192 | +| Poll interval floor / ceiling | 500 ms / 30000 ms | +| Poll attempts / total wait | 240 / 1800 s (demo: 30 / 60 s) | +| Assets per job / download / URL | 16 / 128 MiB / 4096 chars | +| Scan depth / nodes | 8 / 4096 | +| Page walk / items returned | 20 / 1000 | +| Webhook tolerance / body | 300 s / 1 MiB | +| Call table | 256 rows | + +`src/llm` caps still apply (32 MiB request, 64 MiB response). + +## 4. Contracts + +- Every call lives in the provider's table: key, method, path, spec operation + id, body kind (`none|json|multipart|either`), accepted body fields, accepted + query parameters, effect (`read|paid|write|delete`). +- `MediaService::prepare` is pure. It rejects unknown keys, missing or extra + path arguments, ids outside `[A-Za-z0-9._:=~-]` (and `.` / `..`), unknown query + parameters, unknown body fields, files on a JSON row, empty or oversized + files, bad header names. A rejected request never reaches the network. +- `MediaService::execute` refuses any non-read effect unless `confirmed` is + true. Typed methods that delete or cancel take `confirm` with no default. + The only two lowered effects are named and documented: Ideogram `quote` + (`dry_run=true`, defined by the spec as not billed) and Runway + `organization_usage` (a POST that only queries). +- No key: `configured()` is false and every call throws `NotConfiguredError` + (503) before any traffic. Keys travel in headers only. +- Lists report truncation (`"truncated": true`), never silently. +- Poll caps: the runner never goes below the interval floor, throws + `WAIT_EXPIRED` (504) naming the job when attempts or total wait run out, and + never cancels on its own. +- Errors stay `ApiError` with the vendor's status. Envelopes that hide an error + in a 200 are converted: MiniMax `base_resp.status_code`, Kling `code`. + +## 5. Credentials + +| Provider | Variables | Page | +|---|---|---| +| Stability AI | `STABILITY_API_KEY` | https://platform.stability.ai/account/keys (read: redirects to sign-in) | +| Black Forest Labs | `BFL_API_KEY` | https://dashboard.bfl.ai/ (read: redirects to sign-in) | +| Runway | `RUNWAY_API_KEY` (also `RUNWAYML_API_SECRET`, the SDK name) | https://dev.runwayml.com/ (read: 200) | +| Luma | `LUMA_API_KEY` | https://lumalabs.ai/api/keys (read: 200) | +| Replicate | `REPLICATE_API_TOKEN` | https://replicate.com/account/api-tokens (read: redirects to sign-in) | +| fal | `FAL_KEY` | https://fal.ai/dashboard/keys (read: redirects to sign-in) | +| Ideogram | `IDEOGRAM_API_KEY` | https://ideogram.ai/manage-api (unverified: Cloudflare challenge blocked the fetch) | +| Recraft | `RECRAFT_API_TOKEN` | https://www.recraft.ai/profile/api (read: redirects to sign-in with that callback) | +| Leonardo | `LEONARDO_API_KEY` | https://app.leonardo.ai/api-access (docs; fetch answered 429) | +| MiniMax | `MINIMAX_API_KEY` | https://platform.minimax.io/user-center/basic-information/interface-key (read: redirects to /console/access) | +| Kling | `KLING_ACCESS_KEY` and `KLING_SECRET_KEY` | https://app.klingai.com/global/dev/api-key (read: redirect) | +| Higgsfield | `HF_API_KEY_ID` and `HF_API_KEY_SECRET` | https://cloud.higgsfield.ai/ (docs; read: redirect) | + +Add these rows to `src/llm/SPEC.md` section 5 at integration. + +## 6. Sources + +Per provider, the machine-readable source `tools/check_spec.py` reads. + +| Provider | Source | +|---|---| +| Stability | https://docs.stability.ai/openapi.json (v2beta) and https://docs.stability.ai/openapi-v1.json; prose index https://docs.stability.ai/llms.txt | +| BFL | https://api.bfl.ai/openapi.json (version 0.0.1) | +| Replicate | https://api.replicate.com/openapi.json (server `https://api.replicate.com/v1`) | +| fal | queue: https://fal.ai/docs/api-reference/inference-api/openapi/queue.json; platform: https://api.fal.ai/v1/openapi.json; index https://docs.fal.ai/llms.txt | +| Ideogram | https://developer.ideogram.ai/v2/openapi.json | +| Recraft | https://www.recraft.ai/docs/api-reference/openapi.json | +| Higgsfield | https://docs.higgsfield.ai/docs/openapi.json (declared supplementary by the vendor) plus https://docs.higgsfield.ai/docs/llms.txt | +| Leonardo | OpenAPI embedded in https://docs.leonardo.ai/reference/creategeneration.md, `createsyncgeneration.md`, `getmodels.md`; quick start `docs/getting-started.md`, FAQ `docs/api-faq.md` | +| MiniMax | OpenAPI embedded in the `platform.minimax.io/docs/api-reference/video-generation-*.md` pages and `file-management-retrieve.md`; index `llms.txt` | +| Runway | npm `@runwayml/sdk` 4.24.0: `resources/**/*.js` (paths), `*.d.ts` (fields), `client.js` (base URL, `X-Runway-Version` default 2024-11-06) | +| Luma | npm `lumaai` 1.19.1: `resources/**/*.js`, `*.d.ts`, `index.js` (base URL) | +| Kling | none machine-readable; community sources (a third-party OpenAPI listing, a skill pack) plus live probes | + +Cross-check: Stability, BFL, Replicate, Ideogram and Recraft are also covered by +live unauthenticated probes of the same paths (section 8). Standard Webhooks +vector for the Replicate verifier comes from the Standard Webhooks project. + +## 7. Verified against spec + +`python3 -I src/vendors/media/tools/check_spec.py`, run 2026-10-25 (twelve +providers, all green; a deliberately corrupted BFL row failed it with both a row +error and a coverage error, then was restored): + +| Provider | Rows | Source operations | Result | +|---|---|---|---| +| stability | 34 | 37 (3 alpha operations excluded) | ok | +| bfl | 30 | 30 | ok | +| replicate | 36 | 36 | ok | +| fal | 16 | every queue and model-related platform operation (1 SSE stream route excluded; 78 platform operations outside model inference out of scope) | ok | +| ideogram | 70 | 70 | ok | +| recraft | 22 | 22 | ok | +| leonardo | 6 | 3 (3 v1 rows prose only, listed by the checker) | ok | +| runway | 66 | 66 SDK calls | ok | +| luma | 13 | 13 SDK calls | ok | +| minimax | 7 | 7 (5 reference pages listed as not implemented) | ok | +| higgsfield | 10 | 9 (1 generic row by design) | ok | +| kling | 6 | probe only | UNVERIFIED | + +The checker verifies: each row's method and path exist, operation ids match, +body kind matches the spec content types, every body field and query parameter +we send exists, each response schema field we read is reachable, and every +source operation is in the table or in the explicit exclusion list. + +## 8. Verified live + +Unauthenticated probes, 2026-10-25, recorded by `probe` runs and by +`tests/src/vendors/media/test_live.cpp` (`MOLO_LIVE_NET=1`, 13 providers, green): + +| Host and path | Invalid key answered | +|---|---| +| api.stability.ai `/v1/user/balance` | 401 `{"id","message","name"}` | +| api.stability.ai `/v2beta/results/00000000` | 400 `{"errors":["id: string must contain exactly 64 character(s)"],...}` (the result id is 64 characters) | +| api.bfl.ai `/v1/credits` | 422 `{"detail":"Invalid API key format"}` | +| api.replicate.com `/v1/account` | 401 `{"title","detail","status"}` | +| api.fal.ai `/v1/models/pricing` | 401 `{"error":{"type":"authorization_error",...}}` | +| queue.fal.run `/fal-ai/flux/requests//status` | 401 `{"detail":"Authentication is required"}` | +| api.fal.ai `/v1/models`, `/v1/meta` | 200 without any key (public) | +| api.ideogram.ai `/v2/account/api-keys` | 401 `{"message":"Access denied..."}`; `/v2/account/usage` answers 400 for the missing `start_time` before auth | +| external.api.recraft.ai `/v1/users/me` | 401 `request unauthorized` (plain text) | +| cloud.leonardo.ai `/api/rest/v2/models`, `/v1/me` | 401 `{"error","path","code":"access-denied"}` | +| api.dev.runwayml.com `/v1/organization` | 401 `{"error":"...did not start with key_","docUrl"}` | +| api.lumalabs.ai `/dream-machine/v1/credits` | 403 `{"detail":"Not authenticated"}` | +| api.minimax.io `/v1/query/video_generation` | 200 with `base_resp.status_code 1004` (converted to an ApiError) | +| api-singapore.klingai.com `/v1/videos/text2video/...` | 401 `{"code":1002,...}`; api.klingai.com answers `{"code":1000,"message":"Auth failed"}` | +| api.higgsfield.ai `/requests//status` | 401 `{"detail":"Invalid credentials"}` | + +No success body of any provider has been seen: no key exists here. Live tests +with keys (`MOLO_LIVE_NET=1` plus the provider variables) run one free read per +provider (`MediaRegistry::account`). MiniMax, Kling and Higgsfield have no free +account read, so a key is never proven for them without a paid or resource call. + +## 9. Unverified + +Every shape inferred rather than read from a spec or seen: + +- All success and result bodies, for every provider (no key). Status words and + result locations come from the specs or SDK types where they exist (BFL + `StatusResponse` enum and `result.sample`, fal `QueueStatus`, Ideogram + `GenerationResponse`, Higgsfield `RequestStatus`, Luma and Runway SDK + types, MiniMax pages) and are exercised by local mapper tests only. +- **Kling**: all six paths, the request fields, the `{code,data:{task_id, + task_status,task_result}}` envelope, the statuses and the JWT claims + (`iss`, `exp`, `nbf`, 1800 s) come from community sources, not from Kling. + Only the paths and the auth refusal were probed. The JWT builder matches an + independent HS256 implementation but Kling's acceptance of it was never seen. +- **Leonardo**: the polling route `GET /api/rest/v1/generations/{id}`, its + `generations_by_pk.{status,generated_images[].url}` shape, `GET /v1/me`, and + the v1 create body are described by quick start and FAQ prose only (the v1 + reference pages were removed). Whether a v2 `generationId` is pollable on the + v1 route was not confirmed. +- **Higgsfield**: only five model paths are in the OpenAPI document; any other + path goes through the generic row (`model.submit`) with an open body. +- **fal**: result URL locations differ per model; `OutputScanner` finds URLs + generically. A failed request is assumed to answer the result fetch with an + HTTP 4xx (the queue spec says `COMPLETED` also covers failure). +- **BFL**: `polling_url` may point at a regional host; polling uses + `get_result?id=` on the configured host instead. +- **Recraft** default raster output is lossless WebP (docs); inline results are + labelled `image/webp` on that basis. +- **Replicate** model names listed by `models()` and the other adapters' + `models()` lists are names seen in the docs and specs, not a live catalogue. +- Runway `X-Runway-Version` 2024-11-06 is the SDK default; newer versions may + change bodies. +- Console URLs in section 5 marked as such. + +## 10. Gaps + +- Foundation (`src/llm`, report to its owner): no unauthenticated requests (so + fal's public model search needs a key through our client); no GET stream (fal + status stream, excluded); per-call timeouts missing; `error_from_response` + leaves `type` empty for `{"message"}`, `{"detail"}` and `{"errors"}` + envelopes (Stability, BFL, Replicate, Higgsfield, Luma, Runway answer + `UNKNOWN`); the 32 MiB request ceiling blocks large video uploads (package + files are capped at 16 MiB each); `post_multipart` is bypassed through + `request_raw`. +- Another suite in the shared test binary unsets `MOLO_LIVE_NET` + (`tests/src/google/optimize/test_gaps.cpp`); the media live tests read it once + at startup to be immune. +- Not implemented: Stability gRPC and v2beta image-to-video (absent from the + published spec); MiniMax Video Agent, H3-Context-IR and regeneration tasks; + fal Platform assets / serverless / compute / keys / workflows; Replicate list + cursors (first page only, `truncated` is set); Ideogram and Replicate webhook + receivers (Replicate verification exists); WebSocket / realtime routes of + Runway beyond session creation. +- Recipes and avatar conversations of Runway are table rows only, with no typed + wrapper: use `MediaService::call_json`. +- Fetching a result needs a second step: `Downloader` is separate from + `poll()` so that callers choose when to spend bandwidth. diff --git a/src/vendors/media/bfl/bfl.cpp b/src/vendors/media/bfl/bfl.cpp new file mode 100644 index 0000000..103dedb --- /dev/null +++ b/src/vendors/media/bfl/bfl.cpp @@ -0,0 +1,146 @@ +#include "vendors/media/bfl/bfl.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs with_body(const Json& body) { + CallArgs a; + a.body = body; + return a; +} + +JobTicket ticket_from(const Json& out, const std::string& model, MediaKind kind) { + JobTicket t; + t.provider = "bfl"; + t.model = model; + t.kind = kind; + t.id = AdapterSupport::str(out, "id"); + if (t.id.empty()) throw llm::ApiError("bfl", 502, "BAD_RESPONSE", "no id in the submit reply"); + t.meta = Json::object(); + AdapterSupport::put(t.meta, "cost", AdapterSupport::str(out, "cost")); + return t; +} + +JobStatus status_of(BflClient& client, const JobTicket& ticket, MediaKind kind) { + CallArgs a; + a.query["id"] = ticket.id; + Json out = client.call_json("get_result", a); + JobStatus s; + s.raw_state = AdapterSupport::str(out, "status"); + s.state = BflImageGenerator::state_of(s.raw_state); + if (const Json* p = out.find("progress"); p && p->is_number()) s.progress = p->as_number(); + if (s.state == JobState::kSucceeded) { + if (const Json* r = out.find("result")) s.assets = OutputScanner::scan(*r, kind); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "Ready but no result URL in the reply"; + } + } else if (s.state == JobState::kFailed) { + s.error = s.raw_state; + if (const Json* d = out.find("details"); d && d->is_object()) s.error += ": " + d->dump(); + } + return s; +} + +} // namespace + +PreparedCall BflClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); } + +Json BflClient::submit(const std::string& key, const Json& body, bool confirm) { + return call_json(key, with_body(body), confirm); +} + +Json BflClient::get_result(const std::string& id) { + CallArgs a; + a.query["id"] = id; + return call_json("get_result", a); +} + +Json BflClient::credits() { return call_json("credits"); } +Json BflClient::my_finetunes() { return call_json("my_finetunes"); } + +Json BflClient::finetune_details(const std::string& finetune_id) { + CallArgs a; + a.query["finetune_id"] = finetune_id; + return call_json("finetune_details", a); +} + +Json BflClient::delete_finetune(const std::string& finetune_id, bool confirm) { + Json b = Json::object(); + b.set("finetune_id", Json(finetune_id)); + return call_json("delete_finetune", with_body(b), confirm); +} + +JobState BflImageGenerator::state_of(const std::string& status) { + if (status == "Ready") return JobState::kSucceeded; + if (status == "Pending" || status == "Reasoning") return JobState::kQueued; + if (status == "Generating") return JobState::kRunning; + if (status == "Task not found" || status == "Error" || status == "Request Moderated" || + status == "Content Moderated" || status == "No Card Generated") + return JobState::kFailed; + return JobState::kRunning; +} + +std::vector BflImageGenerator::models() const { + return {"flux-2-pro", "flux-2-max", "flux-2-flex", "flux-2-klein-9b", "flux-2-klein-4b", "flux-kontext-pro", + "flux-kontext-max", "flux-pro-1.1", "flux-pro-1.1-ultra", "flux-dev", "flux-3-image"}; +} + +Json BflImageGenerator::body(const ImageRequest& r, std::string& key) const { + r.validate(); + key = r.model.empty() ? "flux-2-pro" : r.model; + if (r.count != 1) throw llm::ApiError::invalid("bfl", "one image per call (count must be 1)"); + Json b = Json::object(); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put_int(b, "width", r.width); + AdapterSupport::put_int(b, "height", r.height); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put_seed(b, "seed", r.seed); + AdapterSupport::put(b, "input_image", r.source_url); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json BflImageGenerator::plan(const ImageRequest& request) const { + std::string key; + Json b = body(request, key); + return client_->plan(key, b).summary(); +} + +JobTicket BflImageGenerator::submit(const ImageRequest& request, bool confirm) { + std::string key; + Json b = body(request, key); + AdapterSupport::need_confirm(confirm, "bfl", "generate"); + return ticket_from(client_->submit(key, b, true), key, MediaKind::kImage); +} + +JobStatus BflImageGenerator::poll(const JobTicket& ticket) { return status_of(*client_, ticket, MediaKind::kImage); } + +Json BflVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + Json b = Json::object(); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put_int(b, "duration", r.duration_seconds); + AdapterSupport::put(b, "resolution", r.resolution); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json BflVideoGenerator::plan(const VideoRequest& request) const { + return client_->plan("flux-3-video", body(request)).summary(); +} + +JobTicket BflVideoGenerator::submit(const VideoRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "bfl", "generate"); + return ticket_from(client_->submit("flux-3-video", b, true), "flux-3-video", MediaKind::kVideo); +} + +JobStatus BflVideoGenerator::poll(const JobTicket& ticket) { return status_of(*client_, ticket, MediaKind::kVideo); } + +} // namespace molo::media diff --git a/src/vendors/media/bfl/bfl.h b/src/vendors/media/bfl/bfl.h new file mode 100644 index 0000000..03a7ec8 --- /dev/null +++ b/src/vendors/media/bfl/bfl.h @@ -0,0 +1,76 @@ +#pragma once + +// Black Forest Labs (FLUX): submit to a model endpoint, poll /v1/get_result. +// Auth is the x-key header. Models are endpoints (flux-2-pro, flux-kontext-pro, +// flux-pro-1.1-ultra, ...), tools are endpoints under /v1/flux-tools/. + +#include "vendors/media/bfl/calls.h" +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" + +namespace molo::media { + +class BflClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.bfl.ai"; + static constexpr const char* kKeyEnv = "BFL_API_KEY"; + + explicit BflClient(std::shared_ptr client) : MediaService(std::move(client), BflCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::header("bfl", kHost, kKeyEnv, "x-key"); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // key = a POST model / tool row ("flux-2-pro", "tools.erase-v1"). Returns {id, polling_url, cost, ...}. + Json submit(const std::string& key, const Json& body, bool confirm); + PreparedCall plan(const std::string& key, const Json& body) const; + Json get_result(const std::string& id); + Json credits(); + Json my_finetunes(); + Json finetune_details(const std::string& finetune_id); + // Destructive: no default for confirm. + Json delete_finetune(const std::string& finetune_id, bool confirm); +}; + +class BflImageGenerator : public ImageGenerator { + public: + explicit BflImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "bfl"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override; + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } // no cancel endpoint + int suggested_poll_ms() const override { return 1500; } + + // Maps the BFL status word onto JobState (shared with the video adapter). + static JobState state_of(const std::string& status); + + private: + Json body(const ImageRequest& request, std::string& key) const; + std::shared_ptr client_; +}; + +class BflVideoGenerator : public VideoGenerator { + public: + explicit BflVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "bfl"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { return {"flux-3-video"}; } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } + int suggested_poll_ms() const override { return 4000; } + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/bfl/calls.cpp b/src/vendors/media/bfl/calls.cpp new file mode 100644 index 0000000..55c66fa --- /dev/null +++ b/src/vendors/media/bfl/calls.cpp @@ -0,0 +1,57 @@ +#include "vendors/media/bfl/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"finetune_details", "GET", "/v1/finetune_details", "finetune_details_v1_finetune_details_get", "none", "", "finetune_id", "read"}, + {"my_finetunes", "GET", "/v1/my_finetunes", "my_finetunes_v1_my_finetunes_get", "none", "", "", "read"}, + {"delete_finetune", "POST", "/v1/delete_finetune", "delete_finetune_v1_delete_finetune_post", "json", "finetune_id", "", "delete"}, + {"flux-pro-1.1", "POST", "/v1/flux-pro-1.1", "flux_pro_1_1_v1_flux_pro_1_1_post", "json", "user,prompt,image_prompt,width,height,prompt_upsampling,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-dev", "POST", "/v1/flux-dev", "flux_dev_v1_flux_dev_post", "json", "user,prompt,image_prompt,width,height,steps,prompt_upsampling,seed,guidance,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-pro-1.1-ultra", "POST", "/v1/flux-pro-1.1-ultra", "generate_flux_1_1_ultra_v1_flux_pro_1_1_ultra_post", "json", "user,prompt,prompt_upsampling,seed,aspect_ratio,safety_tolerance,output_format,raw,image_prompt,image_prompt_strength,webhook_url,webhook_secret", "", "paid"}, + {"flux-pro-1.0-fill", "POST", "/v1/flux-pro-1.0-fill", "fill_v1_flux_pro_1_0_fill_post", "json", "user,image,mask,prompt,steps,prompt_upsampling,seed,guidance,output_format,safety_tolerance,webhook_url,webhook_secret", "", "paid"}, + {"flux-pro-1.0-expand", "POST", "/v1/flux-pro-1.0-expand", "expand_v1_flux_pro_1_0_expand_post", "json", "user,image,top,bottom,left,right,prompt,steps,prompt_upsampling,seed,guidance,output_format,safety_tolerance,webhook_url,webhook_secret", "", "paid"}, + {"flux-pro-1.1-ultra-finetuned", "POST", "/v1/flux-pro-1.1-ultra-finetuned", "generate_bigblue_finetuned_v1_flux_pro_1_1_ultra_finetuned_post", "json", "user,finetune_id,finetune_strength,prompt,seed,aspect_ratio,safety_tolerance,output_format,image_prompt,image_prompt_strength,prompt_upsampling,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-flex", "POST", "/v1/flux-2-flex", "generate_flux_2_flex_v1_flux_2_flex_post", "json", "user,prompt,prompt_upsampling,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,input_image_blob_path,seed,width,height,guidance,steps,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-klein-9b", "POST", "/v1/flux-2-klein-9b", "generate_flux_2_klein_9b_v1_flux_2_klein_9b_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-klein-9b-preview", "POST", "/v1/flux-2-klein-9b-preview", "generate_flux_2_klein_9b_kv_v1_flux_2_klein_9b_preview_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-klein-4b", "POST", "/v1/flux-2-klein-4b", "generate_flux_2_klein_4b_v1_flux_2_klein_4b_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-max", "POST", "/v1/flux-2-max", "generate_flux_2_max_v1_flux_2_max_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-pro-preview", "POST", "/v1/flux-2-pro-preview", "generate_flux_2_pro_preview_v1_flux_2_pro_preview_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-2-pro", "POST", "/v1/flux-2-pro", "generate_flux_2_pro_v1_flux_2_pro_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-3-video", "POST", "/v1/flux-3-video", "flux_3_video_v1_flux_3_video_post", "json", "user,prompt,aspect_ratio,duration,resolution,version,generate_audio,safety_tolerance,draft,mode,keyframes,start_video,draft_cache", "", "paid"}, + {"flux-3-image", "POST", "/v1/flux-3-image", "flux_3_image_v1_flux_3_image_post", "json", "user,prompt,images,aspect_ratio,resolution,width,height,safety_tolerance,grounding,version", "", "paid"}, + {"flux-3-image-klein", "POST", "/v1/flux-3-image-klein", "flux_3_image_klein_v1_flux_3_image_klein_post", "json", "user,prompt,images,aspect_ratio,resolution,width,height,seed,safety_tolerance,output_format,steps", "", "paid"}, + {"tools.video-upscale-v1", "POST", "/v1/flux-tools/video-upscale-v1", "generate_flux_tools_video_upscale_v1_v1_flux_tools_video_upscale_v1_post", "json", "user,input_video,prompt,creativity,upscale_factor,safety_tolerance,webhook_url,webhook_secret", "", "paid"}, + {"tools.video-edit-v1", "POST", "/v1/flux-tools/video-edit-v1", "generate_flux_tools_video_edit_v1_v1_flux_tools_video_edit_v1_post", "json", "user,video,prompt,safety_tolerance", "", "paid"}, + {"tools.outpainting-v1", "POST", "/v1/flux-tools/outpainting-v1", "generate_flux_tools_outpainting_v1_v1_flux_tools_outpainting_v1_post", "json", "user,input_image,width,height,auto_crop,safety_tolerance,output_format,prompt,reference_offset_x,reference_offset_y,mode,disable_pup", "", "paid"}, + {"tools.vto-v1", "POST", "/v1/flux-tools/vto-v1", "generate_flux_tools_vto_v1_v1_flux_tools_vto_v1_post", "json", "user,prompt,person,garment,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"tools.vto-v2", "POST", "/v1/flux-tools/vto-v2", "generate_flux_tools_vto_v2_v1_flux_tools_vto_v2_post", "json", "user,prompt,person,garment,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"tools.erase-v1", "POST", "/v1/flux-tools/erase-v1", "generate_flux_tools_erase_v1_v1_flux_tools_erase_v1_post", "json", "user,image,mask,dilate_pixels,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"tools.deblur-v1", "POST", "/v1/flux-tools/deblur-v1", "generate_flux_tools_deblur_v1_v1_flux_tools_deblur_v1_post", "json", "user,image,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"}, + {"flux-kontext-max", "POST", "/v1/flux-kontext-max", "generate_flux_kontext_max_v1_flux_kontext_max_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,aspect_ratio,output_format,webhook_url,webhook_secret,prompt_upsampling,safety_tolerance", "", "paid"}, + {"flux-kontext-pro", "POST", "/v1/flux-kontext-pro", "generate_flux_kontext_pro_v1_flux_kontext_pro_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,aspect_ratio,output_format,webhook_url,webhook_secret,prompt_upsampling,safety_tolerance", "", "paid"}, + {"get_result", "GET", "/v1/get_result", "get_result_v1_get_result_get", "none", "", "id", "read"}, + {"credits", "GET", "/v1/credits", "get_credits_v1_credits_get", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"AsyncResponse", "id,polling_url,cost,input_mp,output_mp"}, + {"ResultResponse", "id,status,result,progress,details,preview"}, + {"CreditsResponse", "credits"}, +}; +// FIELDS-END + +const CallTable& BflCalls::table() { + static const CallTable t("bfl", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t BflCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* BflCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/bfl/calls.h b/src/vendors/media/bfl/calls.h new file mode 100644 index 0000000..69e5f6a --- /dev/null +++ b/src/vendors/media/bfl/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct BflCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/adapter_support.h b/src/vendors/media/common/adapter_support.h new file mode 100644 index 0000000..f737508 --- /dev/null +++ b/src/vendors/media/common/adapter_support.h @@ -0,0 +1,50 @@ +#pragma once + +// Small helpers every generator adapter shares: the confirmation gate, body +// assembly that skips unset values, and extra-field merging. + +#include + +#include "clients/body/json.h" +#include "llm/error.h" +#include "vendors/media/common/job.h" + +namespace molo::media { + +struct AdapterSupport { + // Every submit / cancel passes through here first: no confirm, no call. + static void need_confirm(bool confirm, const std::string& provider, const std::string& what) { + if (!confirm) + throw llm::ApiError::invalid(provider, what + " needs explicit confirmation (it costs money or changes a resource)"); + } + + static void put(clients::body::Json& obj, const std::string& key, const std::string& value) { + if (!value.empty()) obj.set(key, clients::body::Json(value)); + } + static void put_int(clients::body::Json& obj, const std::string& key, int value) { + if (value > 0) obj.set(key, clients::body::Json(value)); + } + static void put_seed(clients::body::Json& obj, const std::string& key, long seed) { + if (seed >= 0) obj.set(key, clients::body::Json(static_cast(seed))); + } + + // Provider fields from `extra` win over derived ones (the caller asked). + static void merge_extra(clients::body::Json& obj, const clients::body::Json& extra) { + if (!extra.is_object()) return; + for (const auto& [k, v] : extra.as_object()) obj.set(k, v); + } + + static std::string str(const clients::body::Json& doc, const std::string& key) { + const clients::body::Json* v = doc.is_object() ? doc.find(key) : nullptr; + if (v && v->is_string()) return v->as_string(); + if (v && v->is_number()) return v->dump(); + return ""; + } + + static const clients::body::Json* obj_at(const clients::body::Json& doc, const std::string& key) { + const clients::body::Json* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_object() ? v : nullptr; + } +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/call_table.cpp b/src/vendors/media/common/call_table.cpp new file mode 100644 index 0000000..13369cd --- /dev/null +++ b/src/vendors/media/common/call_table.cpp @@ -0,0 +1,162 @@ +#include "vendors/media/common/call_table.h" + +#include +#include + +#include "llm/error.h" +#include "vendors/media/common/caps.h" + +namespace molo::media { + +namespace { + +bool one_of(const char* value, std::initializer_list options) { + for (const char* o : options) + if (std::strcmp(value, o) == 0) return true; + return false; +} + +} // namespace + +CallTable::CallTable(const char* vendor, const CallRow* rows, std::size_t count) + : vendor_(vendor), rows_(rows), count_(count) { + if (count > kMaxRows) throw std::logic_error(vendor_ + ": call table over " + std::to_string(kMaxRows) + " rows"); + for (std::size_t i = 0; i < count; ++i) { + const CallRow& r = rows[i]; + if (!index_.emplace(r.key, i).second) throw std::logic_error(vendor_ + ": duplicate call key " + r.key); + if (!one_of(r.http_method, {"GET", "POST", "PUT", "PATCH", "DELETE"})) + throw std::logic_error(vendor_ + ": bad method on " + r.key); + if (!one_of(r.body_kind, {"none", "json", "multipart", "either", "form"})) + throw std::logic_error(vendor_ + ": bad body kind on " + r.key); + if (!one_of(r.effect, {"read", "paid", "write", "delete"})) + throw std::logic_error(vendor_ + ": bad effect on " + r.key); + if (r.path[0] != '/') throw std::logic_error(vendor_ + ": path must start with / on " + r.key); + } +} + +const CallRow& CallTable::at(std::size_t index) const { + if (index >= count_) throw llm::ApiError::invalid(vendor_, "call index out of range"); + return rows_[index]; +} + +const CallRow* CallTable::find(const std::string& key) const { + auto it = index_.find(key); + return it == index_.end() ? nullptr : &rows_[it->second]; +} + +const CallRow& CallTable::get(const std::string& key) const { + const CallRow* row = find(key); + if (!row) throw llm::ApiError::invalid(vendor_, "unknown call '" + key + "'"); + return *row; +} + +Effect CallTable::effect_of(const CallRow& row) { + if (std::strcmp(row.effect, "paid") == 0) return Effect::kPaid; + if (std::strcmp(row.effect, "write") == 0) return Effect::kWrite; + if (std::strcmp(row.effect, "delete") == 0) return Effect::kDelete; + return Effect::kRead; +} + +const char* CallTable::effect_name(Effect effect) { + switch (effect) { + case Effect::kPaid: return "paid"; + case Effect::kWrite: return "write"; + case Effect::kDelete: return "delete"; + default: return "read"; + } +} + +clients::Method CallTable::method_of(const CallRow& row) { + const char* m = row.http_method; + if (std::strcmp(m, "POST") == 0) return clients::Method::kPost; + if (std::strcmp(m, "PUT") == 0) return clients::Method::kPut; + if (std::strcmp(m, "PATCH") == 0) return clients::Method::kPatch; + if (std::strcmp(m, "DELETE") == 0) return clients::Method::kDelete; + return clients::Method::kGet; +} + +std::vector CallTable::split(const char* csv) { + std::vector out; + std::string cur; + for (const char* p = csv; *p; ++p) { + if (*p == ',') { + if (!cur.empty()) out.push_back(cur); + cur.clear(); + } else { + cur.push_back(*p); + } + } + if (!cur.empty()) out.push_back(cur); + return out; +} + +std::vector CallTable::placeholders(const char* path) { + std::vector out; + std::string p(path); + std::size_t pos = 0; + while ((pos = p.find('{', pos)) != std::string::npos) { + std::size_t end = p.find('}', pos); + if (end == std::string::npos) break; + out.push_back(p.substr(pos + 1, end - pos - 1)); + pos = end + 1; + } + return out; +} + +bool CallTable::body_allows(const CallRow& row, const std::string& field) const { + for (const std::string& f : split(row.body_fields)) + if (f == "*" || f == field) return true; + return false; +} + +bool CallTable::query_allows(const CallRow& row, const std::string& name) const { + for (const std::string& f : split(row.query_params)) + if (f == name) return true; + return false; +} + +bool CallTable::valid_segment(const std::string& id) { + if (id.empty() || id.size() > MediaCaps::kMaxPathArgChars || id == "." || id == "..") return false; + for (char c : id) + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || + c == '_' || c == '.' || c == ':' || c == '=' || c == '~')) + return false; + return true; +} + +std::string CallTable::render_path(const CallRow& row, + const std::map& args) const { + std::vector names = placeholders(row.path); + for (const auto& [name, value] : args) { + bool known = false; + for (const std::string& n : names) known = known || n == name; + if (!known) throw llm::ApiError::invalid(vendor_, "call '" + std::string(row.key) + "' has no path argument '" + name + "'"); + (void)value; + } + std::string path = row.path; + for (const std::string& name : names) { + auto it = args.find(name); + if (it == args.end()) throw llm::ApiError::invalid(vendor_, "call '" + std::string(row.key) + "' needs path argument '" + name + "'"); + const std::string& value = it->second; + bool multi = name == "endpoint_id" || name == "model_path" || name == "path"; + if (multi) { + std::size_t segments = 0, start = 0; + while (true) { + std::size_t slash = value.find('/', start); + std::string seg = value.substr(start, slash == std::string::npos ? std::string::npos : slash - start); + if (!valid_segment(seg)) throw llm::ApiError::invalid(vendor_, "bad " + name); + if (++segments > MediaCaps::kMaxPathSegments) throw llm::ApiError::invalid(vendor_, name + " has too many segments"); + if (slash == std::string::npos) break; + start = slash + 1; + } + } else if (!valid_segment(value)) { + throw llm::ApiError::invalid(vendor_, "bad " + name); + } + std::string token = "{" + name + "}"; + std::size_t at = path.find(token); + path.replace(at, token.size(), value); + } + return path; +} + +} // namespace molo::media diff --git a/src/vendors/media/common/call_table.h b/src/vendors/media/common/call_table.h new file mode 100644 index 0000000..fbd9eb9 --- /dev/null +++ b/src/vendors/media/common/call_table.h @@ -0,0 +1,70 @@ +#pragma once + +// One table row per REST call. Every provider has exactly one table (calls.cpp +// in its directory); tools/check_spec.py checks each row against the +// provider's published spec. Code never builds a path from a literal. + +#include +#include +#include +#include + +#include "clients/request.h" + +namespace molo::media { + +// What a call does to the account. Anything but kRead needs explicit +// confirmation at execution time (MediaService::execute). +enum class Effect { kRead, kPaid, kWrite, kDelete }; + +struct CallRow { + const char* key; // short stable name, unique within the provider + const char* http_method; // GET POST PUT PATCH DELETE + const char* path; // from the host, "{placeholder}" segments + const char* op; // spec operation id when the spec has one + const char* body_kind; // none | json | multipart | either | form + const char* body_fields; // comma list of accepted body fields; "*" = open + const char* query_params; // comma list of accepted query parameters + const char* effect; // read | paid | write | delete +}; + +class CallTable { + public: + static constexpr std::size_t kMaxRows = 256; + + // Throws std::logic_error on a duplicate key, a bad method / kind / effect, + // or more than kMaxRows rows (a table bug, caught by the unit tests). + CallTable(const char* vendor, const CallRow* rows, std::size_t count); + + const std::string& vendor() const { return vendor_; } + std::size_t size() const { return count_; } + const CallRow& at(std::size_t index) const; + const CallRow* find(const std::string& key) const; + // Throws llm::ApiError::invalid for an unknown key. + const CallRow& get(const std::string& key) const; + + static Effect effect_of(const CallRow& row); + static const char* effect_name(Effect effect); + static clients::Method method_of(const CallRow& row); + static std::vector split(const char* csv); + static std::vector placeholders(const char* path); + + bool body_allows(const CallRow& row, const std::string& field) const; + bool query_allows(const CallRow& row, const std::string& name) const; + + // One URL path segment: 1..256 of [A-Za-z0-9._:=~-], not "." or "..". + static bool valid_segment(const std::string& id); + // Replaces every {placeholder}; each must be supplied, none may be extra. + // Placeholders named endpoint_id, model_path or path accept 1..6 "/"-joined + // segments, every other placeholder exactly one. Throws ApiError::invalid. + std::string render_path(const CallRow& row, + const std::map& args) const; + + private: + std::string vendor_; + const CallRow* rows_; + std::size_t count_; + std::map index_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/caps.h b/src/vendors/media/common/caps.h new file mode 100644 index 0000000..c6cc960 --- /dev/null +++ b/src/vendors/media/common/caps.h @@ -0,0 +1,47 @@ +#pragma once + +// Static ceilings for package media. Never autotuned. Request and response +// byte ceilings that come from src/llm (32 MiB / 64 MiB) are not repeated here; +// ours are the ones the media workflows add: files, polls, downloads, pages. + +#include + +namespace molo::media { + +struct MediaCaps { + // Request shaping. + static constexpr std::size_t kMaxPromptChars = 32000; + static constexpr std::size_t kMaxBodyFields = 128; + static constexpr std::size_t kMaxFieldChars = 1u << 20; // one text field, 1 MiB + static constexpr std::size_t kMaxFileBytes = 16u * 1024u * 1024u; // one uploaded file + static constexpr std::size_t kMaxFiles = 16; + static constexpr std::size_t kMaxQueryArgs = 16; + static constexpr std::size_t kMaxQueryValueChars = 1024; + static constexpr std::size_t kMaxPathArgChars = 256; + static constexpr std::size_t kMaxPathSegments = 6; // multi-segment ids (fal endpoints) + static constexpr std::size_t kMaxHeaders = 8; + static constexpr std::size_t kMaxHeaderValueChars = 512; + // Generation limits we enforce before spending money. + static constexpr int kMaxImagesPerRequest = 8; + static constexpr int kMaxVideoSeconds = 60; + static constexpr int kMaxDimension = 8192; + // Job polling: floors and ceilings (the machine never goes below / above). + static constexpr int kMinPollIntervalMs = 500; + static constexpr int kMaxPollIntervalMs = 30000; + static constexpr int kMaxPollAttempts = 240; + static constexpr int kMaxWaitSeconds = 1800; + // Outputs. + static constexpr std::size_t kMaxAssetsPerJob = 16; + static constexpr std::size_t kMaxDownloadBytes = 128u * 1024u * 1024u; + static constexpr std::size_t kMaxUrlChars = 4096; + static constexpr std::size_t kMaxScanDepth = 8; + static constexpr std::size_t kMaxScanNodes = 4096; + // Listings. + static constexpr std::size_t kMaxPages = 20; + static constexpr std::size_t kMaxItemsReturned = 1000; + // Webhooks. + static constexpr long kWebhookToleranceSeconds = 300; + static constexpr std::size_t kMaxWebhookBodyBytes = 1u * 1024u * 1024u; +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/client_factory.h b/src/vendors/media/common/client_factory.h new file mode 100644 index 0000000..d7c38e8 --- /dev/null +++ b/src/vendors/media/common/client_factory.h @@ -0,0 +1,58 @@ +#pragma once + +// Builds the llm::ApiClient every provider client wraps. One place, so the +// credential lookup (environment, then .env.json.local, then .env.json via +// llm::Config) and the transport choice are identical for all providers. + +#include +#include +#include + +#include "clients/transports/http_transport.h" +#include "llm/api_client.h" +#include "llm/config.h" + +namespace molo::media { + +struct ClientFactory { + // First non-empty value among `names` (llm::Config::get), else "". + static std::string env(std::initializer_list names) { + for (const char* n : names) { + std::string v = llm::Config::get(n); + if (!v.empty()) return v; + } + return ""; + } + + // Real HTTPS. An empty key leaves the client "not configured". + static std::shared_ptr real(llm::ApiClientConfig config, const std::string& key) { + return std::make_shared(std::make_unique(), + std::move(config), key); + } + + // Any transport (tests record the requests the real client builds). + static std::shared_ptr with(std::unique_ptr transport, + llm::ApiClientConfig config, const std::string& key) { + return std::make_shared(std::move(transport), std::move(config), key); + } + + static llm::ApiClientConfig bearer(const std::string& vendor, const std::string& base_url, + const std::string& key_env) { + llm::ApiClientConfig c; + c.vendor = vendor; + c.base_url = base_url; + c.key_env = key_env; + c.auth = llm::AuthStyle::kBearer; + return c; + } + + static llm::ApiClientConfig header(const std::string& vendor, const std::string& base_url, + const std::string& key_env, const std::string& header_name) { + llm::ApiClientConfig c = bearer(vendor, base_url, key_env); + c.auth = llm::AuthStyle::kHeader; + c.auth_header = header_name; + return c; + } +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/downloader.cpp b/src/vendors/media/common/downloader.cpp new file mode 100644 index 0000000..eecaf65 --- /dev/null +++ b/src/vendors/media/common/downloader.cpp @@ -0,0 +1,74 @@ +#include "vendors/media/common/downloader.h" + +#include "clients/transports/http_transport.h" +#include "llm/error.h" +#include "security/crypto/Base64.h" +#include "vendors/media/common/caps.h" + +namespace molo::media { + +namespace { + +clients::ClientConfig plain_config() { + clients::ClientConfig c; + c.persist_cookies = false; + return c; +} + +std::unique_ptr real_or(std::unique_ptr t) { + if (t) return t; + return std::make_unique(); +} + +} // namespace + +Downloader::Downloader(std::unique_ptr transport) + : client_(real_or(std::move(transport)), plain_config()) {} + +void Downloader::validate_url(const std::string& url) { + auto bad = [](const std::string& why) { throw llm::ApiError::invalid("media", why); }; + if (url.empty() || url.size() > MediaCaps::kMaxUrlChars) bad("download url length"); + if (url.rfind("https://", 0) != 0) bad("download url must be https"); + for (unsigned char c : url) + if (c <= 0x20 || c == 0x7f) bad("download url has whitespace or control characters"); + std::size_t host_end = url.find_first_of("/?#", 8); + std::string authority = url.substr(8, host_end == std::string::npos ? std::string::npos : host_end - 8); + if (authority.empty()) bad("download url has no host"); + if (authority.find('@') != std::string::npos) bad("download url must not carry credentials"); +} + +std::string Downloader::fetch_url(const std::string& url, std::string* content_type) { + validate_url(url); + try { + clients::ClientRequest req(clients::Method::kGet, clients::Url::parse(url)); + clients::ClientResponse res = client_.send_blocking(std::move(req)); + if (!res.is_success()) + throw llm::ApiError("media", res.status(), "DOWNLOAD_FAILED", "download answered HTTP " + std::to_string(res.status())); + if (res.body().size() > MediaCaps::kMaxDownloadBytes) + throw llm::ApiError("media", 502, "BAD_RESPONSE", "download over the byte cap"); + if (content_type) *content_type = res.content_type(); + return res.body(); + } catch (const llm::ApiError&) { + throw; + } catch (const std::exception& e) { + throw llm::ApiError("media", 502, "UNAVAILABLE", e.what()); + } +} + +FetchedAsset Downloader::fetch(const MediaAsset& asset) { + FetchedAsset out; + out.asset = asset; + if (!asset.data_base64.empty()) { + if (!security::crypto::Base64::decode(asset.data_base64, out.bytes)) + throw llm::ApiError::invalid("media", "inline asset is not valid base64"); + if (out.bytes.size() > MediaCaps::kMaxDownloadBytes) + throw llm::ApiError("media", 502, "BAD_RESPONSE", "inline asset over the byte cap"); + out.content_type = asset.mime; + return out; + } + if (asset.url.empty()) throw llm::ApiError::invalid("media", "asset has neither url nor data"); + out.bytes = fetch_url(asset.url, &out.content_type); + return out; +} + +} // namespace molo::media diff --git a/src/vendors/media/common/downloader.h b/src/vendors/media/common/downloader.h new file mode 100644 index 0000000..62d4662 --- /dev/null +++ b/src/vendors/media/common/downloader.h @@ -0,0 +1,38 @@ +#pragma once + +// Downloader: fetch an output (https URL or inline base64) as bytes under the +// download cap. Sends no credential: result URLs are public or pre-signed. + +#include +#include + +#include "clients/client.h" +#include "vendors/media/common/job.h" + +namespace molo::media { + +struct FetchedAsset { + MediaAsset asset; + std::string bytes; + std::string content_type; +}; + +class Downloader { + public: + // Null transport = real HTTPS. + explicit Downloader(std::unique_ptr transport = nullptr); + + // https only, 1..4096 chars, no userinfo, no whitespace or control chars. + // Throws ApiError::invalid("media", ...). + static void validate_url(const std::string& url); + + // Decodes inline data or downloads the URL. Throws ApiError on a bad status + // (the body is never returned for a failure) or when the cap is exceeded. + FetchedAsset fetch(const MediaAsset& asset); + std::string fetch_url(const std::string& url, std::string* content_type = nullptr); + + private: + clients::Client client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/job.cpp b/src/vendors/media/common/job.cpp new file mode 100644 index 0000000..765a134 --- /dev/null +++ b/src/vendors/media/common/job.cpp @@ -0,0 +1,145 @@ +#include "vendors/media/common/job.h" + +#include "llm/error.h" +#include "vendors/media/common/caps.h" + +namespace molo::media { + +using clients::body::Json; + +const char* media_kind_name(MediaKind kind) { + switch (kind) { + case MediaKind::kImage: return "image"; + case MediaKind::kVideo: return "video"; + case MediaKind::kAudio: return "audio"; + case MediaKind::kModel3d: return "model3d"; + default: return "other"; + } +} + +const char* job_state_name(JobState state) { + switch (state) { + case JobState::kQueued: return "queued"; + case JobState::kRunning: return "running"; + case JobState::kSucceeded: return "succeeded"; + case JobState::kFailed: return "failed"; + default: return "cancelled"; + } +} + +namespace { + +MediaKind kind_from(const std::string& s) { + if (s == "image") return MediaKind::kImage; + if (s == "video") return MediaKind::kVideo; + if (s == "audio") return MediaKind::kAudio; + if (s == "model3d") return MediaKind::kModel3d; + return MediaKind::kOther; +} + +std::string str(const Json& doc, const char* key) { + const Json* v = doc.is_object() ? doc.find(key) : nullptr; + return v && v->is_string() ? v->as_string() : ""; +} + +Json assets_json(const std::vector& assets) { + Json a = Json::array(); + for (const MediaAsset& x : assets) a.push(x.to_json()); + return a; +} + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("media", why); } + +void check_text(const std::string& what, const std::string& v, std::size_t cap) { + if (v.size() > cap) bad(what + " is over " + std::to_string(cap) + " characters"); +} + +} // namespace + +Json MediaAsset::to_json() const { + Json o = Json::object(); + o.set("kind", Json(media_kind_name(kind))); + if (!url.empty()) o.set("url", Json(url)); + if (!data_base64.empty()) o.set("data_base64_bytes", Json(static_cast(data_base64.size()))); + if (!mime.empty()) o.set("mime", Json(mime)); + if (!label.empty()) o.set("label", Json(label)); + return o; +} + +Json JobTicket::to_json() const { + Json o = Json::object(); + o.set("provider", Json(provider)); + o.set("id", Json(id)); + o.set("model", Json(model)); + o.set("kind", Json(media_kind_name(kind))); + o.set("immediate", Json(immediate)); + o.set("assets", assets_json(assets)); + if (!meta.is_null()) o.set("meta", meta); + return o; +} + +JobTicket JobTicket::from_json(const Json& doc) { + if (!doc.is_object()) bad("ticket must be a JSON object"); + JobTicket t; + t.provider = str(doc, "provider"); + t.id = str(doc, "id"); + t.model = str(doc, "model"); + t.kind = kind_from(str(doc, "kind")); + if (t.provider.empty() || t.id.empty()) bad("ticket needs provider and id"); + if (const Json* m = doc.find("meta"); m && m->is_object()) t.meta = *m; + if (const Json* im = doc.find("immediate"); im && im->is_bool()) t.immediate = im->as_bool(); + // Only URL assets travel back: inline (base64) outputs are not echoed by + // to_json(), so an immediate ticket restored from JSON carries its URLs only. + if (const Json* a = doc.find("assets"); a && a->is_array()) + for (const Json& x : a->as_array()) { + std::string url = str(x, "url"); + if (url.empty() || t.assets.size() >= MediaCaps::kMaxAssetsPerJob) continue; + MediaAsset asset; + asset.url = url; + asset.kind = kind_from(str(x, "kind")); + asset.mime = str(x, "mime"); + asset.label = str(x, "label"); + t.assets.push_back(asset); + } + return t; +} + +Json JobStatus::to_json() const { + Json o = Json::object(); + o.set("state", Json(job_state_name(state))); + o.set("raw_state", Json(raw_state)); + if (progress >= 0) o.set("progress", Json(progress)); + if (!error.empty()) o.set("error", Json(error)); + o.set("assets", assets_json(assets)); + return o; +} + +void ImageRequest::validate() const { + if (prompt.empty()) bad("prompt is required"); + check_text("prompt", prompt, MediaCaps::kMaxPromptChars); + check_text("negative_prompt", negative_prompt, MediaCaps::kMaxPromptChars); + check_text("model", model, 256); + check_text("source_url", source_url, MediaCaps::kMaxUrlChars); + if (count < 1 || count > MediaCaps::kMaxImagesPerRequest) + bad("count must be 1.." + std::to_string(MediaCaps::kMaxImagesPerRequest)); + if (width < 0 || width > MediaCaps::kMaxDimension || height < 0 || height > MediaCaps::kMaxDimension) + bad("width and height must be 0.." + std::to_string(MediaCaps::kMaxDimension)); + if ((width == 0) != (height == 0)) bad("width and height go together"); + if (seed < -1 || seed > 4294967295L) bad("seed must be -1 or 0..4294967295"); + if (!extra.is_null() && !extra.is_object()) bad("extra must be a JSON object"); +} + +void VideoRequest::validate() const { + if (prompt.empty() && image_url.empty()) bad("prompt or image_url is required"); + check_text("prompt", prompt, MediaCaps::kMaxPromptChars); + check_text("negative_prompt", negative_prompt, MediaCaps::kMaxPromptChars); + check_text("model", model, 256); + check_text("image_url", image_url, MediaCaps::kMaxUrlChars); + check_text("end_image_url", end_image_url, MediaCaps::kMaxUrlChars); + if (duration_seconds < 0 || duration_seconds > MediaCaps::kMaxVideoSeconds) + bad("duration_seconds must be 0.." + std::to_string(MediaCaps::kMaxVideoSeconds)); + if (seed < -1 || seed > 4294967295L) bad("seed must be -1 or 0..4294967295"); + if (!extra.is_null() && !extra.is_object()) bad("extra must be a JSON object"); +} + +} // namespace molo::media diff --git a/src/vendors/media/common/job.h b/src/vendors/media/common/job.h new file mode 100644 index 0000000..a74ed10 --- /dev/null +++ b/src/vendors/media/common/job.h @@ -0,0 +1,124 @@ +#pragma once + +// The neutral generation seam of package media: submit a request, poll the +// job, fetch the outputs. Every provider adapter implements ImageGenerator +// and / or VideoGenerator; callers (the demo, tests, other packages) code +// against these types only. Nothing here knows a provider's wire format. +// +// Money rule: submit() and cancel() take an explicit confirm flag with no +// default. Without it they throw before any network traffic. + +#include +#include + +#include "clients/body/json.h" + +namespace molo::media { + +enum class MediaKind { kImage, kVideo, kAudio, kModel3d, kOther }; +enum class JobState { kQueued, kRunning, kSucceeded, kFailed, kCancelled }; + +const char* media_kind_name(MediaKind kind); +const char* job_state_name(JobState state); + +// One output. Exactly one of url / data_base64 is set. +struct MediaAsset { + MediaKind kind = MediaKind::kOther; + std::string url; + std::string data_base64; + std::string mime; + std::string label; // provider note: seed, finish reason, model + + clients::body::Json to_json() const; +}; + +// Handle of a submitted job. `immediate` jobs (synchronous providers) carry +// their assets already; poll() answers from the ticket without a network call. +struct JobTicket { + std::string provider; + std::string id; + std::string model; + MediaKind kind = MediaKind::kImage; + bool immediate = false; + std::vector assets; + clients::body::Json meta; // provider extras the poll needs (endpoint, urls, cost) + + clients::body::Json to_json() const; + // Rebuilds a ticket from to_json() output (the demo round-trips tickets). + static JobTicket from_json(const clients::body::Json& doc); +}; + +struct JobStatus { + JobState state = JobState::kQueued; + std::string raw_state; // the provider's own word + double progress = -1; // 0..1, -1 = unknown + std::string error; + std::vector assets; + clients::body::Json meta; + + bool terminal() const { return state == JobState::kSucceeded || state == JobState::kFailed || state == JobState::kCancelled; } + clients::body::Json to_json() const; +}; + +struct ImageRequest { + std::string prompt; + std::string negative_prompt; + std::string model; // provider model / endpoint id; empty = adapter default + int width = 0; // 0 = provider default + int height = 0; + std::string aspect_ratio; // "16:9"; empty = default + int count = 1; + long seed = -1; // -1 = random + std::string source_url; // reference / source image for edit styles + clients::body::Json extra; // provider fields merged last, checked against the call table + + // Throws llm::ApiError::invalid(vendor "media") on a bad value or cap breach. + void validate() const; +}; + +struct VideoRequest { + std::string prompt; + std::string negative_prompt; + std::string model; + int duration_seconds = 0; // 0 = provider default + std::string aspect_ratio; + std::string resolution; // "720p", "1080p"; empty = default + long seed = -1; + std::string image_url; // start frame for image-to-video + std::string end_image_url; // end frame + clients::body::Json extra; + + void validate() const; +}; + +class MediaGenerator { + public: + virtual ~MediaGenerator() = default; + virtual std::string provider() const = 0; + virtual bool configured() const = 0; + // Model / endpoint ids the adapter knows by name (not a live catalogue). + virtual std::vector models() const = 0; + // One status check. A failed job is a JobStatus with state kFailed, a + // transport or auth failure is an ApiError. + virtual JobStatus poll(const JobTicket& ticket) = 0; + // Cancels where the provider supports it; returns false (without any call) + // where it does not. Explicit confirm, no default. + virtual bool cancel(const JobTicket& ticket, bool confirm) = 0; + // Suggested first poll interval; JobRunner clamps it to the caps. + virtual int suggested_poll_ms() const { return 2000; } +}; + +class ImageGenerator : public MediaGenerator { + public: + // Exact request the submit would send, as JSON; no network, no credential. + virtual clients::body::Json plan(const ImageRequest& request) const = 0; + virtual JobTicket submit(const ImageRequest& request, bool confirm) = 0; +}; + +class VideoGenerator : public MediaGenerator { + public: + virtual clients::body::Json plan(const VideoRequest& request) const = 0; + virtual JobTicket submit(const VideoRequest& request, bool confirm) = 0; +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/job_runner.cpp b/src/vendors/media/common/job_runner.cpp new file mode 100644 index 0000000..9e8ed54 --- /dev/null +++ b/src/vendors/media/common/job_runner.cpp @@ -0,0 +1,65 @@ +#include "vendors/media/common/job_runner.h" + +#include +#include +#include + +#include "llm/error.h" +#include "vendors/media/common/caps.h" + +namespace molo::media { + +void WaitPolicy::validate() const { + auto bad = [](const std::string& why) { throw llm::ApiError::invalid("media", why); }; + if (first_interval_ms < 0 || first_interval_ms > MediaCaps::kMaxPollIntervalMs) bad("first_interval_ms out of range"); + if (backoff < 1.0 || backoff > 3.0) bad("backoff must be 1.0..3.0"); + if (max_interval_ms < MediaCaps::kMinPollIntervalMs || max_interval_ms > MediaCaps::kMaxPollIntervalMs) + bad("max_interval_ms must be " + std::to_string(MediaCaps::kMinPollIntervalMs) + ".." + + std::to_string(MediaCaps::kMaxPollIntervalMs)); + if (max_attempts < 1 || max_attempts > MediaCaps::kMaxPollAttempts) + bad("max_attempts must be 1.." + std::to_string(MediaCaps::kMaxPollAttempts)); + if (max_wait_seconds < 1 || max_wait_seconds > MediaCaps::kMaxWaitSeconds) + bad("max_wait_seconds must be 1.." + std::to_string(MediaCaps::kMaxWaitSeconds)); +} + +JobRunner::JobRunner(WaitPolicy policy, Sleeper sleeper, Clock clock) + : policy_(policy), sleeper_(std::move(sleeper)), clock_(std::move(clock)) { + policy_.validate(); + if (!sleeper_) + sleeper_ = [](int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); }; + if (!clock_) + clock_ = [] { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + }; +} + +JobStatus JobRunner::wait(MediaGenerator& generator, const JobTicket& ticket) const { + if (ticket.immediate) { + JobStatus s; + s.state = JobState::kSucceeded; + s.raw_state = "immediate"; + s.progress = 1; + s.assets = ticket.assets; + return s; + } + const long start = clock_(); + const long deadline_ms = static_cast(policy_.max_wait_seconds) * 1000L; + int interval = policy_.first_interval_ms > 0 ? policy_.first_interval_ms : generator.suggested_poll_ms(); + interval = std::clamp(interval, MediaCaps::kMinPollIntervalMs, policy_.max_interval_ms); + for (int attempt = 1; attempt <= policy_.max_attempts; ++attempt) { + JobStatus s = generator.poll(ticket); + if (s.terminal()) return s; + if (clock_() - start + interval > deadline_ms) break; + sleeper_(interval); + interval = std::clamp(static_cast(interval * policy_.backoff), MediaCaps::kMinPollIntervalMs, + policy_.max_interval_ms); + } + throw llm::ApiError("media", 504, "WAIT_EXPIRED", + generator.provider() + " job " + ticket.id + " is not finished after the poll caps (" + + std::to_string(policy_.max_attempts) + " polls, " + std::to_string(policy_.max_wait_seconds) + + " s); it was NOT cancelled, check it with poll"); +} + +} // namespace molo::media diff --git a/src/vendors/media/common/job_runner.h b/src/vendors/media/common/job_runner.h new file mode 100644 index 0000000..af6f67b --- /dev/null +++ b/src/vendors/media/common/job_runner.h @@ -0,0 +1,45 @@ +#pragma once + +// JobRunner: poll a job to a terminal state under hard caps. The caps are a +// floor on the interval (never hammer a provider) and ceilings on attempts +// and total wait (never hang). When a cap is hit the runner throws; it never +// cancels the job on its own (a human decides, the money may already be spent). + +#include + +#include "vendors/media/common/job.h" + +namespace molo::media { + +struct WaitPolicy { + int first_interval_ms = 0; // 0 = the generator's suggestion + double backoff = 1.5; // interval multiplier per attempt, 1.0 .. 3.0 + int max_interval_ms = 15000; // <= MediaCaps::kMaxPollIntervalMs + int max_attempts = 120; // <= MediaCaps::kMaxPollAttempts + int max_wait_seconds = 900; // <= MediaCaps::kMaxWaitSeconds + + // Throws ApiError::invalid when a value is outside the caps. + void validate() const; +}; + +class JobRunner { + public: + using Sleeper = std::function; + using Clock = std::function; // monotonic milliseconds + + // Null sleeper / clock use the real ones. Tests inject fakes. + explicit JobRunner(WaitPolicy policy = {}, Sleeper sleeper = nullptr, Clock clock = nullptr); + + const WaitPolicy& policy() const { return policy_; } + + // Immediate tickets answer without a poll. Otherwise polls until terminal. + // Throws ApiError (504, WAIT_EXPIRED) when attempts or wait are exhausted. + JobStatus wait(MediaGenerator& generator, const JobTicket& ticket) const; + + private: + WaitPolicy policy_; + Sleeper sleeper_; + Clock clock_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/media_service.cpp b/src/vendors/media/common/media_service.cpp new file mode 100644 index 0000000..5133399 --- /dev/null +++ b/src/vendors/media/common/media_service.cpp @@ -0,0 +1,171 @@ +#include "vendors/media/common/media_service.h" + +#include + +#include "clients/body/form.h" +#include "clients/url.h" +#include "llm/error.h" +#include "vendors/media/common/caps.h" + +namespace molo::media { + +namespace { + +bool valid_header_name(const std::string& n) { + if (n.empty() || n.size() > 64) return false; + for (char c : n) + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-')) return false; + return true; +} + +std::string field_text(const Json& v) { + if (v.is_string()) return v.as_string(); + if (v.is_bool()) return v.as_bool() ? "true" : "false"; + return v.dump(); // numbers, arrays, objects travel as JSON text +} + +} // namespace + +Json PreparedCall::summary() const { + Json o = Json::object(); + o.set("vendor", Json(vendor)); + o.set("key", Json(key)); + o.set("method", Json(method)); + o.set("target", Json(target)); + o.set("effect", Json(CallTable::effect_name(effect))); + if (!content_type.empty()) o.set("content_type", Json(content_type.substr(0, content_type.find(';')))); + if (!preview.is_null()) o.set("body", preview); + return o; +} + +MediaService::MediaService(std::shared_ptr client, const CallTable& table) + : llm::Service(std::move(client)), table_(table) {} + +PreparedCall MediaService::prepare(const std::string& key, const CallArgs& args) const { + const std::string v = table_.vendor(); + const CallRow& row = table_.get(key); + PreparedCall out; + out.vendor = v; + out.key = key; + out.method = row.http_method; + out.effect = CallTable::effect_of(row); + + std::string target = table_.render_path(row, args.path); + + if (args.query.size() > MediaCaps::kMaxQueryArgs) throw llm::ApiError::invalid(v, "too many query parameters"); + std::string qs; + for (const auto& [name, value] : args.query) { + if (!table_.query_allows(row, name)) + throw llm::ApiError::invalid(v, "call '" + key + "' does not accept query parameter '" + name + "'"); + if (value.size() > MediaCaps::kMaxQueryValueChars) throw llm::ApiError::invalid(v, "query value too long: " + name); + qs += qs.empty() ? '?' : '&'; + qs += clients::Url::encode_component(name) + "=" + clients::Url::encode_component(value); + } + out.target = target + qs; + + if (args.headers.size() > MediaCaps::kMaxHeaders) throw llm::ApiError::invalid(v, "too many headers"); + for (const auto& [k, val] : args.headers) { + if (!valid_header_name(k) || val.size() > MediaCaps::kMaxHeaderValueChars || + val.find_first_of("\r\n") != std::string::npos) + throw llm::ApiError::invalid(v, "bad header '" + k + "'"); + out.headers[k] = val; + } + + const std::string kind = row.body_kind; + const bool has_body = !args.body.is_null() && !(args.body.is_object() && args.body.as_object().empty()); + if (kind == "none") { + if (has_body || !args.files.empty()) throw llm::ApiError::invalid(v, "call '" + key + "' takes no body"); + return out; + } + if (kind == "form") throw llm::ApiError::invalid(v, "form bodies are not supported by this package"); + + if (!args.body.is_null() && !args.body.is_object()) throw llm::ApiError::invalid(v, "body must be a JSON object"); + Json fields = args.body.is_null() ? Json::object() : args.body; + if (fields.as_object().size() > MediaCaps::kMaxBodyFields) throw llm::ApiError::invalid(v, "too many body fields"); + for (const auto& [name, value] : fields.as_object()) { + if (!table_.body_allows(row, name)) + throw llm::ApiError::invalid(v, "call '" + key + "' does not accept body field '" + name + "'"); + if (value.is_string() && value.as_string().size() > MediaCaps::kMaxFieldChars) + throw llm::ApiError::invalid(v, "field '" + name + "' is over the size cap"); + } + + const bool multipart = kind == "multipart" || (kind == "either" && !args.files.empty()); + if (!args.files.empty() && !multipart) throw llm::ApiError::invalid(v, "call '" + key + "' does not take files"); + if (args.files.size() > MediaCaps::kMaxFiles) throw llm::ApiError::invalid(v, "too many files"); + + if (!multipart) { + out.body = fields.dump(); + out.content_type = "application/json"; + out.preview = fields; + return out; + } + + clients::body::MultipartBody form; + Json preview = Json::object(); + for (const auto& [name, value] : fields.as_object()) { + form.add_text(name, field_text(value)); + preview.set(name, value); + } + Json files = Json::array(); + for (const FilePart& f : args.files) { + if (!table_.body_allows(row, f.field)) + throw llm::ApiError::invalid(v, "call '" + key + "' does not accept file field '" + f.field + "'"); + if (f.data.empty()) throw llm::ApiError::invalid(v, "file '" + f.field + "' is empty"); + if (f.data.size() > MediaCaps::kMaxFileBytes) throw llm::ApiError::invalid(v, "file '" + f.field + "' is over the size cap"); + if (f.filename.find_first_of("\"\r\n") != std::string::npos || f.content_type.find_first_of("\r\n") != std::string::npos) + throw llm::ApiError::invalid(v, "bad file name or content type"); + form.add_file(f.field, f.filename.empty() ? f.field : f.filename, + f.content_type.empty() ? "application/octet-stream" : f.content_type, f.data); + Json info = Json::object(); + info.set("field", Json(f.field)); + info.set("filename", Json(f.filename)); + info.set("bytes", Json(static_cast(f.data.size()))); + files.push(info); + } + if (!args.files.empty()) preview.set("_files", files); + out.body = form.encode(); + out.content_type = form.content_type(); + out.preview = preview; + return out; +} + +llm::RawResponse MediaService::execute(const PreparedCall& call, bool confirmed) { + if (call.effect != Effect::kRead && !confirmed) + throw llm::ApiError::invalid(table_.vendor(), "call '" + call.key + "' has effect " + + CallTable::effect_name(call.effect) + + " and needs explicit confirmation"); + const CallRow& row = table_.get(call.key); + const bool has_body = !call.body.empty(); + return client_->request_raw(CallTable::method_of(row), call.target, has_body ? &call.body : nullptr, + call.content_type, call.headers); +} + +llm::RawResponse MediaService::call_raw(const std::string& key, const CallArgs& args, bool confirmed) { + return execute(prepare(key, args), confirmed); +} + +Json MediaService::parse_json(const std::string& vendor, const llm::RawResponse& raw) { + if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object(); + try { + return Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError(vendor, 502, "BAD_RESPONSE", std::string("not JSON: ") + e.what()); + } +} + +Json MediaService::call_json(const std::string& key, const CallArgs& args, bool confirmed) { + return parse_json(table_.vendor(), call_raw(key, args, confirmed)); +} + +const Json* MediaService::member(const Json& doc, const std::string& key) { + return doc.is_object() ? doc.find(key) : nullptr; +} + +std::string MediaService::text(const Json& doc, const std::string& key, const std::string& fallback) { + const Json* v = member(doc, key); + if (v && v->is_string()) return v->as_string(); + if (v && v->is_number()) return v->dump(); + return fallback; +} + +} // namespace molo::media diff --git a/src/vendors/media/common/media_service.h b/src/vendors/media/common/media_service.h new file mode 100644 index 0000000..6e6dd2c --- /dev/null +++ b/src/vendors/media/common/media_service.h @@ -0,0 +1,78 @@ +#pragma once + +// MediaService: the validated path from "I want call X with these arguments" +// to bytes on the wire. prepare() is pure (no network): it checks the key, +// path arguments, query parameters, body fields and files against the call +// table row and builds the exact request. execute() sends it and refuses any +// call whose effect is not "read" unless the caller confirms explicitly. + +#include +#include +#include +#include + +#include "clients/body/json.h" +#include "llm/api_client.h" +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +using Json = clients::body::Json; + +struct FilePart { + std::string field; + std::string filename; + std::string content_type; + std::string data; +}; + +struct CallArgs { + std::map path; + std::map query; + Json body; // null = no body; otherwise a JSON object + std::vector files; // multipart rows only + llm::Headers headers; // extra request headers (Accept, Prefer, ...) +}; + +struct PreparedCall { + std::string vendor; + std::string key; + std::string method; // GET POST ... + std::string target; // path plus query string + std::string body; // exact bytes ("" = none) + std::string content_type; // "" when no body + llm::Headers headers; + Effect effect = Effect::kRead; + Json preview; // body as JSON, or the multipart part list + + // What a dry run shows: method, target, effect, content type, body preview. + Json summary() const; +}; + +class MediaService : public llm::Service { + public: + MediaService(std::shared_ptr client, const CallTable& table); + + const CallTable& table() const { return table_; } + const std::string& vendor() const { return table_.vendor(); } + + PreparedCall prepare(const std::string& key, const CallArgs& args = {}) const; + + // Non-read effects throw ApiError::invalid unless confirmed is true. The + // missing-credential error surfaces here, before any network traffic. + llm::RawResponse execute(const PreparedCall& call, bool confirmed); + + // prepare + execute + parse. Empty body -> {}. Non-JSON body -> ApiError. + Json call_json(const std::string& key, const CallArgs& args = {}, bool confirmed = false); + llm::RawResponse call_raw(const std::string& key, const CallArgs& args = {}, bool confirmed = false); + + // Helpers for typed wrappers. + static Json parse_json(const std::string& vendor, const llm::RawResponse& raw); + static const Json* member(const Json& doc, const std::string& key); + static std::string text(const Json& doc, const std::string& key, const std::string& fallback = ""); + + private: + const CallTable& table_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/output_scanner.cpp b/src/vendors/media/common/output_scanner.cpp new file mode 100644 index 0000000..71075e3 --- /dev/null +++ b/src/vendors/media/common/output_scanner.cpp @@ -0,0 +1,119 @@ +#include "vendors/media/common/output_scanner.h" + +#include +#include + +#include "vendors/media/common/caps.h" + +namespace molo::media { + +using clients::body::Json; + +namespace { + +std::string lower_ext(const std::string& name) { + std::string n = name.substr(0, name.find_first_of("?#")); + std::size_t dot = n.rfind('.'); + if (dot == std::string::npos || n.size() - dot > 8) return ""; + std::string e = n.substr(dot + 1); + std::transform(e.begin(), e.end(), e.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return e; +} + +struct Walk { + std::vector out; + std::set seen; + std::size_t nodes = 0; + MediaKind fallback; + + void add(const std::string& s, const std::string& key) { + if (out.size() >= MediaCaps::kMaxAssetsPerJob) return; + bool http = s.rfind("https://", 0) == 0 || s.rfind("http://", 0) == 0; + bool data = s.rfind("data:", 0) == 0; + if (!http && !data) return; + if (s.size() > MediaCaps::kMaxUrlChars && !data) return; + if (!seen.insert(s).second) return; + MediaAsset a; + if (data) { + std::size_t comma = s.find(','); + std::size_t semi = s.find(';'); + if (comma == std::string::npos) return; + a.mime = s.substr(5, std::min(semi, comma) - 5); + a.data_base64 = s.substr(comma + 1); + a.kind = fallback; + if (a.mime.rfind("video/", 0) == 0) a.kind = MediaKind::kVideo; + else if (a.mime.rfind("image/", 0) == 0) a.kind = MediaKind::kImage; + else if (a.mime.rfind("audio/", 0) == 0) a.kind = MediaKind::kAudio; + } else { + a.url = s; + a.kind = OutputScanner::kind_from_name(s, fallback); + if (a.kind == fallback) { + if (key.find("video") != std::string::npos) a.kind = MediaKind::kVideo; + else if (key.find("image") != std::string::npos) a.kind = MediaKind::kImage; + else if (key.find("audio") != std::string::npos) a.kind = MediaKind::kAudio; + } + a.mime = OutputScanner::mime_from_name(s); + } + out.push_back(a); + } + + void go(const Json& v, const std::string& key, std::size_t depth) { + if (depth > MediaCaps::kMaxScanDepth || ++nodes > MediaCaps::kMaxScanNodes) return; + if (v.is_string()) { + add(v.as_string(), key); + } else if (v.is_array()) { + for (const Json& x : v.as_array()) go(x, key, depth + 1); + } else if (v.is_object()) { + for (const auto& [k, x] : v.as_object()) { + if (OutputScanner::skipped_key(k)) continue; + go(x, k, depth + 1); + } + } + } +}; + +} // namespace + +bool OutputScanner::skipped_key(const std::string& key) { + static const std::set skip = { + "input", "request", "urls", "logs", "metrics", "webhook", "webhook_url", "callback_url", + "status_url", "cancel_url", "response_url", "polling_url", "get", "cancel", "stream", "web", + "params", "parameters", "meta", "links", "self", "thumbnail_url", "model_url"}; + return skip.count(key) > 0; +} + +MediaKind OutputScanner::kind_from_name(const std::string& name, MediaKind fallback) { + std::string e = lower_ext(name); + if (e == "mp4" || e == "webm" || e == "mov" || e == "mkv" || e == "m4v") return MediaKind::kVideo; + if (e == "png" || e == "jpg" || e == "jpeg" || e == "webp" || e == "gif" || e == "svg" || e == "avif") return MediaKind::kImage; + if (e == "mp3" || e == "wav" || e == "flac" || e == "ogg" || e == "m4a" || e == "aac") return MediaKind::kAudio; + if (e == "glb" || e == "gltf" || e == "obj" || e == "ply" || e == "usdz" || e == "fbx" || e == "stl") return MediaKind::kModel3d; + return fallback; +} + +std::string OutputScanner::mime_from_name(const std::string& name) { + std::string e = lower_ext(name); + if (e == "mp4" || e == "m4v") return "video/mp4"; + if (e == "webm") return "video/webm"; + if (e == "mov") return "video/quicktime"; + if (e == "png") return "image/png"; + if (e == "jpg" || e == "jpeg") return "image/jpeg"; + if (e == "webp") return "image/webp"; + if (e == "gif") return "image/gif"; + if (e == "svg") return "image/svg+xml"; + if (e == "mp3") return "audio/mpeg"; + if (e == "wav") return "audio/wav"; + if (e == "flac") return "audio/flac"; + if (e == "ogg") return "audio/ogg"; + if (e == "glb") return "model/gltf-binary"; + return ""; +} + +std::vector OutputScanner::scan(const Json& doc, MediaKind fallback) { + Walk w; + w.fallback = fallback; + w.go(doc, "", 0); + return std::move(w.out); +} + +} // namespace molo::media diff --git a/src/vendors/media/common/output_scanner.h b/src/vendors/media/common/output_scanner.h new file mode 100644 index 0000000..55095ca --- /dev/null +++ b/src/vendors/media/common/output_scanner.h @@ -0,0 +1,29 @@ +#pragma once + +// OutputScanner: finds result URLs / inline data in a provider's result JSON. +// Providers shape results differently (fal: images[].url, Replicate: output as +// string or array, Runway: output[]), so the adapters share one bounded walk +// instead of seven hand-rolled ones. Keys that carry request echoes or +// control links (input, request, urls, webhook, status_url, ...) are skipped. + +#include + +#include "vendors/media/common/job.h" + +namespace molo::media { + +class OutputScanner { + public: + // Walks `doc` (depth and node caps in MediaCaps) and returns de-duplicated + // assets in document order, at most MediaCaps::kMaxAssetsPerJob. + static std::vector scan(const clients::body::Json& doc, MediaKind fallback); + + // Kind from a URL / file name extension, else `fallback`. + static MediaKind kind_from_name(const std::string& name, MediaKind fallback); + // MIME for a kind + extension, "" when unknown. + static std::string mime_from_name(const std::string& name); + // True for keys whose subtree is request echo or control metadata. + static bool skipped_key(const std::string& key); +}; + +} // namespace molo::media diff --git a/src/vendors/media/common/registry.cpp b/src/vendors/media/common/registry.cpp new file mode 100644 index 0000000..41351ce --- /dev/null +++ b/src/vendors/media/common/registry.cpp @@ -0,0 +1,195 @@ +#include "vendors/media/common/registry.h" + +#include + +#include "llm/config.h" +#include "llm/error.h" +#include "vendors/media/bfl/bfl.h" +#include "vendors/media/fal/fal.h" +#include "vendors/media/higgsfield/higgsfield.h" +#include "vendors/media/ideogram/ideogram.h" +#include "vendors/media/kling/kling.h" +#include "vendors/media/leonardo/leonardo.h" +#include "vendors/media/luma/luma.h" +#include "vendors/media/minimax/minimax.h" +#include "vendors/media/recraft/recraft.h" +#include "vendors/media/replicate/replicate.h" +#include "vendors/media/runway/runway.h" +#include "vendors/media/stability/stability.h" + +namespace molo::media { + +const std::vector& MediaRegistry::providers() { + static const std::vector list = { + {"stability", "Stability AI", {"STABILITY_API_KEY"}, true, false, Evidence::kOpenApi, + "https://platform.stability.ai/account/keys", "v1.user.balance", + "v2beta image (generate, edit, upscale, control), 3D, audio; v1 engines. No video route in the published spec."}, + {"bfl", "Black Forest Labs", {"BFL_API_KEY"}, true, true, Evidence::kOpenApi, + "https://dashboard.bfl.ai/", "credits", "FLUX models and tools, polling via get_result, fine-tunes."}, + {"replicate", "Replicate", {"REPLICATE_API_TOKEN"}, true, true, Evidence::kOpenApi, + "https://replicate.com/account/api-tokens", "account.get", + "Predictions, models, deployments, files, trainings, webhook verification. Any model by owner/name."}, + {"fal", "fal", {"FAL_KEY"}, true, true, Evidence::kOpenApi, "https://fal.ai/dashboard/keys", + "platform.models.pricing", "Queue API (any endpoint id) and the model-related Platform API."}, + {"ideogram", "Ideogram", {"IDEOGRAM_API_KEY"}, true, true, Evidence::kOpenApi, + "https://ideogram.ai/manage-api", "account.usage", + "v2 image, SVG, video and ad tools; dry_run=true gives a free price quote."}, + {"recraft", "Recraft", {"RECRAFT_API_TOKEN"}, true, false, Evidence::kOpenApi, + "https://www.recraft.ai/profile/api", "users.me", "Generation, vector, background, upscale, styles."}, + {"leonardo", "Leonardo.Ai", {"LEONARDO_API_KEY"}, true, true, Evidence::kDocs, + "https://app.leonardo.ai/api-access", "v2.models", + "v2 generations for image, video, audio and 3D models. v1 reference pages are gone from the docs."}, + {"runway", "Runway", {"RUNWAY_API_KEY", "RUNWAYML_API_SECRET"}, true, true, Evidence::kSdk, + "https://dev.runwayml.com/", "organization.get", + "Text / image / video generation, tasks, uploads, avatars, voices, documents, routers, workflows."}, + {"luma", "Luma Dream Machine", {"LUMA_API_KEY"}, true, true, Evidence::kSdk, + "https://lumalabs.ai/api/keys", "credits.get", "Ray video, Photon image, reframe, modify, upscale, audio."}, + {"minimax", "MiniMax Hailuo", {"MINIMAX_API_KEY"}, false, true, Evidence::kDocs, + "https://platform.minimax.io/user-center/basic-information/interface-key", "", + "Hailuo video v1 and v2, file retrieval. No free account read exists."}, + {"kling", "Kling AI", {"KLING_ACCESS_KEY", "KLING_SECRET_KEY"}, false, true, Evidence::kProbeOnly, + "https://app.klingai.com/global/dev/api-key", "", + "Official API with JWT auth; docs are a JS app with no machine-readable spec: paths are unverified."}, + {"higgsfield", "Higgsfield", {"HF_API_KEY_ID", "HF_API_KEY_SECRET"}, true, true, Evidence::kOpenApi, + "https://cloud.higgsfield.ai/", "", + "One lifecycle for every model path; the OpenAPI document is explicitly supplementary."}, + }; + return list; +} + +const std::vector& MediaRegistry::unsupported() { + static const std::vector list = { + {"Midjourney", + "No public API. Midjourney documents only its web and Discord products; third-party relays are unofficial " + "and violate its terms, so none is implemented."}, + {"Pika", + "pika.art/api advertises a membership API that resells other vendors' models, but no endpoint, schema or " + "authentication document was found (2026-10-25), so nothing is implemented. Pika models are reachable " + "through fal."}, + }; + return list; +} + +const ProviderInfo* MediaRegistry::find(const std::string& id) { + for (const ProviderInfo& p : providers()) + if (p.id == id) return &p; + return nullptr; +} + +const char* MediaRegistry::evidence_name(Evidence e) { + switch (e) { + case Evidence::kOpenApi: return "openapi"; + case Evidence::kSdk: return "sdk"; + case Evidence::kDocs: return "docs"; + default: return "probe-only"; + } +} + +const CallTable& MediaRegistry::table(const std::string& id) { + if (id == "stability") return StabilityCalls::table(); + if (id == "bfl") return BflCalls::table(); + if (id == "replicate") return ReplicateCalls::table(); + if (id == "fal") return FalCalls::table(); + if (id == "ideogram") return IdeogramCalls::table(); + if (id == "recraft") return RecraftCalls::table(); + if (id == "leonardo") return LeonardoCalls::table(); + if (id == "runway") return RunwayCalls::table(); + if (id == "luma") return LumaCalls::table(); + if (id == "minimax") return MinimaxCalls::table(); + if (id == "kling") return KlingCalls::table(); + if (id == "higgsfield") return HiggsfieldCalls::table(); + throw llm::ApiError::invalid("media", "unknown provider '" + id + "'"); +} + +bool MediaRegistry::configured(const std::string& id) { + const ProviderInfo* p = find(id); + if (!p) throw llm::ApiError::invalid("media", "unknown provider '" + id + "'"); + if (id == "runway") return !llm::Config::get("RUNWAY_API_KEY").empty() || !llm::Config::get("RUNWAYML_API_SECRET").empty(); + for (const std::string& e : p->env) + if (llm::Config::get(e.c_str()).empty()) return false; + return true; +} + +std::shared_ptr MediaRegistry::service(const std::string& id, const std::string& key) { + std::string slot = id; + if (id == "fal") slot = key.rfind("platform.", 0) == 0 ? "fal.platform" : "fal.queue"; + auto it = services_.find(slot); + if (it != services_.end()) return it->second; + std::shared_ptr s; + if (id == "stability") s = StabilityClient::from_env(); + else if (id == "bfl") s = BflClient::from_env(); + else if (id == "replicate") s = ReplicateClient::from_env(); + else if (id == "fal") s = slot == "fal.platform" ? std::shared_ptr(FalPlatform::from_env()) + : std::shared_ptr(FalQueue::from_env()); + else if (id == "ideogram") s = IdeogramClient::from_env(); + else if (id == "recraft") s = RecraftClient::from_env(); + else if (id == "leonardo") s = LeonardoClient::from_env(); + else if (id == "runway") s = RunwayClient::from_env(); + else if (id == "luma") s = LumaClient::from_env(); + else if (id == "minimax") s = MinimaxClient::from_env(); + else if (id == "kling") s = KlingClient::from_env(); + else if (id == "higgsfield") s = HiggsfieldClient::from_env(); + else throw llm::ApiError::invalid("media", "unknown provider '" + id + "'"); + services_[slot] = s; + return s; +} + +std::shared_ptr MediaRegistry::image(const std::string& id) { + auto it = images_.find(id); + if (it != images_.end()) return it->second; + std::shared_ptr g; + if (id == "stability") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "bfl") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "replicate") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "fal") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "ideogram") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "recraft") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "leonardo") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "runway") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "luma") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "higgsfield") g = std::make_shared(std::static_pointer_cast(service(id))); + else throw llm::ApiError::invalid("media", "provider '" + id + "' has no image generator"); + images_[id] = g; + return g; +} + +std::shared_ptr MediaRegistry::video(const std::string& id) { + auto it = videos_.find(id); + if (it != videos_.end()) return it->second; + std::shared_ptr g; + if (id == "bfl") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "replicate") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "fal") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "ideogram") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "leonardo") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "runway") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "luma") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "minimax") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "kling") g = std::make_shared(std::static_pointer_cast(service(id))); + else if (id == "higgsfield") g = std::make_shared(std::static_pointer_cast(service(id))); + else throw llm::ApiError::invalid("media", "provider '" + id + "' has no video generator"); + videos_[id] = g; + return g; +} + +Json MediaRegistry::account(const std::string& id) { + const ProviderInfo* p = find(id); + if (!p) throw llm::ApiError::invalid("media", "unknown provider '" + id + "'"); + if (p->account_key.empty()) + throw llm::ApiError::invalid("media", id + " has no free account read; use a generation poll to prove the credential"); + std::shared_ptr s = service(id, p->account_key); + CallArgs args; + if (id == "fal") args.query["endpoint_id"] = "fal-ai/flux/dev"; + if (id == "stability") args.headers["Accept"] = "application/json"; + if (id == "ideogram") { // usage requires start_time (spec: required, date-time) + std::time_t t = std::time(nullptr) - 24 * 3600; + std::tm tm{}; + gmtime_r(&t, &tm); + char buf[32]; + std::strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%SZ", &tm); + args.query["start_time"] = buf; + } + return s->call_json(p->account_key, args); +} + +} // namespace molo::media diff --git a/src/vendors/media/common/registry.h b/src/vendors/media/common/registry.h new file mode 100644 index 0000000..5c82410 --- /dev/null +++ b/src/vendors/media/common/registry.h @@ -0,0 +1,72 @@ +#pragma once + +// MediaRegistry: the one list of providers in package media, what each one +// can do, which environment variables it needs and how well its API surface +// is verified. Builds generators and call services from the environment on +// demand and caches them. Products without a public API are listed too, with +// the reason, so nothing is silently missing. + +#include +#include +#include +#include + +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" + +namespace molo::media { + +// How the provider's call table was checked (see each SPEC). +enum class Evidence { + kOpenApi, // vendor-published OpenAPI document, checked by tools/check_spec.py + kSdk, // official SDK source, checked by tools/check_spec.py + kDocs, // vendor doc pages, checked by tools/check_spec.py + kProbeOnly // no machine-readable source: community sources plus live probes +}; + +struct ProviderInfo { + std::string id; + std::string name; + std::vector env; // every variable the credential needs + bool image = false; + bool video = false; + Evidence evidence = Evidence::kOpenApi; + std::string console_url; // where the key is created + std::string account_key; // table key of a free read that proves the credential ("" = none) + std::string note; +}; + +struct UnsupportedProduct { + std::string name; + std::string reason; +}; + +class MediaRegistry { + public: + static const std::vector& providers(); + static const std::vector& unsupported(); + static const ProviderInfo* find(const std::string& id); + static const char* evidence_name(Evidence evidence); + // Table of one provider (no credential needed). Throws ApiError::invalid. + static const CallTable& table(const std::string& id); + + // True when every env variable the provider needs is set (llm::Config). + static bool configured(const std::string& id); + + // Built from the environment on first use, then cached for this registry. + // `key` picks the host for providers with two (fal: queue.* / platform.*). + std::shared_ptr service(const std::string& id, const std::string& key = ""); + std::shared_ptr image(const std::string& id); + std::shared_ptr video(const std::string& id); + + // Free read that proves the credential works (balance, account, models). + // Returns the provider's JSON. Throws NotConfiguredError without a key. + Json account(const std::string& id); + + private: + std::map> services_; + std::map> images_; + std::map> videos_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/fal/calls.cpp b/src/vendors/media/fal/calls.cpp new file mode 100644 index 0000000..6b30876 --- /dev/null +++ b/src/vendors/media/fal/calls.cpp @@ -0,0 +1,41 @@ +#include "vendors/media/fal/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"queue.submit", "POST", "/{owner}/{app}", "submitQueueRequest", "json", "*", "fal_webhook,fal_max_queue_length", "paid"}, + {"queue.submit_path", "POST", "/{owner}/{app}/{path}", "submitQueueRequestWithPath", "json", "*", "fal_webhook,fal_max_queue_length", "paid"}, + {"queue.status", "GET", "/{owner}/{app}/requests/{request_id}/status", "getQueueRequestStatus", "none", "", "logs", "read"}, + {"queue.result", "GET", "/{owner}/{app}/requests/{request_id}", "getQueueRequestResult", "none", "", "", "read"}, + {"queue.cancel", "PUT", "/{owner}/{app}/requests/{request_id}/cancel", "cancelQueueRequest", "none", "", "", "write"}, + {"platform.models.insights", "GET", "/v1/models/insights", "getModelInsights", "none", "", "endpoint_id,q,limit", "read"}, + {"platform.models.search", "GET", "/v1/models", "getModels", "none", "", "limit,cursor,endpoint_id,q,category,status,sort,expand", "read"}, + {"platform.models.pricing", "GET", "/v1/models/pricing", "getPricing", "none", "", "endpoint_id", "read"}, + {"platform.models.estimate", "POST", "/v1/models/pricing/estimate", "estimatePricing", "json", "estimate_type,endpoints", "", "read"}, + {"platform.models.usage", "GET", "/v1/models/usage", "getUsage", "none", "", "limit,cursor,start,end,timezone,timeframe,bound_to_timeframe,source,endpoint_id,api_key_id,login_username,tag,expand", "read"}, + {"platform.models.analytics", "GET", "/v1/models/analytics", "getAnalytics", "none", "", "limit,cursor,start,end,timezone,timeframe,bound_to_timeframe,endpoint_id,expand", "read"}, + {"platform.models.billing_events", "GET", "/v1/models/billing-events", "getBillingEvents", "none", "", "limit,cursor,start,end,source,endpoint_id,request_id,tag,api_key_id,login_username,expand", "read"}, + {"platform.models.delete_payloads", "DELETE", "/v1/models/requests/{request_id}/payloads", "deleteRequestPayloads", "none", "", "", "delete"}, + {"platform.models.requests_by_endpoint", "GET", "/v1/models/requests/by-endpoint", "listRequestsByEndpoint", "none", "", "limit,cursor,endpoint_id,start,end,status,request_id,expand,sort_by", "read"}, + {"platform.models.requests_search", "GET", "/v1/models/requests/search", "searchRequests", "none", "", "limit,cursor,query,image_url,video_url,endpoint_id,endpoint,exclude_api_requests,only_api_requests,min_similarity", "read"}, + {"platform.meta", "GET", "/v1/meta", "getMeta", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"QueueStatus", "status,request_id,response_url,status_url,cancel_url,logs,metrics,queue_position"}, +}; +// FIELDS-END + +const CallTable& FalCalls::table() { + static const CallTable t("fal", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t FalCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* FalCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/fal/calls.h b/src/vendors/media/fal/calls.h new file mode 100644 index 0000000..22fe83f --- /dev/null +++ b/src/vendors/media/fal/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct FalCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/fal/fal.cpp b/src/vendors/media/fal/fal.cpp new file mode 100644 index 0000000..8de72cc --- /dev/null +++ b/src/vendors/media/fal/fal.cpp @@ -0,0 +1,260 @@ +#include "vendors/media/fal/fal.h" + +#include + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/caps.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs ids(const FalEndpoint& e, const std::string& request_id = "") { + CallArgs a; + a.path["owner"] = e.owner; + a.path["app"] = e.app; + if (!request_id.empty()) a.path["request_id"] = request_id; + return a; +} + +} // namespace + +FalEndpoint FalEndpoint::parse(const std::string& id) { + std::vector parts; + std::size_t start = 0; + while (true) { + std::size_t slash = id.find('/', start); + parts.push_back(id.substr(start, slash == std::string::npos ? std::string::npos : slash - start)); + if (slash == std::string::npos) break; + start = slash + 1; + } + if (parts.size() < 2 || parts.size() > 5) + throw llm::ApiError::invalid("fal", "endpoint must look like owner/app or owner/app/path"); + for (const std::string& p : parts) + if (!CallTable::valid_segment(p)) throw llm::ApiError::invalid("fal", "bad endpoint id"); + FalEndpoint e; + e.owner = parts[0]; + e.app = parts[1]; + // Everything below owner/app is the {path} of the submit_path row; the + // call table accepts several "/"-joined segments for it. + for (std::size_t i = 2; i < parts.size(); ++i) e.path += (i > 2 ? "/" : "") + parts[i]; + return e; +} + +PreparedCall FalQueue::plan(const FalEndpoint& e, const Json& body, const FalSubmitOptions& o) const { + CallArgs a = ids(e); + a.body = body; + if (!e.path.empty()) { + a.path["path"] = e.path; + } + if (!o.webhook.empty()) a.query["fal_webhook"] = o.webhook; + if (o.max_queue_length > 0) a.query["fal_max_queue_length"] = std::to_string(o.max_queue_length); + if (!o.priority.empty()) { + if (o.priority != "low" && o.priority != "normal" && o.priority != "high") + throw llm::ApiError::invalid("fal", "priority must be low, normal or high"); + a.headers["X-Fal-Queue-Priority"] = o.priority; + } + return prepare(e.path.empty() ? "queue.submit" : "queue.submit_path", a); +} + +Json FalQueue::submit(const FalEndpoint& e, const Json& body, const FalSubmitOptions& o, bool confirm) { + return parse_json("fal", execute(plan(e, body, o), confirm)); +} + +Json FalQueue::status(const FalEndpoint& e, const std::string& request_id, bool logs) { + CallArgs a = ids(e, request_id); + if (logs) a.query["logs"] = "1"; + return call_json("queue.status", a); +} + +Json FalQueue::result(const FalEndpoint& e, const std::string& request_id) { + return call_json("queue.result", ids(e, request_id)); +} + +Json FalQueue::cancel(const FalEndpoint& e, const std::string& request_id, bool confirm) { + return call_json("queue.cancel", ids(e, request_id), confirm); +} + +// ---------------------------------------------------------------- platform + +Json FalPlatform::models(const std::string& query, const std::string& category, std::size_t max_pages, + std::size_t max_items) { + if (max_pages < 1 || max_pages > MediaCaps::kMaxPages) throw llm::ApiError::invalid("fal", "max_pages out of range"); + if (max_items < 1 || max_items > MediaCaps::kMaxItemsReturned) throw llm::ApiError::invalid("fal", "max_items out of range"); + Json all = Json::array(); + std::string cursor; + bool more = false; + std::size_t pages = 0; + while (pages < max_pages) { + CallArgs a; + a.query["limit"] = std::to_string(std::min(max_items - all.as_array().size(), 100)); + if (!query.empty()) a.query["q"] = query; + if (!category.empty()) a.query["category"] = category; + if (!cursor.empty()) a.query["cursor"] = cursor; + Json page = call_json("platform.models.search", a); + ++pages; + if (const Json* m = page.find("models"); m && m->is_array()) + for (const Json& x : m->as_array()) + if (all.as_array().size() < max_items) all.push(x); + const Json* hm = page.find("has_more"); + more = hm && hm->is_bool() && hm->as_bool(); + cursor = AdapterSupport::str(page, "next_cursor"); + if (!more || cursor.empty() || all.as_array().size() >= max_items) break; + } + Json out = Json::object(); + out.set("models", all); + out.set("pages", Json(static_cast(pages))); + out.set("truncated", Json(more)); + return out; +} + +Json FalPlatform::pricing(const std::string& endpoint_id) { + CallArgs a; + a.query["endpoint_id"] = endpoint_id; + return call_json("platform.models.pricing", a); +} + +Json FalPlatform::estimate(const Json& body) { + CallArgs a; + a.body = body; + return call_json("platform.models.estimate", a); +} + +Json FalPlatform::usage(const std::map& query) { + CallArgs a; + a.query = query; + return call_json("platform.models.usage", a); +} + +Json FalPlatform::analytics(const std::map& query) { + CallArgs a; + a.query = query; + return call_json("platform.models.analytics", a); +} + +Json FalPlatform::meta() { return call_json("platform.meta"); } + +Json FalPlatform::delete_payloads(const std::string& request_id, bool confirm) { + CallArgs a; + a.path["request_id"] = request_id; + return call_json("platform.models.delete_payloads", a, confirm); +} + +// -------------------------------------------------------------------- jobs + +JobTicket FalJobs::ticket_from(const Json& reply, const FalEndpoint& e, MediaKind kind) { + JobTicket t; + t.provider = "fal"; + t.model = e.full(); + t.kind = kind; + t.id = AdapterSupport::str(reply, "request_id"); + if (t.id.empty()) throw llm::ApiError("fal", 502, "BAD_RESPONSE", "no request_id in the submit reply"); + t.meta = Json::object(); + t.meta.set("endpoint", Json(e.full())); + return t; +} + +JobStatus FalJobs::poll(FalQueue& queue, const JobTicket& ticket) { + FalEndpoint e = FalEndpoint::parse(AdapterSupport::str(ticket.meta, "endpoint")); + Json st = queue.status(e, ticket.id); + JobStatus s; + s.raw_state = AdapterSupport::str(st, "status"); + if (s.raw_state == "IN_QUEUE") { + s.state = JobState::kQueued; + return s; + } + if (s.raw_state != "COMPLETED") { + s.state = JobState::kRunning; + return s; + } + // COMPLETED is final for success, failure and cancellation alike: the + // result fetch tells which (a failed request answers with an HTTP error). + try { + Json res = queue.result(e, ticket.id); + s.state = JobState::kSucceeded; + s.assets = OutputScanner::scan(res, ticket.kind); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "completed without a result URL: " + res.dump().substr(0, 300); + } + } catch (const llm::ApiError& err) { + if (err.http_status() == 401 || err.http_status() == 403 || err.http_status() >= 500 || err.http_status() == 429) throw; + s.state = JobState::kFailed; + s.error = err.message(); + } + return s; +} + +bool FalJobs::cancel(FalQueue& queue, const JobTicket& ticket, bool confirm) { + AdapterSupport::need_confirm(confirm, "fal", "cancel"); + queue.cancel(FalEndpoint::parse(AdapterSupport::str(ticket.meta, "endpoint")), ticket.id, true); + return true; +} + +Json FalImageGenerator::body(const ImageRequest& r) const { + r.validate(); + Json b = Json::object(); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "negative_prompt", r.negative_prompt); + if (r.width && r.height) { + Json size = Json::object(); + size.set("width", Json(r.width)); + size.set("height", Json(r.height)); + b.set("image_size", size); + } + AdapterSupport::put_seed(b, "seed", r.seed); + if (r.count > 1) b.set("num_images", Json(r.count)); + AdapterSupport::put(b, "image_url", r.source_url); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json FalImageGenerator::plan(const ImageRequest& r) const { + if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required"); + return queue_->plan(FalEndpoint::parse(r.model), body(r), {}).summary(); +} + +JobTicket FalImageGenerator::submit(const ImageRequest& r, bool confirm) { + if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required"); + FalEndpoint e = FalEndpoint::parse(r.model); + Json b = body(r); + AdapterSupport::need_confirm(confirm, "fal", "queue submit"); + return FalJobs::ticket_from(queue_->submit(e, b, {}, true), e, MediaKind::kImage); +} + +JobStatus FalImageGenerator::poll(const JobTicket& t) { return FalJobs::poll(*queue_, t); } +bool FalImageGenerator::cancel(const JobTicket& t, bool confirm) { return FalJobs::cancel(*queue_, t, confirm); } + +Json FalVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + Json b = Json::object(); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "negative_prompt", r.negative_prompt); + if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds))); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put(b, "resolution", r.resolution); + AdapterSupport::put_seed(b, "seed", r.seed); + AdapterSupport::put(b, "image_url", r.image_url); + AdapterSupport::put(b, "end_image_url", r.end_image_url); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json FalVideoGenerator::plan(const VideoRequest& r) const { + if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required"); + return queue_->plan(FalEndpoint::parse(r.model), body(r), {}).summary(); +} + +JobTicket FalVideoGenerator::submit(const VideoRequest& r, bool confirm) { + if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required"); + FalEndpoint e = FalEndpoint::parse(r.model); + Json b = body(r); + AdapterSupport::need_confirm(confirm, "fal", "queue submit"); + return FalJobs::ticket_from(queue_->submit(e, b, {}, true), e, MediaKind::kVideo); +} + +JobStatus FalVideoGenerator::poll(const JobTicket& t) { return FalJobs::poll(*queue_, t); } +bool FalVideoGenerator::cancel(const JobTicket& t, bool confirm) { return FalJobs::cancel(*queue_, t, confirm); } + +} // namespace molo::media diff --git a/src/vendors/media/fal/fal.h b/src/vendors/media/fal/fal.h new file mode 100644 index 0000000..a220c16 --- /dev/null +++ b/src/vendors/media/fal/fal.h @@ -0,0 +1,128 @@ +#pragma once + +// fal: the queue API (queue.fal.run: submit, status, result, cancel) and the +// model-related Platform API (api.fal.ai/v1: model search, pricing, estimate, +// usage, analytics, billing events, request listing). Auth is +// "Authorization: Key " on both hosts. Platform groups outside model +// inference (assets, serverless, compute, keys, organization) are out of scope +// and listed as exclusions in tools/check_spec.py. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/fal/calls.h" + +namespace molo::media { + +// "owner/app" or "owner/app/path" (fal-ai/flux/dev, fal-ai/flux/dev/image-to-image). +struct FalEndpoint { + std::string owner, app, path; + static FalEndpoint parse(const std::string& id); // throws ApiError::invalid + std::string app_id() const { return owner + "/" + app; } + std::string full() const { return path.empty() ? app_id() : app_id() + "/" + path; } +}; + +struct FalSubmitOptions { + std::string priority; // "", "low", "normal", "high" -> X-Fal-Queue-Priority + std::string webhook; // https URL -> fal_webhook + int max_queue_length = 0; // 0 = unset -> fal_max_queue_length +}; + +class FalQueue : public MediaService { + public: + static constexpr const char* kHost = "https://queue.fal.run"; + static constexpr const char* kKeyEnv = "FAL_KEY"; + + explicit FalQueue(std::shared_ptr client) : MediaService(std::move(client), FalCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::header("fal", kHost, kKeyEnv, "Authorization"); } + // fal sends the key as "Key ". + static std::string header_value(const std::string& key) { return key.empty() ? "" : "Key " + key; } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), header_value(ClientFactory::env({kKeyEnv})))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), header_value(key))); + } + + PreparedCall plan(const FalEndpoint& endpoint, const Json& body, const FalSubmitOptions& options) const; + // Paid. Returns {request_id, response_url, status_url, cancel_url, ...}. + Json submit(const FalEndpoint& endpoint, const Json& body, const FalSubmitOptions& options, bool confirm); + Json status(const FalEndpoint& endpoint, const std::string& request_id, bool logs = false); + Json result(const FalEndpoint& endpoint, const std::string& request_id); + Json cancel(const FalEndpoint& endpoint, const std::string& request_id, bool confirm); +}; + +class FalPlatform : public MediaService { + public: + static constexpr const char* kHost = "https://api.fal.ai"; + + explicit FalPlatform(std::shared_ptr client) : MediaService(std::move(client), FalCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::header("fal", kHost, FalQueue::kKeyEnv, "Authorization"); } + static std::shared_ptr from_env() { + return std::make_shared( + ClientFactory::real(config(), FalQueue::header_value(ClientFactory::env({FalQueue::kKeyEnv})))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), FalQueue::header_value(key))); + } + + // Search; walks next_cursor up to max_pages / max_items and says so loudly + // ("truncated": true) when the server had more. + Json models(const std::string& query, const std::string& category, std::size_t max_pages = 3, + std::size_t max_items = 200); + Json pricing(const std::string& endpoint_id); + Json estimate(const Json& body); // pure computation on fal's side, no charge + Json usage(const std::map& query); + Json analytics(const std::map& query); + Json meta(); + Json delete_payloads(const std::string& request_id, bool confirm); // destructive, no default +}; + +class FalImageGenerator : public ImageGenerator { + public: + explicit FalImageGenerator(std::shared_ptr q) : queue_(std::move(q)) {} + std::string provider() const override { return "fal"; } + bool configured() const override { return queue_->configured(); } + std::vector models() const override { + return {"fal-ai/flux/dev", "fal-ai/flux/schnell", "fal-ai/flux-pro/v1.1", "fal-ai/recraft/v3/text-to-image", + "fal-ai/ideogram/v3", "fal-ai/stable-diffusion-v35-large"}; + } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 1500; } + + private: + Json body(const ImageRequest& request) const; + std::shared_ptr queue_; +}; + +class FalVideoGenerator : public VideoGenerator { + public: + explicit FalVideoGenerator(std::shared_ptr q) : queue_(std::move(q)) {} + std::string provider() const override { return "fal"; } + bool configured() const override { return queue_->configured(); } + std::vector models() const override { + return {"fal-ai/kling-video/v2.1/master/text-to-video", "fal-ai/minimax/hailuo-02/standard/text-to-video", + "fal-ai/veo3", "fal-ai/luma-dream-machine"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 4000; } + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr queue_; +}; + +// Shared queue polling for both fal adapters. +struct FalJobs { + static JobTicket ticket_from(const Json& submit_reply, const FalEndpoint& endpoint, MediaKind kind); + static JobStatus poll(FalQueue& queue, const JobTicket& ticket); + static bool cancel(FalQueue& queue, const JobTicket& ticket, bool confirm); +}; + +} // namespace molo::media diff --git a/src/vendors/media/higgsfield/calls.cpp b/src/vendors/media/higgsfield/calls.cpp new file mode 100644 index 0000000..070cb50 --- /dev/null +++ b/src/vendors/media/higgsfield/calls.cpp @@ -0,0 +1,35 @@ +#include "vendors/media/higgsfield/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"higgsfield-ai.soul.standard", "POST", "/higgsfield-ai/soul/standard", "post_higgsfield_ai_soul_standard", "json", "prompt,num_images,resolution,aspect_ratio", "", "paid"}, + {"requests.id.status", "GET", "/requests/{request_id}/status", "get_request_status", "none", "", "", "read"}, + {"requests.id.cancel", "POST", "/requests/{request_id}/cancel", "cancel_request", "none", "", "", "write"}, + {"kling-video.v2.5-turbo.pro.image-to-video", "POST", "/kling-video/v2.5-turbo/pro/image-to-video", "post_kling_video_v2_5_turbo_pro_image_to_video", "json", "prompt,duration,cfg_scale,image_url,negative_prompt", "", "paid"}, + {"kling-video.v2.5-turbo.pro.text-to-video", "POST", "/kling-video/v2.5-turbo/pro/text-to-video", "post_kling_video_v2_5_turbo_pro_text_to_video", "json", "prompt,duration,cfg_scale,negative_prompt", "", "paid"}, + {"kling-video.v2.5-turbo.standard.image-to-video", "POST", "/kling-video/v2.5-turbo/standard/image-to-video", "post_kling_video_v2_5_turbo_standard_image_to_video", "json", "prompt,duration,cfg_scale,image_url,negative_prompt", "", "paid"}, + {"minimax.hailuo-2.3.standard.image-to-video", "POST", "/minimax/hailuo-2.3/standard/image-to-video", "post_minimax_hailuo_2_3_standard_image_to_video", "json", "prompt,duration,image_url,prompt_optimizer", "", "paid"}, + {"minimax.hailuo-2.3.standard.text-to-video", "POST", "/minimax/hailuo-2.3/standard/text-to-video", "post_minimax_hailuo_2_3_standard_text_to_video", "json", "prompt,duration,prompt_optimizer", "", "paid"}, + {"requests.id", "DELETE", "/requests/{request_id}", "delete_request", "none", "", "", "delete"}, + {"model.submit", "POST", "/{model_path}", "", "json", "*", "", "paid"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"RequestStatus", "status,request_id,status_url,cancel_url,error,images,video,url"}, +}; +// FIELDS-END + +const CallTable& HiggsfieldCalls::table() { + static const CallTable t("higgsfield", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t HiggsfieldCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* HiggsfieldCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/higgsfield/calls.h b/src/vendors/media/higgsfield/calls.h new file mode 100644 index 0000000..bfe48be --- /dev/null +++ b/src/vendors/media/higgsfield/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct HiggsfieldCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/higgsfield/higgsfield.cpp b/src/vendors/media/higgsfield/higgsfield.cpp new file mode 100644 index 0000000..be086db --- /dev/null +++ b/src/vendors/media/higgsfield/higgsfield.cpp @@ -0,0 +1,156 @@ +#include "vendors/media/higgsfield/higgsfield.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs for_model(const std::string& model_path, const Json& body) { + CallArgs a; + a.body = body; + if (HiggsfieldClient::row_for(model_path) == "model.submit") a.path["model_path"] = model_path; + return a; +} + +CallArgs by_request(const std::string& id) { + CallArgs a; + a.path["request_id"] = id; + return a; +} + +} // namespace + +std::string HiggsfieldClient::row_for(const std::string& model_path) { + std::string key = model_path; + for (char& c : key) + if (c == '/') c = '.'; + return HiggsfieldCalls::table().find(key) ? key : "model.submit"; +} + +PreparedCall HiggsfieldClient::plan(const std::string& model_path, const Json& body) const { + return prepare(row_for(model_path), for_model(model_path, body)); +} + +Json HiggsfieldClient::submit(const std::string& model_path, const Json& body, bool confirm) { + return call_json(row_for(model_path), for_model(model_path, body), confirm); +} + +Json HiggsfieldClient::status(const std::string& id) { return call_json("requests.id.status", by_request(id)); } +Json HiggsfieldClient::cancel(const std::string& id, bool confirm) { + return call_json("requests.id.cancel", by_request(id), confirm); +} +Json HiggsfieldClient::remove(const std::string& id, bool confirm) { + return call_json("requests.id", by_request(id), confirm); +} + +JobState HiggsfieldJobs::state_of(const std::string& status) { + if (status == "completed") return JobState::kSucceeded; + if (status == "failed" || status == "nsfw") return JobState::kFailed; + if (status == "canceled") return JobState::kCancelled; + if (status == "in_progress") return JobState::kRunning; + return JobState::kQueued; // queued +} + +JobTicket HiggsfieldJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) { + JobTicket t; + t.provider = "higgsfield"; + t.model = model; + t.kind = kind; + t.id = AdapterSupport::str(reply, "request_id"); + if (t.id.empty()) throw llm::ApiError("higgsfield", 502, "BAD_RESPONSE", "no request_id in the reply"); + return t; +} + +JobStatus HiggsfieldJobs::poll(HiggsfieldClient& client, const JobTicket& ticket) { + Json st = client.status(ticket.id); + JobStatus s; + s.raw_state = AdapterSupport::str(st, "status"); + s.state = state_of(s.raw_state); + if (s.raw_state == "nsfw") s.error = "rejected by the content filter (nsfw)"; + if (s.state == JobState::kFailed && s.error.empty()) s.error = AdapterSupport::str(st, "error"); + if (s.state == JobState::kSucceeded) { + for (const char* key : {"images", "video"}) + if (const Json* x = st.find(key)) { + auto found = OutputScanner::scan(*x, ticket.kind); + s.assets.insert(s.assets.end(), found.begin(), found.end()); + } + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "completed without images or video"; + } + } + return s; +} + +Json HiggsfieldImageGenerator::body(const ImageRequest& r, std::string& model) const { + r.validate(); + model = r.model.empty() ? "higgsfield-ai/soul/standard" : r.model; + Json b = Json::object(); + AdapterSupport::put(b, "prompt", r.prompt); + if (r.count > 1) b.set("num_images", Json(r.count)); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put_seed(b, "seed", r.seed); + AdapterSupport::put(b, "image_url", r.source_url); + if (r.width || r.height || !r.negative_prompt.empty()) + throw llm::ApiError::invalid("higgsfield", "width / height / negative_prompt are model-specific: pass them in extra"); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json HiggsfieldImageGenerator::plan(const ImageRequest& request) const { + std::string model; + Json b = body(request, model); + return client_->plan(model, b).summary(); +} + +JobTicket HiggsfieldImageGenerator::submit(const ImageRequest& request, bool confirm) { + std::string model; + Json b = body(request, model); + AdapterSupport::need_confirm(confirm, "higgsfield", "submit"); + return HiggsfieldJobs::ticket_from(client_->submit(model, b, true), model, MediaKind::kImage); +} + +bool HiggsfieldImageGenerator::cancel(const JobTicket& t, bool confirm) { + AdapterSupport::need_confirm(confirm, "higgsfield", "cancel"); + client_->cancel(t.id, true); + return true; +} + +Json HiggsfieldVideoGenerator::body(const VideoRequest& r, std::string& model) const { + r.validate(); + if (r.model.empty()) throw llm::ApiError::invalid("higgsfield", "model path is required (see models())"); + model = r.model; + Json b = Json::object(); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "negative_prompt", r.negative_prompt); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put(b, "resolution", r.resolution); + AdapterSupport::put_int(b, "duration", r.duration_seconds); + AdapterSupport::put(b, "image_url", r.image_url); + AdapterSupport::put(b, "end_image_url", r.end_image_url); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json HiggsfieldVideoGenerator::plan(const VideoRequest& request) const { + std::string model; + Json b = body(request, model); + return client_->plan(model, b).summary(); +} + +JobTicket HiggsfieldVideoGenerator::submit(const VideoRequest& request, bool confirm) { + std::string model; + Json b = body(request, model); + AdapterSupport::need_confirm(confirm, "higgsfield", "submit"); + return HiggsfieldJobs::ticket_from(client_->submit(model, b, true), model, MediaKind::kVideo); +} + +bool HiggsfieldVideoGenerator::cancel(const JobTicket& t, bool confirm) { + AdapterSupport::need_confirm(confirm, "higgsfield", "cancel"); + client_->cancel(t.id, true); + return true; +} + +} // namespace molo::media diff --git a/src/vendors/media/higgsfield/higgsfield.h b/src/vendors/media/higgsfield/higgsfield.h new file mode 100644 index 0000000..0bdb8a4 --- /dev/null +++ b/src/vendors/media/higgsfield/higgsfield.h @@ -0,0 +1,94 @@ +#pragma once + +// Higgsfield cloud API (https://api.higgsfield.ai): every model is an +// endpoint path ("higgsfield-ai/soul/standard", "kling-video/v2.5-turbo/pro/ +// text-to-video", ...); the shared lifecycle is submit -> GET +// /requests/{id}/status -> optional cancel / delete. Auth is +// "Authorization: Key :". The published +// OpenAPI document is explicitly supplementary: a model absent from it is not +// unavailable, so submit() accepts any model path through the generic row. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/higgsfield/calls.h" + +namespace molo::media { + +class HiggsfieldClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.higgsfield.ai"; + static constexpr const char* kIdEnv = "HF_API_KEY_ID"; + static constexpr const char* kSecretEnv = "HF_API_KEY_SECRET"; + + explicit HiggsfieldClient(std::shared_ptr client) + : MediaService(std::move(client), HiggsfieldCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::header("higgsfield", kHost, kIdEnv, "Authorization"); } + // "Key id:secret"; empty when either half is missing. + static std::string header_value(const std::string& id, const std::string& secret) { + return id.empty() || secret.empty() ? "" : "Key " + id + ":" + secret; + } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real( + config(), header_value(ClientFactory::env({kIdEnv}), ClientFactory::env({kSecretEnv})))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& id, + const std::string& secret) { + return std::make_shared(ClientFactory::with(std::move(t), config(), header_value(id, secret))); + } + + // Uses the spec's typed row when the path is one of them, else model.submit. + Json submit(const std::string& model_path, const Json& body, bool confirm); + PreparedCall plan(const std::string& model_path, const Json& body) const; + Json status(const std::string& request_id); + Json cancel(const std::string& request_id, bool confirm); // queued requests only + Json remove(const std::string& request_id, bool confirm); // deletes files and parameters + static std::string row_for(const std::string& model_path); +}; + +class HiggsfieldJobs { + public: + static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind); + static JobStatus poll(HiggsfieldClient& client, const JobTicket& ticket); + static JobState state_of(const std::string& status); +}; + +class HiggsfieldImageGenerator : public ImageGenerator { + public: + explicit HiggsfieldImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "higgsfield"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { return {"higgsfield-ai/soul/standard"}; } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override { return HiggsfieldJobs::poll(*client_, ticket); } + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 3000; } + + private: + Json body(const ImageRequest& request, std::string& model) const; + std::shared_ptr client_; +}; + +class HiggsfieldVideoGenerator : public VideoGenerator { + public: + explicit HiggsfieldVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "higgsfield"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"kling-video/v2.5-turbo/pro/text-to-video", "kling-video/v2.5-turbo/pro/image-to-video", + "kling-video/v2.5-turbo/standard/image-to-video", "minimax/hailuo-2.3/standard/text-to-video", + "minimax/hailuo-2.3/standard/image-to-video"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override { return HiggsfieldJobs::poll(*client_, ticket); } + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 5000; } + + private: + Json body(const VideoRequest& request, std::string& model) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/ideogram/calls.cpp b/src/vendors/media/ideogram/calls.cpp new file mode 100644 index 0000000..6fb6264 --- /dev/null +++ b/src/vendors/media/ideogram/calls.cpp @@ -0,0 +1,96 @@ +#include "vendors/media/ideogram/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"image.precise-edit.ideogram-4-5", "POST", "/v2/image/precise-edit/ideogram-4-5", "imagesPreciseEdit_post_precise_edit_image_v2_ideogram45", "multipart", "prompt,image,reference_images,mask,context_window,quality,seed,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.auto", "POST", "/v2/image/generate/auto", "autoModel_post_generate_image_v2_auto", "multipart", "prompt,images,negative_prompt,aspect_ratio,resolution,resolution_tier,magic_prompt,num_images,seed,private,async", "dry_run", "paid"}, + {"image.generate.ideogram-4-5", "POST", "/v2/image/generate/ideogram-4-5", "imagesGenerate_post_generate_image_v2_ideogram45", "multipart", "prompt,magic_prompt,images,mask,size,background,quality,seed,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-4", "POST", "/v2/image/generate/ideogram-4", "imagesGenerate_post_generate_image_v2_ideogram_v4", "json", "prompt,magic_prompt,seed,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-4-custom-model", "POST", "/v2/image/generate/ideogram-4-custom-model", "imagesGenerate_post_generate_image_v2_ideogram_v4_custom_model", "json", "prompt,custom_model_uri,transparent_background,magic_prompt,seed,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-4-transparent", "POST", "/v2/image/generate/ideogram-4-transparent", "imagesGenerate_post_generate_image_v2_ideogram_v4_transparent", "json", "prompt,magic_prompt,seed,num_images,aspect_ratio,output_resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-3", "POST", "/v2/image/generate/ideogram-3", "imagesGenerate_post_generate_image_v2_ideogram_v3", "multipart", "prompt,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_type,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-3-character", "POST", "/v2/image/generate/ideogram-3-character", "imagesGenerate_post_generate_image_v2_ideogram_v3_character", "multipart", "prompt,negative_prompt,character_reference_images,character_reference_mask,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-3-custom-model", "POST", "/v2/image/generate/ideogram-3-custom-model", "imagesGenerate_post_generate_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,transparent_background,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-3-transparent", "POST", "/v2/image/generate/ideogram-3-transparent", "imagesGenerate_post_generate_image_v2_ideogram_v3_transparent", "json", "prompt,negative_prompt,seed,upscale_factor,aspect_ratio,rendering_speed,magic_prompt,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-2a", "POST", "/v2/image/generate/ideogram-2a", "imagesGenerate_post_generate_image_v2_ideogram_v2_a", "json", "prompt,seed,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.ideogram-2", "POST", "/v2/image/generate/ideogram-2", "imagesGenerate_post_generate_image_v2_ideogram_v2", "json", "prompt,negative_prompt,seed,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.gpt-image-2-5-flare", "POST", "/v2/image/generate/gpt-image-2-5-flare", "imagesGenerate_post_generate_image_v2_gpt_image25_flare", "multipart", "prompt,images,mask,background,quality,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"}, + {"image.generate.gpt-image-2-5-sunburst", "POST", "/v2/image/generate/gpt-image-2-5-sunburst", "imagesGenerate_post_generate_image_v2_gpt_image25_sunburst", "multipart", "prompt,images,mask,background,quality,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"}, + {"image.generate.gpt-image-2", "POST", "/v2/image/generate/gpt-image-2", "imagesGenerate_post_generate_image_v2_gpt_image2", "multipart", "prompt,images,mask,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"}, + {"image.generate.nano-banana-2", "POST", "/v2/image/generate/nano-banana-2", "imagesGenerate_post_generate_image_v2_nano_banana2", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"}, + {"image.generate.nano-banana-2-1", "POST", "/v2/image/generate/nano-banana-2-1", "imagesGenerate_post_generate_image_v2_nano_banana21", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"}, + {"image.generate.nano-banana-pro", "POST", "/v2/image/generate/nano-banana-pro", "imagesGenerate_post_generate_image_v2_nano_banana_pro", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"}, + {"image.generate.p-image-ideogram", "POST", "/v2/image/generate/p-image-ideogram", "imagesGenerate_post_generate_image_v2_p_image_ideogram", "json", "prompt,magic_prompt,seed,num_images,quality,resolution,aspect_ratio,custom_width,custom_height,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.generate.z-image", "POST", "/v2/image/generate/z-image", "imagesGenerate_post_generate_image_v2_z_image", "json", "prompt,resolution,num_inference_steps,seed,num_images,async,webhook_url", "dry_run", "paid"}, + {"image.remix.auto", "POST", "/v2/image/remix/auto", "autoModel_post_tool_remix", "multipart", "prompt,image,image_weight,negative_prompt,resolution,aspect_ratio,resolution_tier,magic_prompt,seed,style_preset,color_palette,style_codes,style_type,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.remix.ideogram-4", "POST", "/v2/image/remix/ideogram-4", "imagesRemix_post_remix_image_v2_ideogram_v4", "multipart", "prompt,image,image_weight,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.remix.ideogram-3", "POST", "/v2/image/remix/ideogram-3", "imagesRemix_post_remix_image_v2_ideogram_v3", "multipart", "prompt,image,image_weight,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_type,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.remix.ideogram-3-character", "POST", "/v2/image/remix/ideogram-3-character", "imagesRemix_post_remix_image_v2_ideogram_v3_character", "multipart", "prompt,image,image_weight,negative_prompt,character_reference_images,character_reference_mask,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.remix.ideogram-3-custom-model", "POST", "/v2/image/remix/ideogram-3-custom-model", "imagesRemix_post_remix_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,image,image_weight,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.inpaint.ideogram-3", "POST", "/v2/image/inpaint/ideogram-3", "imagesInpaint_post_inpaint_image_v2_ideogram_v3", "multipart", "prompt,image,mask,magic_prompt,num_images,seed,rendering_speed,style_type,style_preset,style_codes,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.inpaint.ideogram-3-character", "POST", "/v2/image/inpaint/ideogram-3-character", "imagesInpaint_post_inpaint_image_v2_ideogram_v3_character", "multipart", "prompt,image,mask,character_reference_images,character_reference_mask,magic_prompt,num_images,seed,rendering_speed,style_type,style_codes,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"}, + {"image.inpaint.ideogram-3-custom-model", "POST", "/v2/image/inpaint/ideogram-3-custom-model", "imagesInpaint_post_inpaint_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,image,mask,magic_prompt,num_images,seed,rendering_speed,style_preset,style_codes,style_reference_images,enable_copyright_detection,async", "dry_run", "paid"}, + {"image.reframe.ideogram-3", "POST", "/v2/image/reframe/ideogram-3", "reframe_ideogramV3", "multipart", "image,resolution,num_images,seed,rendering_speed,style_preset,color_palette,style_codes,style_reference_images,async", "dry_run", "paid"}, + {"image.reframe.nano-banana-2", "POST", "/v2/image/reframe/nano-banana-2", "reframe_nanoBanana2", "multipart", "image,aspect_ratio,private,num_images,seed", "dry_run", "paid"}, + {"image.upscale.auto", "POST", "/v2/image/upscale/auto", "tools_post_upscale", "multipart", "image,upscale_factor,seed,async,webhook_url", "dry_run", "paid"}, + {"image.upscale.nano-banana-pro", "POST", "/v2/image/upscale/nano-banana-pro", "imagesUpscale_post_upscale_image_nano_banana_pro", "multipart", "image,prompt,resolution_tier,seed,async,webhook_url", "dry_run", "paid"}, + {"image.upscale.topaz-bloom-2", "POST", "/v2/image/upscale/topaz-bloom-2", "imagesUpscale_post_upscale_image_topaz_bloom2", "multipart", "image,upscale_factor,prompt,creativity,color_preservation,seed,async,webhook_url", "dry_run", "paid"}, + {"image.upscale.topaz-redefine", "POST", "/v2/image/upscale/topaz-redefine", "imagesUpscale_post_upscale_image_topaz_redefine", "multipart", "image,upscale_factor,prompt,creativity,autoprompt,texture,detail,detail_strength,sharpen,denoise,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,seed,async,webhook_url", "dry_run", "paid"}, + {"image.upscale.topaz-standard-2", "POST", "/v2/image/upscale/topaz-standard-2", "imagesUpscale_post_upscale_image_topaz_standard_v2", "multipart", "image,upscale_factor,strength,sharpen,denoise,fix_compression,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,async,webhook_url", "dry_run", "paid"}, + {"image.upscale.topaz-text-refine", "POST", "/v2/image/upscale/topaz-text-refine", "imagesUpscale_post_upscale_image_topaz_text_refine", "multipart", "image,upscale_factor,strength,sharpen,denoise,fix_compression,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,seed,async,webhook_url", "dry_run", "paid"}, + {"image.upscale.topaz-wonder-3-5", "POST", "/v2/image/upscale/topaz-wonder-3-5", "imagesUpscale_post_upscale_image_topaz_wonder35", "multipart", "image,upscale_factor,enhancement_strength,seed,async,webhook_url", "dry_run", "paid"}, + {"image.replace-background.ideogram-3", "POST", "/v2/image/replace-background/ideogram-3", "tool_replaceBackgroundIdeogramV3", "multipart", "image,prompt,rendering_speed,num_images,private,webhook_url", "dry_run", "paid"}, + {"image.replace-background.gpt-image-2", "POST", "/v2/image/replace-background/gpt-image-2", "tool_replaceBackground", "multipart", "image,prompt,quality,num_images,private,webhook_url", "dry_run", "paid"}, + {"image.remove-background.ideogram-1", "POST", "/v2/image/remove-background/ideogram-1", "tool_removeBackground", "multipart", "image,private,async", "dry_run", "paid"}, + {"image.remove-object.ideogram-1", "POST", "/v2/image/remove-object/ideogram-1", "objectRemoval_post_remove_object_from_v2_assets", "multipart", "image,mask,seed", "dry_run", "paid"}, + {"image.describe.ideogram-4", "POST", "/v2/image/describe/ideogram-4", "imageDescribe_post_describe_image_ideogram_v4", "multipart", "image,include_bbox,include_style_descriptions,include_tags", "", "paid"}, + {"image.describe.ideogram-3", "POST", "/v2/image/describe/ideogram-3", "imageDescribe_post_describe_image_ideogram_v3", "multipart", "image", "", "paid"}, + {"design.layerize.ideogram-3", "POST", "/v2/design/layerize/ideogram-3", "design_post_layerize_design_ideogram_v3", "multipart", "image,prompt,seed,font_candidate_files,async,webhook_url", "dry_run", "paid"}, + {"video.generate.kling-3-standard-image-to-video", "POST", "/v2/video/generate/kling-3-standard-image-to-video", "videoGenerate_post_generate_video_kling_v3_standard_image_to_video", "multipart", "image,end_image,prompt,negative_prompt,duration,generate_audio,shot_type,cfg_scale,webhook_url", "dry_run", "paid"}, + {"video.generate.kling-3-standard-text-to-video", "POST", "/v2/video/generate/kling-3-standard-text-to-video", "videoGenerate_post_generate_video_kling_v3_standard_text_to_video", "json", "prompt,negative_prompt,aspect_ratio,duration,generate_audio,shot_type,cfg_scale,webhook_url", "dry_run", "paid"}, + {"video.generate.minimax-h3-image-to-video", "POST", "/v2/video/generate/minimax-h3-image-to-video", "videoGenerate_post_generate_video_minimax_h3_image_to_video", "multipart", "image,end_image,prompt,resolution,duration,webhook_url", "dry_run", "paid"}, + {"video.generate.minimax-h3-reference-to-video", "POST", "/v2/video/generate/minimax-h3-reference-to-video", "videoEdit_post_edit_video_minimax_h3_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,prompt_expansion_mode,webhook_url", "dry_run", "paid"}, + {"video.generate.minimax-h3-text-to-video", "POST", "/v2/video/generate/minimax-h3-text-to-video", "videoGenerate_post_generate_video_minimax_h3_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,webhook_url", "dry_run", "paid"}, + {"video.generate.seedance-2-5-image-to-video", "POST", "/v2/video/generate/seedance-2-5-image-to-video", "videoGenerate_post_generate_video_seed_dance25_image_to_video", "multipart", "image,end_image,prompt,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"}, + {"video.generate.seedance-2-5-reference-to-video", "POST", "/v2/video/generate/seedance-2-5-reference-to-video", "videoGenerate_post_generate_video_seed_dance25_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"}, + {"video.generate.seedance-2-5-text-to-video", "POST", "/v2/video/generate/seedance-2-5-text-to-video", "videoGenerate_post_generate_video_seed_dance25_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"}, + {"video.generate.seedance-2-image-to-video", "POST", "/v2/video/generate/seedance-2-image-to-video", "videoGenerate_post_generate_video_seed_dance2_image_to_video", "multipart", "image,end_image,prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"}, + {"video.generate.seedance-2-reference-to-video", "POST", "/v2/video/generate/seedance-2-reference-to-video", "videoGenerate_post_generate_video_seed_dance2_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"}, + {"video.generate.seedance-2-text-to-video", "POST", "/v2/video/generate/seedance-2-text-to-video", "videoGenerate_post_generate_video_seed_dance2_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"}, + {"svg.generate.ideogram-vector-1", "POST", "/v2/svg/generate/ideogram-vector-1", "svgGenerate_post_generate_svg", "multipart", "prompt,image_asset_identifiers,images,aspect_ratio,quality,num_images,seed,private,target_collection_id,webhook_url", "dry_run", "paid"}, + {"svg.vectorize.ideogram-vector-1", "POST", "/v2/svg/vectorize/ideogram-vector-1", "svgGenerate_post_vectorizer", "multipart", "image,image_asset_identifier,mode,private,target_collection_id,webhook_url", "dry_run", "paid"}, + {"tool.ad-localizer", "POST", "/v2/tool/ad-localizer", "editWorkflow_post_ad_localizer", "multipart", "image,language,exact_copy,prompt,quality,seed,num_images,webhook_url", "dry_run", "paid"}, + {"tool.ad-resizer", "POST", "/v2/tool/ad-resizer", "editWorkflow_post_ad_resizer", "multipart", "image,resolution,platform,prompt,quality,num_images,webhook_url", "dry_run", "paid"}, + {"tool.ad-variations", "POST", "/v2/tool/ad-variations", "editWorkflow_post_ad_variations", "multipart", "image,variation_type,prompt,quality,num_images,webhook_url", "", "paid"}, + {"tool.colorways", "POST", "/v2/tool/colorways", "editWorkflow_post_colorways", "multipart", "image,masks,colors,aspect_ratio,quality,webhook_url", "dry_run", "paid"}, + {"tool.ghost-mannequin", "POST", "/v2/tool/ghost-mannequin", "editWorkflow_post_ghost_mannequin", "multipart", "garment_images,front_image,back_image,left_image,right_image,top_image,bottom_image,output_reference_asset_identifier,output_reference_image,view,instruction,metadata,aspect_ratio,quality,seed,webhook_url", "dry_run", "paid"}, + {"tool.material-swap", "POST", "/v2/tool/material-swap", "editWorkflow_post_material_swap", "multipart", "image,masks,materials,aspect_ratio,quality,webhook_url", "dry_run", "paid"}, + {"tool.model-pose-variants", "POST", "/v2/tool/model-pose-variants", "editWorkflow_post_model_pose_variants", "multipart", "source_image,instruction,aspect_ratio,quality,seed,num_images,webhook_url", "", "paid"}, + {"tool.sketch-to-render", "POST", "/v2/tool/sketch-to-render", "editWorkflow_post_sketch_to_render", "multipart", "sketch_image,instruction,aspect_ratio,quality,seed,num_images,webhook_url", "", "paid"}, + {"generations.id", "GET", "/v2/generations/{generation_id}", "generations_get_generation_v2", "none", "", "", "read"}, + {"account.api-keys", "GET", "/v2/account/api-keys", "account_get_account_api_keys", "none", "", "", "read"}, + {"account.invoices", "GET", "/v2/account/invoices", "account_get_account_invoices", "none", "", "", "read"}, + {"account.usage", "GET", "/v2/account/usage", "account_get_account_usage", "none", "", "start_time,end_time,bucket_width,sources", "read"}, + {"image.replace-background.ideogram-4-5", "POST", "/v2/image/replace-background/ideogram-4-5", "tool_replaceBackgroundIdeogram45", "multipart", "image,background_reference,num_images,private,webhook_url", "dry_run", "paid"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"GenerateImageIdeogramV3Response", "generation_id,data,seed"}, + {"GenerationResponse", "generation_id,status,created,response_type,usage_cost_usd_micros,failure_reason,data"}, +}; +// FIELDS-END + +const CallTable& IdeogramCalls::table() { + static const CallTable t("ideogram", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t IdeogramCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* IdeogramCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/ideogram/calls.h b/src/vendors/media/ideogram/calls.h new file mode 100644 index 0000000..5e1f8ce --- /dev/null +++ b/src/vendors/media/ideogram/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct IdeogramCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/ideogram/ideogram.cpp b/src/vendors/media/ideogram/ideogram.cpp new file mode 100644 index 0000000..c94a7a0 --- /dev/null +++ b/src/vendors/media/ideogram/ideogram.cpp @@ -0,0 +1,174 @@ +#include "vendors/media/ideogram/ideogram.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs args(const Json& fields, const std::vector& files, bool dry_run) { + CallArgs a; + a.body = fields; + a.files = files; + if (dry_run) a.query["dry_run"] = "true"; + return a; +} + +// Image routes take `async` in the body; video routes are always asynchronous. +bool is_async(const Json& fields) { + const Json* a = fields.find("async"); + return a && a->is_bool() && a->as_bool(); +} + +JobStatus status_from(const Json& g, MediaKind kind) { + JobStatus s; + s.raw_state = AdapterSupport::str(g, "status"); + if (s.raw_state == "completed") { + s.state = JobState::kSucceeded; + if (const Json* d = g.find("data")) s.assets = OutputScanner::scan(*d, kind); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "completed without a URL (images that fail the safety check have an empty url)"; + } + } else if (s.raw_state == "failed") { + s.state = JobState::kFailed; + s.error = AdapterSupport::str(g, "failure_reason"); + } else { + s.state = s.raw_state == "pending" ? JobState::kRunning : JobState::kQueued; + } + return s; +} + +} // namespace + +PreparedCall IdeogramClient::plan(const std::string& key, const Json& fields, const std::vector& files, + bool dry_run) const { + return prepare(key, args(fields, files, dry_run)); +} + +Json IdeogramClient::generate(const std::string& key, const Json& fields, const std::vector& files, + bool confirm) { + return call_json(key, args(fields, files, false), confirm); +} + +Json IdeogramClient::quote(const std::string& key, const Json& fields, const std::vector& files) { + PreparedCall c = plan(key, fields, files, true); + c.effect = Effect::kRead; // dry_run=true is defined by the spec as not billed + return parse_json("ideogram", execute(c, false)); +} + +Json IdeogramClient::generation(const std::string& id) { + CallArgs a; + a.path["generation_id"] = id; + return call_json("generations.id", a); +} + +Json IdeogramClient::usage(const std::map& query) { + CallArgs a; + a.query = query; + return call_json("account.usage", a); +} +Json IdeogramClient::api_keys() { return call_json("account.api-keys"); } +Json IdeogramClient::invoices() { return call_json("account.invoices"); } + +std::string IdeogramImageGenerator::key_for(const std::string& model) { + std::string m = model.empty() ? "ideogram-3" : model; + return "image.generate." + m; +} + +Json IdeogramImageGenerator::fields(const ImageRequest& r, std::string& key) const { + r.validate(); + key = key_for(r.model); + Json f = Json::object(); + AdapterSupport::put(f, "prompt", r.prompt); + AdapterSupport::put(f, "negative_prompt", r.negative_prompt); + AdapterSupport::put_seed(f, "seed", r.seed > 2147483647L ? 2147483647L : r.seed); + if (r.count > 1) f.set("num_images", Json(r.count)); + AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio); + if (r.width && r.height) f.set("resolution", Json(std::to_string(r.width) + "x" + std::to_string(r.height))); + if (!r.source_url.empty()) + throw llm::ApiError::invalid("ideogram", "source images are files: use IdeogramClient::generate with a FilePart and an edit row"); + AdapterSupport::merge_extra(f, r.extra); + return f; +} + +Json IdeogramImageGenerator::plan(const ImageRequest& request) const { + std::string key; + Json f = fields(request, key); + return client_->plan(key, f, {}).summary(); +} + +JobTicket IdeogramImageGenerator::submit(const ImageRequest& request, bool confirm) { + std::string key; + Json f = fields(request, key); + AdapterSupport::need_confirm(confirm, "ideogram", "generate"); + Json out = client_->generate(key, f, {}, true); + JobTicket t; + t.provider = "ideogram"; + t.model = request.model.empty() ? "ideogram-3" : request.model; + t.kind = MediaKind::kImage; + t.id = AdapterSupport::str(out, "generation_id"); + if (is_async(f)) { + if (t.id.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no generation_id in the async reply"); + return t; + } + if (t.id.empty()) t.id = "immediate"; + t.immediate = true; + if (const Json* d = out.find("data")) t.assets = OutputScanner::scan(*d, MediaKind::kImage); + if (t.assets.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no image URL in the reply (safety check?)"); + return t; +} + +JobStatus IdeogramImageGenerator::poll(const JobTicket& ticket) { + if (ticket.immediate) { + JobStatus s; + s.state = JobState::kSucceeded; + s.raw_state = "immediate"; + s.assets = ticket.assets; + return s; + } + return status_from(client_->generation(ticket.id), MediaKind::kImage); +} + +Json IdeogramVideoGenerator::fields(const VideoRequest& r, std::string& key) const { + r.validate(); + if (r.model.empty()) throw llm::ApiError::invalid("ideogram", "model is required (for example kling-3-standard-text-to-video)"); + key = "video.generate." + r.model; + Json f = Json::object(); + AdapterSupport::put(f, "prompt", r.prompt); + AdapterSupport::put(f, "negative_prompt", r.negative_prompt); + AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put(f, "resolution", r.resolution); + AdapterSupport::put_int(f, "duration", r.duration_seconds); + if (!r.image_url.empty()) + throw llm::ApiError::invalid("ideogram", "start frames are files: use IdeogramClient::generate with an image-to-video row"); + AdapterSupport::merge_extra(f, r.extra); + return f; +} + +Json IdeogramVideoGenerator::plan(const VideoRequest& request) const { + std::string key; + Json f = fields(request, key); + return client_->plan(key, f, {}).summary(); +} + +JobTicket IdeogramVideoGenerator::submit(const VideoRequest& request, bool confirm) { + std::string key; + Json f = fields(request, key); + AdapterSupport::need_confirm(confirm, "ideogram", "generate"); + Json out = client_->generate(key, f, {}, true); + JobTicket t; + t.provider = "ideogram"; + t.model = request.model; + t.kind = MediaKind::kVideo; + t.id = AdapterSupport::str(out, "generation_id"); + if (t.id.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no generation_id in the reply"); + return t; +} + +JobStatus IdeogramVideoGenerator::poll(const JobTicket& ticket) { + return status_from(client_->generation(ticket.id), MediaKind::kVideo); +} + +} // namespace molo::media diff --git a/src/vendors/media/ideogram/ideogram.h b/src/vendors/media/ideogram/ideogram.h new file mode 100644 index 0000000..cd6badb --- /dev/null +++ b/src/vendors/media/ideogram/ideogram.h @@ -0,0 +1,84 @@ +#pragma once + +// Ideogram API v2: image generate / remix / inpaint / reframe / upscale / +// background tools, SVG, video models, ad tools, generation polling, account. +// Auth is the Api-Key header. Every POST accepts dry_run=true, which the spec +// defines as "validated and priced but not run: nothing generated, stored or +// billed": quote() uses it, so a price check is free. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/ideogram/calls.h" + +namespace molo::media { + +class IdeogramClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.ideogram.ai"; + static constexpr const char* kKeyEnv = "IDEOGRAM_API_KEY"; + + explicit IdeogramClient(std::shared_ptr client) + : MediaService(std::move(client), IdeogramCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::header("ideogram", kHost, kKeyEnv, "Api-Key"); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // Paid: any POST row of the table (image.*, video.*, svg.*, tool.*). + Json generate(const std::string& key, const Json& fields, const std::vector& files, bool confirm); + PreparedCall plan(const std::string& key, const Json& fields, const std::vector& files, + bool dry_run = false) const; + // Free price quote (dry_run=true): the only non-read call that needs no confirm. + Json quote(const std::string& key, const Json& fields, const std::vector& files); + Json generation(const std::string& generation_id); + Json usage(const std::map& query); + Json api_keys(); + Json invoices(); +}; + +class IdeogramImageGenerator : public ImageGenerator { + public: + explicit IdeogramImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "ideogram"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"ideogram-4-5", "ideogram-4", "ideogram-3", "ideogram-3-transparent", "ideogram-2a", "ideogram-2", "auto"}; + } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the API + int suggested_poll_ms() const override { return 2000; } + + static std::string key_for(const std::string& model); + + private: + Json fields(const ImageRequest& request, std::string& key) const; + std::shared_ptr client_; +}; + +class IdeogramVideoGenerator : public VideoGenerator { + public: + explicit IdeogramVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "ideogram"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"kling-3-standard-text-to-video", "minimax-h3-text-to-video", "seedance-2-text-to-video", + "seedance-2-5-text-to-video"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } + int suggested_poll_ms() const override { return 4000; } + + private: + Json fields(const VideoRequest& request, std::string& key) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/kling/calls.cpp b/src/vendors/media/kling/calls.cpp new file mode 100644 index 0000000..75f6043 --- /dev/null +++ b/src/vendors/media/kling/calls.cpp @@ -0,0 +1,30 @@ +#include "vendors/media/kling/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"text2video.create", "POST", "/v1/videos/text2video", "", "json", "model_name,prompt,negative_prompt,cfg_scale,mode,aspect_ratio,duration,callback_url,external_task_id", "", "paid"}, + {"text2video.get", "GET", "/v1/videos/text2video/{id}", "", "none", "", "", "read"}, + {"image2video.create", "POST", "/v1/videos/image2video", "", "json", "model_name,image,image_tail,prompt,negative_prompt,cfg_scale,mode,duration,callback_url,external_task_id", "", "paid"}, + {"image2video.get", "GET", "/v1/videos/image2video/{id}", "", "none", "", "", "read"}, + {"images.create", "POST", "/v1/images/generations", "", "json", "model_name,prompt,negative_prompt,image,image_reference,image_fidelity,n,aspect_ratio,callback_url", "", "paid"}, + {"images.get", "GET", "/v1/images/generations/{id}", "", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { +}; +// FIELDS-END + +const CallTable& KlingCalls::table() { + static const CallTable t("kling", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t KlingCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* KlingCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/kling/calls.h b/src/vendors/media/kling/calls.h new file mode 100644 index 0000000..a218aca --- /dev/null +++ b/src/vendors/media/kling/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct KlingCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/kling/kling.cpp b/src/vendors/media/kling/kling.cpp new file mode 100644 index 0000000..19156d2 --- /dev/null +++ b/src/vendors/media/kling/kling.cpp @@ -0,0 +1,156 @@ +#include "vendors/media/kling/kling.h" + +#include +#include + +#include "security/crypto/Base64.h" +#include "security/crypto/Hmac.h" +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +std::shared_ptr build_client(const std::string& access, const std::string& secret, long now, + const std::function()>& factory) { + std::string token = KlingClient::credentials_present(access, secret) ? KlingJwt::make(access, secret, now) : ""; + if (factory) return ClientFactory::with(factory(), KlingClient::config(), token); + return ClientFactory::real(KlingClient::config(), token); +} + +CallArgs with_body(const Json& body) { + CallArgs a; + a.body = body; + return a; +} + +} // namespace + +std::string KlingJwt::make(const std::string& access_key, const std::string& secret_key, long now) { + Json header = Json::object(); + header.set("alg", Json("HS256")); + header.set("typ", Json("JWT")); + Json payload = Json::object(); + payload.set("iss", Json(access_key)); + payload.set("exp", Json(static_cast(now + kTtlSeconds))); + payload.set("nbf", Json(static_cast(now - 5))); + std::string signing = security::crypto::Base64::encode_url(header.dump()) + "." + + security::crypto::Base64::encode_url(payload.dump()); + auto mac = security::crypto::Hmac::sha256(secret_key, signing); + return signing + "." + security::crypto::Base64::encode_url(reinterpret_cast(mac.data()), mac.size()); +} + +KlingClient::KlingClient(std::string access_key, std::string secret_key, + std::function()> transport) + : MediaService(build_client(access_key, secret_key, static_cast(std::time(nullptr)), transport), + KlingCalls::table()), + access_(std::move(access_key)), + secret_(std::move(secret_key)), + transport_(std::move(transport)), + minted_at_(static_cast(std::time(nullptr))) {} + +std::shared_ptr KlingClient::from_env() { + return std::make_shared(ClientFactory::env({kAccessEnv}), ClientFactory::env({kSecretEnv})); +} + +void KlingClient::renew() { + long now = static_cast(std::time(nullptr)); + if (!credentials_present(access_, secret_) || now - minted_at_ < KlingJwt::kRenewAfterSeconds) return; + client_ = build_client(access_, secret_, now, transport_); + minted_at_ = now; +} + +void KlingClient::check_envelope(const Json& doc) { + const Json* code = doc.is_object() ? doc.find("code") : nullptr; + if (!code || !code->is_number() || code->as_int() == 0) return; + throw llm::ApiError("kling", 400, std::to_string(code->as_int()), AdapterSupport::str(doc, "message"), + AdapterSupport::str(doc, "request_id")); +} + +PreparedCall KlingClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); } + +Json KlingClient::submit(const std::string& key, const Json& body, bool confirm) { + renew(); + Json out = call_json(key, with_body(body), confirm); + check_envelope(out); + return out; +} + +Json KlingClient::get(const std::string& key, const std::string& task_id) { + renew(); + CallArgs a; + a.path["id"] = task_id; + Json out = call_json(key, a); + check_envelope(out); + return out; +} + +JobState KlingVideoGenerator::state_of(const std::string& status) { + if (status == "succeed") return JobState::kSucceeded; + if (status == "failed") return JobState::kFailed; + if (status == "processing") return JobState::kRunning; + return JobState::kQueued; // submitted +} + +Json KlingVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + Json b = Json::object(); + b.set("model_name", Json(r.model.empty() ? "kling-v1-6" : r.model)); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "negative_prompt", r.negative_prompt); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds))); + AdapterSupport::put(b, "image", r.image_url); + AdapterSupport::put(b, "image_tail", r.end_image_url); + if (r.seed >= 0) throw llm::ApiError::invalid("kling", "no seed parameter is documented"); + if (!r.resolution.empty()) throw llm::ApiError::invalid("kling", "resolution is set through mode (std / pro): pass mode in extra"); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json KlingVideoGenerator::plan(const VideoRequest& request) const { + return client_->plan(request.image_url.empty() ? "text2video.create" : "image2video.create", body(request)).summary(); +} + +JobTicket KlingVideoGenerator::submit(const VideoRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "kling", "generate"); + const bool image = !request.image_url.empty(); + Json out = client_->submit(image ? "image2video.create" : "text2video.create", b, true); + const Json* data = out.find("data"); + JobTicket t; + t.provider = "kling"; + t.model = b.find("model_name")->as_string(); + t.kind = MediaKind::kVideo; + t.id = data ? AdapterSupport::str(*data, "task_id") : ""; + if (t.id.empty()) throw llm::ApiError("kling", 502, "BAD_RESPONSE", "no data.task_id in the reply"); + t.meta = Json::object(); + t.meta.set("route", Json(image ? "image2video" : "text2video")); + return t; +} + +JobStatus KlingVideoGenerator::poll(const JobTicket& ticket) { + std::string route = AdapterSupport::str(ticket.meta, "route"); + if (route != "image2video" && route != "text2video") throw llm::ApiError::invalid("kling", "ticket has no route"); + Json out = client_->get(route + ".get", ticket.id); + const Json* data = out.find("data"); + JobStatus s; + if (!data) { + s.state = JobState::kQueued; + return s; + } + s.raw_state = AdapterSupport::str(*data, "task_status"); + s.state = state_of(s.raw_state); + if (s.state == JobState::kFailed) s.error = AdapterSupport::str(*data, "task_status_msg"); + if (s.state == JobState::kSucceeded) { + if (const Json* r = data->find("task_result")) s.assets = OutputScanner::scan(*r, MediaKind::kVideo); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "succeed without videos"; + } + } + return s; +} + +} // namespace molo::media diff --git a/src/vendors/media/kling/kling.h b/src/vendors/media/kling/kling.h new file mode 100644 index 0000000..99e7050 --- /dev/null +++ b/src/vendors/media/kling/kling.h @@ -0,0 +1,80 @@ +#pragma once + +// Kling AI (Kuaishou) API. Kling publishes its developer docs as a JavaScript +// app with no machine-readable spec, so this adapter is built from community +// sources and live probes only: see SPEC "Unverified". Auth is a short-lived +// HS256 JWT built from KLING_ACCESS_KEY (iss) and KLING_SECRET_KEY. Hosts +// api-singapore.klingai.com and api.klingai.com both answered the documented +// path with `{"code":1001,"message":"Authorization is empty"}` unauthenticated. + +#include +#include + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/kling/calls.h" + +namespace molo::media { + +class KlingJwt { + public: + static constexpr long kTtlSeconds = 1800; // token lifetime + static constexpr long kRenewAfterSeconds = 1500; // the client rebuilds before this + // header {"alg":"HS256","typ":"JWT"}, payload {iss, exp, nbf}; base64url, no padding. + static std::string make(const std::string& access_key, const std::string& secret_key, long now_seconds); +}; + +class KlingClient : public MediaService { + public: + static constexpr const char* kHost = "https://api-singapore.klingai.com"; + static constexpr const char* kAccessEnv = "KLING_ACCESS_KEY"; + static constexpr const char* kSecretEnv = "KLING_SECRET_KEY"; + + // The token is fixed per ApiClient, so a KlingClient holds a factory and + // renews the ApiClient when the token nears expiry. + KlingClient(std::string access_key, std::string secret_key, + std::function()> transport = nullptr); + static llm::ApiClientConfig config() { return ClientFactory::bearer("kling", kHost, kAccessEnv); } + static std::shared_ptr from_env(); + static bool credentials_present(const std::string& access, const std::string& secret) { + return !access.empty() && !secret.empty(); + } + + // Paid submits: text2video.create, image2video.create, images.create. + Json submit(const std::string& key, const Json& body, bool confirm); + PreparedCall plan(const std::string& key, const Json& body) const; + Json get(const std::string& key, const std::string& task_id); // *.get rows + // Throws ApiError when the envelope's code is not 0. + static void check_envelope(const Json& doc); + + private: + void renew(); + std::string access_, secret_; + std::function()> transport_; + long minted_at_ = 0; +}; + +class KlingVideoGenerator : public VideoGenerator { + public: + explicit KlingVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "kling"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"kling-v1", "kling-v1-6", "kling-v2-master", "kling-v2-1", "kling-v2-5-turbo"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } // not documented + int suggested_poll_ms() const override { return 6000; } + + // submitted / processing / succeed / failed (community sources). + static JobState state_of(const std::string& status); + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/leonardo/calls.cpp b/src/vendors/media/leonardo/calls.cpp new file mode 100644 index 0000000..4f32124 --- /dev/null +++ b/src/vendors/media/leonardo/calls.cpp @@ -0,0 +1,30 @@ +#include "vendors/media/leonardo/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"v2.generations.create", "POST", "/v2/generations", "createGeneration", "json", "model,parameters,public", "", "paid"}, + {"v2.generations.sync", "POST", "/v2/generationssync", "createSyncGeneration", "json", "model,parameters,public", "", "paid"}, + {"v2.models", "GET", "/v2/models", "getModels", "none", "", "", "read"}, + {"v1.generations.create", "POST", "/v1/generations", "", "json", "prompt,modelId,width,height,num_images,alchemy,ultra,contrast,styleUUID,negative_prompt,seed,public", "", "paid"}, + {"v1.generations.get", "GET", "/v1/generations/{id}", "getGenerationById", "none", "", "", "read"}, + {"v1.me", "GET", "/v1/me", "getUserSelf", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { +}; +// FIELDS-END + +const CallTable& LeonardoCalls::table() { + static const CallTable t("leonardo", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t LeonardoCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* LeonardoCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/leonardo/calls.h b/src/vendors/media/leonardo/calls.h new file mode 100644 index 0000000..5884154 --- /dev/null +++ b/src/vendors/media/leonardo/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct LeonardoCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/leonardo/leonardo.cpp b/src/vendors/media/leonardo/leonardo.cpp new file mode 100644 index 0000000..30f648f --- /dev/null +++ b/src/vendors/media/leonardo/leonardo.cpp @@ -0,0 +1,129 @@ +#include "vendors/media/leonardo/leonardo.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs with_body(const Json& body) { + CallArgs a; + a.body = body; + return a; +} + +} // namespace + +PreparedCall LeonardoClient::plan(const Json& body) const { return prepare("v2.generations.create", with_body(body)); } +Json LeonardoClient::create(const Json& body, bool confirm) { + return call_json("v2.generations.create", with_body(body), confirm); +} +Json LeonardoClient::create_sync(const Json& body, bool confirm) { + return call_json("v2.generations.sync", with_body(body), confirm); +} +Json LeonardoClient::models() { return call_json("v2.models"); } +Json LeonardoClient::generation(const std::string& id) { + CallArgs a; + a.path["id"] = id; + return call_json("v1.generations.get", a); +} +Json LeonardoClient::me() { return call_json("v1.me"); } + +Json LeonardoJobs::body(const std::string& model, const Json& parameters, bool is_public) { + Json b = Json::object(); + b.set("model", Json(model)); + b.set("parameters", parameters); + b.set("public", Json(is_public)); // never publish to the community feed by default + return b; +} + +JobTicket LeonardoJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) { + JobTicket t; + t.provider = "leonardo"; + t.model = model; + t.kind = kind; + t.id = AdapterSupport::str(reply, "generationId"); + if (t.id.empty()) throw llm::ApiError("leonardo", 502, "BAD_RESPONSE", "no generationId in the reply"); + t.meta = Json::object(); + AdapterSupport::put(t.meta, "apiCreditCost", AdapterSupport::str(reply, "apiCreditCost")); + return t; +} + +// v1 GET /generations/{id}: {"generations_by_pk": {"status": "PENDING|COMPLETE|FAILED", +// "generated_images": [{"url": ...}], ...}} (status words from the FAQ; shape unverified). +JobStatus LeonardoJobs::status_of(const Json& doc, MediaKind kind) { + const Json* g = doc.find("generations_by_pk"); + const Json& gen = g && g->is_object() ? *g : doc; + JobStatus s; + s.raw_state = AdapterSupport::str(gen, "status"); + if (s.raw_state == "COMPLETE") { + s.state = JobState::kSucceeded; + for (const char* key : {"generated_images", "generated_videos", "generated_assets"}) + if (const Json* a = gen.find(key)) { + auto found = OutputScanner::scan(*a, kind); + s.assets.insert(s.assets.end(), found.begin(), found.end()); + } + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "COMPLETE without generated images"; + } + } else if (s.raw_state == "FAILED") { + s.state = JobState::kFailed; + s.error = "FAILED"; + } else { + s.state = JobState::kRunning; // PENDING + } + return s; +} + +JobStatus LeonardoJobs::poll(LeonardoClient& client, const JobTicket& ticket) { + return status_of(client.generation(ticket.id), ticket.kind); +} + +Json LeonardoImageGenerator::body(const ImageRequest& r) const { + r.validate(); + if (r.model.empty()) throw llm::ApiError::invalid("leonardo", "model is required (see LeonardoClient::models)"); + Json p = Json::object(); + AdapterSupport::put(p, "prompt", r.prompt); + AdapterSupport::put_int(p, "width", r.width); + AdapterSupport::put_int(p, "height", r.height); + p.set("quantity", Json(r.count)); + AdapterSupport::put_seed(p, "seed", r.seed); + if (!r.aspect_ratio.empty() || !r.negative_prompt.empty() || !r.source_url.empty()) + throw llm::ApiError::invalid("leonardo", "aspect_ratio / negative_prompt / source images are model-specific: pass them in extra (parameters)"); + AdapterSupport::merge_extra(p, r.extra); + return LeonardoJobs::body(r.model, p, false); +} + +Json LeonardoImageGenerator::plan(const ImageRequest& request) const { return client_->plan(body(request)).summary(); } + +JobTicket LeonardoImageGenerator::submit(const ImageRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "leonardo", "generate"); + return LeonardoJobs::ticket_from(client_->create(b, true), request.model, MediaKind::kImage); +} + +Json LeonardoVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + if (r.model.empty()) throw llm::ApiError::invalid("leonardo", "model is required (see LeonardoClient::models)"); + Json p = Json::object(); + AdapterSupport::put(p, "prompt", r.prompt); + AdapterSupport::put_int(p, "duration", r.duration_seconds); + AdapterSupport::put(p, "resolution", r.resolution); + AdapterSupport::put_seed(p, "seed", r.seed); + if (!r.aspect_ratio.empty() || !r.negative_prompt.empty() || !r.image_url.empty() || !r.end_image_url.empty()) + throw llm::ApiError::invalid("leonardo", "frames / aspect_ratio / negative_prompt are model-specific: pass them in extra (parameters)"); + AdapterSupport::merge_extra(p, r.extra); + return LeonardoJobs::body(r.model, p, false); +} + +Json LeonardoVideoGenerator::plan(const VideoRequest& request) const { return client_->plan(body(request)).summary(); } + +JobTicket LeonardoVideoGenerator::submit(const VideoRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "leonardo", "generate"); + return LeonardoJobs::ticket_from(client_->create(b, true), request.model, MediaKind::kVideo); +} + +} // namespace molo::media diff --git a/src/vendors/media/leonardo/leonardo.h b/src/vendors/media/leonardo/leonardo.h new file mode 100644 index 0000000..407b420 --- /dev/null +++ b/src/vendors/media/leonardo/leonardo.h @@ -0,0 +1,89 @@ +#pragma once + +// Leonardo.Ai production API. v2 (https://cloud.leonardo.ai/api/rest/v2): +// POST /generations (async, any image / video / audio / 3D model by id), +// POST /generationssync, GET /models. Polling uses the documented v1 +// GET /generations/{id} (the v1 reference pages are gone from the docs; the +// shape is described in prose only, see SPEC Unverified). Bearer auth with +// LEONARDO_API_KEY. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/leonardo/calls.h" + +namespace molo::media { + +class LeonardoClient : public MediaService { + public: + static constexpr const char* kHost = "https://cloud.leonardo.ai/api/rest"; + static constexpr const char* kKeyEnv = "LEONARDO_API_KEY"; + + explicit LeonardoClient(std::shared_ptr client) + : MediaService(std::move(client), LeonardoCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::bearer("leonardo", kHost, kKeyEnv); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // body = {"model": "", "parameters": {...}, "public": false} + Json create(const Json& body, bool confirm); + PreparedCall plan(const Json& body) const; + Json create_sync(const Json& body, bool confirm); // up to ~27 s, finished result in the reply + Json models(); // v2, parameter specs per model + Json generation(const std::string& id); // v1 + Json me(); // v1, account and token balance +}; + +// Shared by the image and video adapters (the API does not tell them apart). +class LeonardoJobs { + public: + static Json body(const std::string& model, const Json& parameters, bool is_public); + static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind); + static JobStatus poll(LeonardoClient& client, const JobTicket& ticket); + static JobStatus status_of(const Json& generation, MediaKind kind); +}; + +class LeonardoImageGenerator : public ImageGenerator { + public: + explicit LeonardoImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "leonardo"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"flux-pro-2.0", "flux-kontext-pro", "phoenix-v1.0", "lucid-origin", "ideogram-v3.0", "gpt-image-2", + "nano-banana-2", "seedream-4.5"}; + } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override { return LeonardoJobs::poll(*client_, ticket); } + bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the v2 spec + int suggested_poll_ms() const override { return 3000; } + + private: + Json body(const ImageRequest& request) const; + std::shared_ptr client_; +}; + +class LeonardoVideoGenerator : public VideoGenerator { + public: + explicit LeonardoVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "leonardo"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"motion_2.0", "veo-3.1-fast-generate-001", "kling-3.0", "hailuo-03", "seedance-2.0", "wan-2.7"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override { return LeonardoJobs::poll(*client_, ticket); } + bool cancel(const JobTicket&, bool) override { return false; } + int suggested_poll_ms() const override { return 5000; } + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/luma/calls.cpp b/src/vendors/media/luma/calls.cpp new file mode 100644 index 0000000..5f2931b --- /dev/null +++ b/src/vendors/media/luma/calls.cpp @@ -0,0 +1,39 @@ +#include "vendors/media/luma/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"generations.create_video", "POST", "/generations/video", "", "json", "model,aspect_ratio,callback_url,concepts,duration,generation_type,keyframes,loop,prompt,resolution", "", "paid"}, + {"generations.create_image", "POST", "/generations/image", "", "json", "model,aspect_ratio,callback_url,character_ref,format,generation_type,image_ref,modify_image_ref,prompt,style_ref,sync,sync_timeout", "", "paid"}, + {"generations.reframe_image", "POST", "/generations/image/reframe", "", "json", "aspect_ratio,generation_type,media,model,callback_url,format,grid_position_x,grid_position_y,prompt,resized_height,resized_width,x_end,x_start,y_end,y_start", "", "paid"}, + {"generations.reframe_video", "POST", "/generations/video/reframe", "", "json", "aspect_ratio,generation_type,media,model,callback_url,first_frame,grid_position_x,grid_position_y,prompt,resized_height,resized_width,x_end,x_start,y_end,y_start", "", "paid"}, + {"generations.modify_video", "POST", "/generations/video/modify", "", "json", "generation_type,media,mode,model,callback_url,first_frame,prompt", "", "paid"}, + {"generations.upscale", "POST", "/generations/{id}/upscale", "", "json", "callback_url,generation_type,resolution", "", "paid"}, + {"generations.audio", "POST", "/generations/{id}/audio", "", "json", "callback_url,generation_type,negative_prompt,prompt", "", "paid"}, + {"generations.get", "GET", "/generations/{id}", "", "none", "", "", "read"}, + {"generations.list", "GET", "/generations", "", "none", "", "limit,offset", "read"}, + {"generations.delete", "DELETE", "/generations/{id}", "", "none", "", "", "delete"}, + {"concepts.list", "GET", "/generations/concepts/list", "", "none", "", "", "read"}, + {"credits.get", "GET", "/credits", "", "none", "", "", "read"}, + {"ping", "GET", "/ping", "", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"Generation", "id,state,failure_reason,created_at,assets,generation_type,model,request,image,video,progress_video"}, + {"CreditGetResponse", "credit_balance"}, +}; +// FIELDS-END + +const CallTable& LumaCalls::table() { + static const CallTable t("luma", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t LumaCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* LumaCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/luma/calls.h b/src/vendors/media/luma/calls.h new file mode 100644 index 0000000..9bc67d1 --- /dev/null +++ b/src/vendors/media/luma/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct LumaCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/luma/luma.cpp b/src/vendors/media/luma/luma.cpp new file mode 100644 index 0000000..107513a --- /dev/null +++ b/src/vendors/media/luma/luma.cpp @@ -0,0 +1,174 @@ +#include "vendors/media/luma/luma.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs with_body(const Json& body) { + CallArgs a; + a.body = body; + return a; +} + +CallArgs by_id(const std::string& id, const Json& body = Json()) { + CallArgs a; + a.path["id"] = id; + a.body = body; + return a; +} + +} // namespace + +PreparedCall LumaClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); } +Json LumaClient::create(const std::string& key, const Json& body, bool confirm) { + return call_json(key, with_body(body), confirm); +} +Json LumaClient::upscale(const std::string& id, const Json& body, bool confirm) { + return call_json("generations.upscale", by_id(id, body), confirm); +} +Json LumaClient::add_audio(const std::string& id, const Json& body, bool confirm) { + return call_json("generations.audio", by_id(id, body), confirm); +} +Json LumaClient::generation(const std::string& id) { return call_json("generations.get", by_id(id)); } + +Json LumaClient::list(int limit, int offset) { + CallArgs a; + if (limit > 0) a.query["limit"] = std::to_string(limit); + if (offset > 0) a.query["offset"] = std::to_string(offset); + return call_json("generations.list", a); +} + +Json LumaClient::remove(const std::string& id, bool confirm) { return call_json("generations.delete", by_id(id), confirm); } +Json LumaClient::credits() { return call_json("credits.get"); } +Json LumaClient::concepts() { return call_json("concepts.list"); } +Json LumaClient::ping() { return call_json("ping"); } + +JobTicket LumaJobs::ticket_from(const Json& g, const std::string& model, MediaKind kind) { + JobTicket t; + t.provider = "luma"; + t.model = model; + t.kind = kind; + t.id = AdapterSupport::str(g, "id"); + if (t.id.empty()) throw llm::ApiError("luma", 502, "BAD_RESPONSE", "no id in the generation"); + return t; +} + +JobStatus LumaJobs::status_of(const Json& g, MediaKind kind) { + JobStatus s; + s.raw_state = AdapterSupport::str(g, "state"); + if (s.raw_state == "completed") { + s.state = JobState::kSucceeded; + if (const Json* a = g.find("assets")) s.assets = OutputScanner::scan(*a, kind); + // progress_video is a preview, not the result + std::vector keep; + for (const MediaAsset& x : s.assets) + if (x.url.find("progress") == std::string::npos) keep.push_back(x); + if (!keep.empty()) s.assets = keep; + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "completed without assets"; + } + } else if (s.raw_state == "failed") { + s.state = JobState::kFailed; + s.error = AdapterSupport::str(g, "failure_reason"); + } else if (s.raw_state == "dreaming") { + s.state = JobState::kRunning; + } else { + s.state = JobState::kQueued; + } + return s; +} + +JobStatus LumaJobs::poll(LumaClient& client, const JobTicket& ticket) { + return status_of(client.generation(ticket.id), ticket.kind); +} + +Json LumaVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + Json b = Json::object(); + b.set("model", Json(r.model.empty() ? "ray-2" : r.model)); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put(b, "resolution", r.resolution); + if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds) + "s")); + if (!r.image_url.empty() || !r.end_image_url.empty()) { + Json kf = Json::object(); + if (!r.image_url.empty()) { + Json f0 = Json::object(); + f0.set("type", Json("image")); + f0.set("url", Json(r.image_url)); + kf.set("frame0", f0); + } + if (!r.end_image_url.empty()) { + Json f1 = Json::object(); + f1.set("type", Json("image")); + f1.set("url", Json(r.end_image_url)); + kf.set("frame1", f1); + } + b.set("keyframes", kf); + } + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json LumaVideoGenerator::plan(const VideoRequest& request) const { + return client_->plan("generations.create_video", body(request)).summary(); +} + +JobTicket LumaVideoGenerator::submit(const VideoRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "luma", "generate"); + return LumaJobs::ticket_from(client_->create("generations.create_video", b, true), + b.find("model")->as_string(), MediaKind::kVideo); +} + +JobStatus LumaVideoGenerator::poll(const JobTicket& ticket) { return LumaJobs::poll(*client_, ticket); } + +bool LumaVideoGenerator::cancel(const JobTicket& ticket, bool confirm) { + AdapterSupport::need_confirm(confirm, "luma", "delete generation"); + client_->remove(ticket.id, true); + return true; +} + +Json LumaImageGenerator::body(const ImageRequest& r) const { + r.validate(); + if (r.count != 1) throw llm::ApiError::invalid("luma", "one image per call (count must be 1)"); + if (r.width || r.height) throw llm::ApiError::invalid("luma", "use aspect_ratio, not width / height"); + Json b = Json::object(); + b.set("model", Json(r.model.empty() ? "photon-1" : r.model)); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio); + if (!r.source_url.empty()) { + Json ref = Json::array(); + Json one = Json::object(); + one.set("url", Json(r.source_url)); + ref.push(one); + b.set("image_ref", ref); + } + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json LumaImageGenerator::plan(const ImageRequest& request) const { + return client_->plan("generations.create_image", body(request)).summary(); +} + +JobTicket LumaImageGenerator::submit(const ImageRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "luma", "generate"); + return LumaJobs::ticket_from(client_->create("generations.create_image", b, true), + b.find("model")->as_string(), MediaKind::kImage); +} + +JobStatus LumaImageGenerator::poll(const JobTicket& ticket) { return LumaJobs::poll(*client_, ticket); } + +bool LumaImageGenerator::cancel(const JobTicket& ticket, bool confirm) { + AdapterSupport::need_confirm(confirm, "luma", "delete generation"); + client_->remove(ticket.id, true); + return true; +} + +} // namespace molo::media diff --git a/src/vendors/media/luma/luma.h b/src/vendors/media/luma/luma.h new file mode 100644 index 0000000..4847449 --- /dev/null +++ b/src/vendors/media/luma/luma.h @@ -0,0 +1,82 @@ +#pragma once + +// Luma Dream Machine API (https://api.lumalabs.ai/dream-machine/v1): video +// (Ray models), images (Photon), reframe, modify, upscale, audio, concepts, +// credits. Bearer auth with LUMA_API_KEY. States: queued, dreaming, +// completed, failed (from the official SDK types). + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/luma/calls.h" + +namespace molo::media { + +class LumaClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.lumalabs.ai/dream-machine/v1"; + static constexpr const char* kKeyEnv = "LUMA_API_KEY"; + + explicit LumaClient(std::shared_ptr client) : MediaService(std::move(client), LumaCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::bearer("luma", kHost, kKeyEnv); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // key: generations.create_video | create_image | reframe_image | reframe_video | modify_video. + Json create(const std::string& key, const Json& body, bool confirm); + PreparedCall plan(const std::string& key, const Json& body) const; + Json upscale(const std::string& generation_id, const Json& body, bool confirm); + Json add_audio(const std::string& generation_id, const Json& body, bool confirm); + Json generation(const std::string& id); + Json list(int limit, int offset); + Json remove(const std::string& id, bool confirm); // destructive, no default + Json credits(); + Json concepts(); + Json ping(); +}; + +class LumaVideoGenerator : public VideoGenerator { + public: + explicit LumaVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "luma"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { return {"ray-2", "ray-flash-2"}; } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket& ticket, bool confirm) override; // deletes the generation + int suggested_poll_ms() const override { return 4000; } + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr client_; +}; + +class LumaImageGenerator : public ImageGenerator { + public: + explicit LumaImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "luma"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { return {"photon-1", "photon-flash-1"}; } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 2000; } + + private: + Json body(const ImageRequest& request) const; + std::shared_ptr client_; +}; + +struct LumaJobs { + static JobTicket ticket_from(const Json& generation, const std::string& model, MediaKind kind); + static JobStatus poll(LumaClient& client, const JobTicket& ticket); + static JobStatus status_of(const Json& generation, MediaKind kind); +}; + +} // namespace molo::media diff --git a/src/vendors/media/minimax/calls.cpp b/src/vendors/media/minimax/calls.cpp new file mode 100644 index 0000000..d533d30 --- /dev/null +++ b/src/vendors/media/minimax/calls.cpp @@ -0,0 +1,31 @@ +#include "vendors/media/minimax/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"video.create", "POST", "/v1/video_generation", "videoGeneration", "json", "model,prompt,first_frame_image,last_frame_image,subject_reference,prompt_optimizer,fast_pretreatment,duration,resolution,callback_url", "", "paid"}, + {"video.query", "GET", "/v1/query/video_generation", "", "none", "", "task_id", "read"}, + {"files.retrieve", "GET", "/v1/files/retrieve", "", "none", "", "file_id", "read"}, + {"v2.video.create", "POST", "/v2/video_generation", "", "json", "model,content,resolution,duration,ratio,extra,callback_url", "", "paid"}, + {"v2.video.query", "GET", "/v2/query/video_generation/{task_id}", "", "none", "", "", "read"}, + {"v2.video.list", "GET", "/v2/query/video_generation", "", "none", "", "page_num,page_size", "read"}, + {"v2.video.delete", "DELETE", "/v2/video_generation/{task_id}", "", "none", "", "", "delete"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { +}; +// FIELDS-END + +const CallTable& MinimaxCalls::table() { + static const CallTable t("minimax", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t MinimaxCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* MinimaxCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/minimax/calls.h b/src/vendors/media/minimax/calls.h new file mode 100644 index 0000000..9ab2dca --- /dev/null +++ b/src/vendors/media/minimax/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct MinimaxCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/minimax/minimax.cpp b/src/vendors/media/minimax/minimax.cpp new file mode 100644 index 0000000..3287b99 --- /dev/null +++ b/src/vendors/media/minimax/minimax.cpp @@ -0,0 +1,138 @@ +#include "vendors/media/minimax/minimax.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs with_body(const Json& body) { + CallArgs a; + a.body = body; + return a; +} + +CallArgs task(const std::string& id) { + CallArgs a; + a.path["task_id"] = id; + return a; +} + +} // namespace + +void MinimaxClient::check_base_resp(const Json& doc) { + const Json* br = doc.is_object() ? doc.find("base_resp") : nullptr; + if (!br || !br->is_object()) return; + const Json* code = br->find("status_code"); + if (!code || !code->is_number() || code->as_int() == 0) return; + throw llm::ApiError("minimax", 400, std::to_string(code->as_int()), AdapterSupport::str(*br, "status_msg")); +} + +PreparedCall MinimaxClient::plan_video(const Json& body) const { return prepare("video.create", with_body(body)); } + +Json MinimaxClient::create_video(const Json& body, bool confirm) { + Json out = call_json("video.create", with_body(body), confirm); + check_base_resp(out); + return out; +} + +Json MinimaxClient::query_video(const std::string& task_id) { + CallArgs a; + a.query["task_id"] = task_id; + Json out = call_json("video.query", a); + check_base_resp(out); + return out; +} + +Json MinimaxClient::retrieve_file(const std::string& file_id) { + CallArgs a; + a.query["file_id"] = file_id; + Json out = call_json("files.retrieve", a); + check_base_resp(out); + return out; +} + +Json MinimaxClient::create_video_v2(const Json& body, bool confirm) { + return call_json("v2.video.create", with_body(body), confirm); +} +Json MinimaxClient::query_video_v2(const std::string& id) { return call_json("v2.video.query", task(id)); } + +Json MinimaxClient::list_video_v2(int page_num, int page_size) { + CallArgs a; + if (page_num > 0) a.query["page_num"] = std::to_string(page_num); + if (page_size > 0) a.query["page_size"] = std::to_string(page_size); + return call_json("v2.video.list", a); +} + +Json MinimaxClient::delete_video_v2(const std::string& id, bool confirm) { + return call_json("v2.video.delete", task(id), confirm); +} + +JobState MinimaxVideoGenerator::state_of(const std::string& status) { + if (status == "Success") return JobState::kSucceeded; + if (status == "Fail") return JobState::kFailed; + if (status == "Processing") return JobState::kRunning; + return JobState::kQueued; // Preparing, Queueing +} + +Json MinimaxVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + Json b = Json::object(); + b.set("model", Json(r.model.empty() ? "MiniMax-Hailuo-2.3" : r.model)); + AdapterSupport::put(b, "prompt", r.prompt); + AdapterSupport::put_int(b, "duration", r.duration_seconds); + if (!r.resolution.empty()) { + std::string res = r.resolution; + for (char& c : res) c = static_cast(std::toupper(static_cast(c))); + b.set("resolution", Json(res)); // docs: 768P / 1080P + } + AdapterSupport::put(b, "first_frame_image", r.image_url); + AdapterSupport::put(b, "last_frame_image", r.end_image_url); + if (!r.aspect_ratio.empty()) throw llm::ApiError::invalid("minimax", "v1 video_generation has no aspect_ratio"); + if (r.seed >= 0) throw llm::ApiError::invalid("minimax", "v1 video_generation has no seed"); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json MinimaxVideoGenerator::plan(const VideoRequest& request) const { + return client_->plan_video(body(request)).summary(); +} + +JobTicket MinimaxVideoGenerator::submit(const VideoRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "minimax", "generate"); + Json out = client_->create_video(b, true); + JobTicket t; + t.provider = "minimax"; + t.model = b.find("model")->as_string(); + t.kind = MediaKind::kVideo; + t.id = AdapterSupport::str(out, "task_id"); + if (t.id.empty()) throw llm::ApiError("minimax", 502, "BAD_RESPONSE", "no task_id in the reply"); + return t; +} + +JobStatus MinimaxVideoGenerator::poll(const JobTicket& ticket) { + Json q = client_->query_video(ticket.id); + JobStatus s; + s.raw_state = AdapterSupport::str(q, "status"); + s.state = state_of(s.raw_state); + if (s.state == JobState::kFailed) s.error = "Fail"; + if (s.state == JobState::kSucceeded) { + std::string file_id = AdapterSupport::str(q, "file_id"); + if (file_id.empty()) { + s.state = JobState::kFailed; + s.error = "Success without a file_id"; + return s; + } + Json f = client_->retrieve_file(file_id); + if (const Json* file = f.find("file")) s.assets = OutputScanner::scan(*file, MediaKind::kVideo); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "file retrieved without a download_url"; + } + } + return s; +} + +} // namespace molo::media diff --git a/src/vendors/media/minimax/minimax.h b/src/vendors/media/minimax/minimax.h new file mode 100644 index 0000000..701c431 --- /dev/null +++ b/src/vendors/media/minimax/minimax.h @@ -0,0 +1,66 @@ +#pragma once + +// MiniMax (Hailuo) video API on https://api.minimax.io: v1 video_generation +// (text, image, first/last frame, subject reference), the query endpoint, file +// retrieval for the download URL, and the v2 content-array endpoints. +// Bearer auth with MINIMAX_API_KEY. Errors can arrive as HTTP 200 with +// base_resp.status_code != 0: MinimaxClient turns those into ApiError. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/minimax/calls.h" + +namespace molo::media { + +class MinimaxClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.minimax.io"; + static constexpr const char* kKeyEnv = "MINIMAX_API_KEY"; + + explicit MinimaxClient(std::shared_ptr client) + : MediaService(std::move(client), MinimaxCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::bearer("minimax", kHost, kKeyEnv); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + Json create_video(const Json& body, bool confirm); // v1, paid + PreparedCall plan_video(const Json& body) const; + Json query_video(const std::string& task_id); // v1 + Json retrieve_file(const std::string& file_id); // v1 + Json create_video_v2(const Json& body, bool confirm); + Json query_video_v2(const std::string& task_id); + Json list_video_v2(int page_num, int page_size); + Json delete_video_v2(const std::string& task_id, bool confirm); // no default + + // Throws ApiError when base_resp.status_code is present and not 0. + static void check_base_resp(const Json& doc); +}; + +class MinimaxVideoGenerator : public VideoGenerator { + public: + explicit MinimaxVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "minimax"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"MiniMax-Hailuo-2.3", "MiniMax-Hailuo-02", "T2V-01-Director", "T2V-01"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } // v1 has no cancel + int suggested_poll_ms() const override { return 8000; } + + // Preparing / Queueing / Processing / Success / Fail (v1 query). + static JobState state_of(const std::string& status); + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/recraft/calls.cpp b/src/vendors/media/recraft/calls.cpp new file mode 100644 index 0000000..11b625f --- /dev/null +++ b/src/vendors/media/recraft/calls.cpp @@ -0,0 +1,48 @@ +#include "vendors/media/recraft/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"images.creativeUpscale", "POST", "/v1/images/creativeUpscale", "CreativeUpscale", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"}, + {"images.crispUpscale", "POST", "/v1/images/crispUpscale", "CrispUpscale", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"}, + {"images.enhance", "POST", "/v1/images/enhance", "EnhanceImage", "either", "image_format,refinement,response_format,zdr,image_url,image", "", "paid"}, + {"images.eraseRegion", "POST", "/v1/images/eraseRegion", "EraseRegion", "either", "image_format,response_format,zdr,image_url,mask_url,image,mask", "", "paid"}, + {"images.generateBackground", "POST", "/v1/images/generateBackground", "GenerateBackground", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,mask_url,image,mask", "", "paid"}, + {"images.generations", "POST", "/v1/images/generations", "GenerateImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"}, + {"images.generations.raster", "POST", "/v1/images/generations/raster", "GenerateImageRaster", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"}, + {"images.generations.vector", "POST", "/v1/images/generations/vector", "GenerateImageVector", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"}, + {"images.imageToImage", "POST", "/v1/images/imageToImage", "ImageToImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,strength,image", "", "paid"}, + {"images.inpaint", "POST", "/v1/images/inpaint", "InpaintImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,mask_url,image,mask", "", "paid"}, + {"images.outpaint", "POST", "/v1/images/outpaint", "OutpaintImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,expand_bottom,expand_left,expand_right,expand_top,size,zoom_out_percentage,image_url,image", "", "paid"}, + {"images.removeBackground", "POST", "/v1/images/removeBackground", "RemoveBackground", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"}, + {"images.replaceBackground", "POST", "/v1/images/replaceBackground", "ReplaceBackground", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,image", "", "paid"}, + {"images.variateImage", "POST", "/v1/images/variateImage", "VariateImage", "either", "image_format,model,n,random_seed,response_format,size,zdr,image_url,image", "", "paid"}, + {"images.vectorize", "POST", "/v1/images/vectorize", "VectorizeImage", "either", "image_format,response_format,upscale,zdr,color_reduction,limit_num_shapes,max_num_colors,max_num_shapes,return_gradients,shape_stacking,small_shape_filter,strict_color_palette,svg_compression,image_url,image", "", "paid"}, + {"styles", "GET", "/v1/styles", "ListStyles", "none", "", "", "read"}, + {"styles.create", "POST", "/v1/styles", "CreateStyle", "either", "image_weights,match,mix_policy,model,palette,private,prompt,source_style_weights,source_styles,style,image_urls,images", "", "paid"}, + {"styles.id.delete", "DELETE", "/v1/styles/{style_id}", "DeleteStyle", "none", "", "", "delete"}, + {"styles.id", "GET", "/v1/styles/{style_id}", "GetStyle", "none", "", "", "read"}, + {"styles.basic", "GET", "/v1/styles/basic", "ListBasicStyles", "none", "", "", "read"}, + {"users.me", "GET", "/v1/users/me", "GetCurrentUser", "none", "", "", "read"}, + {"prompts.enhance", "POST", "/v1/prompts/enhance", "EnhancePrompt", "json", "prompt", "", "paid"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"GenerateImageResponse", "created,data,credits,style_id"}, + {"User", "id,email,name,credits"}, +}; +// FIELDS-END + +const CallTable& RecraftCalls::table() { + static const CallTable t("recraft", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t RecraftCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* RecraftCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/recraft/calls.h b/src/vendors/media/recraft/calls.h new file mode 100644 index 0000000..a536883 --- /dev/null +++ b/src/vendors/media/recraft/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct RecraftCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/recraft/recraft.cpp b/src/vendors/media/recraft/recraft.cpp new file mode 100644 index 0000000..3ca751f --- /dev/null +++ b/src/vendors/media/recraft/recraft.cpp @@ -0,0 +1,102 @@ +#include "vendors/media/recraft/recraft.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs args(const Json& fields, const std::vector& files) { + CallArgs a; + a.body = fields; + a.files = files; + return a; +} + +CallArgs style_id(const std::string& id) { + CallArgs a; + a.path["style_id"] = id; + return a; +} + +} // namespace + +PreparedCall RecraftClient::plan(const std::string& key, const Json& fields, const std::vector& files) const { + return prepare(key, args(fields, files)); +} + +Json RecraftClient::run(const std::string& key, const Json& fields, const std::vector& files, bool confirm) { + return call_json(key, args(fields, files), confirm); +} + +Json RecraftClient::me() { return call_json("users.me"); } +Json RecraftClient::styles() { return call_json("styles"); } +Json RecraftClient::basic_styles() { return call_json("styles.basic"); } +Json RecraftClient::style(const std::string& id) { return call_json("styles.id", style_id(id)); } +Json RecraftClient::create_style(const Json& fields, const std::vector& files, bool confirm) { + return call_json("styles.create", args(fields, files), confirm); +} +Json RecraftClient::delete_style(const std::string& id, bool confirm) { + return call_json("styles.id.delete", style_id(id), confirm); +} + +Json RecraftImageGenerator::fields(const ImageRequest& r) const { + r.validate(); + Json f = Json::object(); + AdapterSupport::put(f, "prompt", r.prompt); + AdapterSupport::put(f, "negative_prompt", r.negative_prompt); + AdapterSupport::put(f, "model", r.model); + if (r.count > 1) f.set("n", Json(r.count)); + if (r.width && r.height) f.set("size", Json(std::to_string(r.width) + "x" + std::to_string(r.height))); + AdapterSupport::put_seed(f, "random_seed", r.seed); + if (!r.aspect_ratio.empty()) + throw llm::ApiError::invalid("recraft", "Recraft sizes images with size (WxH), not aspect_ratio"); + if (!r.source_url.empty()) + throw llm::ApiError::invalid("recraft", "source images use the images.* edit rows: RecraftClient::run with image_url"); + AdapterSupport::merge_extra(f, r.extra); + return f; +} + +Json RecraftImageGenerator::plan(const ImageRequest& request) const { + return client_->plan("images.generations", fields(request), {}).summary(); +} + +JobTicket RecraftImageGenerator::submit(const ImageRequest& request, bool confirm) { + Json f = fields(request); + AdapterSupport::need_confirm(confirm, "recraft", "generate"); + Json out = client_->run("images.generations", f, {}, true); + JobTicket t; + t.provider = "recraft"; + t.model = request.model; + t.kind = MediaKind::kImage; + t.id = "immediate"; + t.immediate = true; + if (const Json* d = out.find("data"); d && d->is_array()) { + for (const Json& item : d->as_array()) { + std::string b64 = AdapterSupport::str(item, "b64_json"); + if (!b64.empty()) { + MediaAsset a; + a.kind = MediaKind::kImage; + a.data_base64 = b64; + a.mime = "image/webp"; // Recraft's default raster format is lossless WebP + t.assets.push_back(a); + } + } + if (t.assets.empty()) t.assets = OutputScanner::scan(*d, MediaKind::kImage); + } + if (t.assets.empty()) throw llm::ApiError("recraft", 502, "BAD_RESPONSE", "no image in the reply"); + t.meta = Json::object(); + AdapterSupport::put(t.meta, "credits", AdapterSupport::str(out, "credits")); + return t; +} + +JobStatus RecraftImageGenerator::poll(const JobTicket& ticket) { + JobStatus s; + s.state = JobState::kSucceeded; + s.raw_state = "immediate"; + s.assets = ticket.assets; + return s; +} + +} // namespace molo::media diff --git a/src/vendors/media/recraft/recraft.h b/src/vendors/media/recraft/recraft.h new file mode 100644 index 0000000..cb72f61 --- /dev/null +++ b/src/vendors/media/recraft/recraft.h @@ -0,0 +1,61 @@ +#pragma once + +// Recraft external API: generation (general, raster, vector), image tools +// (vectorize, background, upscale, erase, remix, inpaint, outpaint, enhance), +// styles, prompt enhancement and the user endpoint. Bearer auth with +// RECRAFT_API_TOKEN, base https://external.api.recraft.ai. Every image call +// is synchronous: the reply carries the result, so tickets are immediate. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/recraft/calls.h" + +namespace molo::media { + +class RecraftClient : public MediaService { + public: + static constexpr const char* kHost = "https://external.api.recraft.ai"; + static constexpr const char* kKeyEnv = "RECRAFT_API_TOKEN"; + + explicit RecraftClient(std::shared_ptr client) + : MediaService(std::move(client), RecraftCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::bearer("recraft", kHost, kKeyEnv); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // Paid image rows (images.*) and prompts.enhance. JSON bodies take image_url + // fields; multipart is used when files are given. + Json run(const std::string& key, const Json& fields, const std::vector& files, bool confirm); + PreparedCall plan(const std::string& key, const Json& fields, const std::vector& files) const; + Json me(); // credits + identity: the free read + Json styles(); + Json basic_styles(); + Json style(const std::string& style_id); + Json create_style(const Json& fields, const std::vector& files, bool confirm); // $0.005 + Json delete_style(const std::string& style_id, bool confirm); // destructive +}; + +class RecraftImageGenerator : public ImageGenerator { + public: + explicit RecraftImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "recraft"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"recraftv4_1", "recraftv4_1_pro", "recraftv4", "recraftv4_pro", "recraftv3", "recraftv3_vector", "recraftv2"}; + } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } // synchronous API + + private: + Json fields(const ImageRequest& request) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/replicate/calls.cpp b/src/vendors/media/replicate/calls.cpp new file mode 100644 index 0000000..1711afd --- /dev/null +++ b/src/vendors/media/replicate/calls.cpp @@ -0,0 +1,61 @@ +#include "vendors/media/replicate/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"account.get", "GET", "/account", "account.get", "none", "", "", "read"}, + {"collections.list", "GET", "/collections", "collections.list", "none", "", "", "read"}, + {"collections.get", "GET", "/collections/{collection_slug}", "collections.get", "none", "", "", "read"}, + {"deployments.list", "GET", "/deployments", "deployments.list", "none", "", "", "read"}, + {"deployments.create", "POST", "/deployments", "deployments.create", "json", "hardware,max_instances,min_instances,model,name,version", "", "write"}, + {"deployments.delete", "DELETE", "/deployments/{deployment_owner}/{deployment_name}", "deployments.delete", "none", "", "", "delete"}, + {"deployments.get", "GET", "/deployments/{deployment_owner}/{deployment_name}", "deployments.get", "none", "", "", "read"}, + {"deployments.update", "PATCH", "/deployments/{deployment_owner}/{deployment_name}", "deployments.update", "json", "hardware,max_instances,min_instances,version", "", "write"}, + {"deployments.predictions.create", "POST", "/deployments/{deployment_owner}/{deployment_name}/predictions", "deployments.predictions.create", "json", "input,stream,webhook,webhook_events_filter", "", "paid"}, + {"files.list", "GET", "/files", "files.list", "none", "", "", "read"}, + {"files.create", "POST", "/files", "files.create", "multipart", "content,filename,metadata,type", "", "write"}, + {"files.delete", "DELETE", "/files/{file_id}", "files.delete", "none", "", "", "delete"}, + {"files.get", "GET", "/files/{file_id}", "files.get", "none", "", "", "read"}, + {"files.download", "GET", "/files/{file_id}/download", "files.download", "none", "", "owner,expiry,signature", "read"}, + {"hardware.list", "GET", "/hardware", "hardware.list", "none", "", "", "read"}, + {"models.list", "GET", "/models", "models.list", "none", "", "sort_by,sort_direction", "read"}, + {"models.create", "POST", "/models", "models.create", "json", "cover_image_url,description,github_url,hardware,license_url,name,owner,paper_url,visibility", "", "write"}, + {"models.delete", "DELETE", "/models/{model_owner}/{model_name}", "models.delete", "none", "", "", "delete"}, + {"models.get", "GET", "/models/{model_owner}/{model_name}", "models.get", "none", "", "", "read"}, + {"models.update", "PATCH", "/models/{model_owner}/{model_name}", "models.update", "json", "description,github_url,license_url,paper_url,readme,weights_url", "", "write"}, + {"models.examples.list", "GET", "/models/{model_owner}/{model_name}/examples", "models.examples.list", "none", "", "", "read"}, + {"models.predictions.create", "POST", "/models/{model_owner}/{model_name}/predictions", "models.predictions.create", "json", "input,stream,webhook,webhook_events_filter", "", "paid"}, + {"models.readme.get", "GET", "/models/{model_owner}/{model_name}/readme", "models.readme.get", "none", "", "", "read"}, + {"models.versions.list", "GET", "/models/{model_owner}/{model_name}/versions", "models.versions.list", "none", "", "", "read"}, + {"models.versions.delete", "DELETE", "/models/{model_owner}/{model_name}/versions/{version_id}", "models.versions.delete", "none", "", "", "delete"}, + {"models.versions.get", "GET", "/models/{model_owner}/{model_name}/versions/{version_id}", "models.versions.get", "none", "", "", "read"}, + {"trainings.create", "POST", "/models/{model_owner}/{model_name}/versions/{version_id}/trainings", "trainings.create", "json", "destination,input,webhook,webhook_events_filter", "", "paid"}, + {"predictions.list", "GET", "/predictions", "predictions.list", "none", "", "created_after,created_before,source", "read"}, + {"predictions.create", "POST", "/predictions", "predictions.create", "json", "input,stream,version,webhook,webhook_events_filter", "", "paid"}, + {"predictions.get", "GET", "/predictions/{prediction_id}", "predictions.get", "none", "", "", "read"}, + {"predictions.cancel", "POST", "/predictions/{prediction_id}/cancel", "predictions.cancel", "none", "", "", "write"}, + {"search", "GET", "/search", "search", "none", "", "query,limit", "read"}, + {"trainings.list", "GET", "/trainings", "trainings.list", "none", "", "", "read"}, + {"trainings.get", "GET", "/trainings/{training_id}", "trainings.get", "none", "", "", "read"}, + {"trainings.cancel", "POST", "/trainings/{training_id}/cancel", "trainings.cancel", "none", "", "", "write"}, + {"webhooks.default.secret.get", "GET", "/webhooks/default/secret", "webhooks.default.secret.get", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"schemas_prediction_response", "id,model,version,input,output,status,error,logs,metrics,created_at,started_at,completed_at,urls,get,cancel"}, +}; +// FIELDS-END + +const CallTable& ReplicateCalls::table() { + static const CallTable t("replicate", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t ReplicateCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* ReplicateCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/replicate/calls.h b/src/vendors/media/replicate/calls.h new file mode 100644 index 0000000..8742266 --- /dev/null +++ b/src/vendors/media/replicate/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct ReplicateCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/replicate/replicate.cpp b/src/vendors/media/replicate/replicate.cpp new file mode 100644 index 0000000..b50b8f2 --- /dev/null +++ b/src/vendors/media/replicate/replicate.cpp @@ -0,0 +1,278 @@ +#include "vendors/media/replicate/replicate.h" + +#include +#include + +#include "security/crypto/Base64.h" +#include "security/crypto/Hmac.h" +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/caps.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs args_for(const ReplicateClient::ModelRef& m, const Json& input, const Json& options) { + CallArgs a; + Json body = Json::object(); + body.set("input", input.is_null() ? Json::object() : input); + if (options.is_object()) + for (const auto& [k, v] : options.as_object()) body.set(k, v); // webhook, webhook_events_filter, stream + if (m.version.empty()) { + a.path["model_owner"] = m.owner; + a.path["model_name"] = m.name; + } else { + body.set("version", Json(m.version)); + } + a.body = body; + return a; +} + +CallArgs one(const char* name, const std::string& value) { + CallArgs a; + a.path[name] = value; + return a; +} + +Json mark_truncated(Json page) { + const Json* next = page.is_object() ? page.find("next") : nullptr; + if (next && next->is_string() && !next->as_string().empty()) page.set("truncated", Json(true)); + return page; +} + +} // namespace + +ReplicateClient::ModelRef ReplicateClient::ModelRef::parse(const std::string& ref) { + ModelRef m; + std::string head = ref; + std::size_t colon = ref.find(':'); + if (colon != std::string::npos) { + head = ref.substr(0, colon); + m.version = ref.substr(colon + 1); + } + std::size_t slash = head.find('/'); + if (slash == std::string::npos || head.find('/', slash + 1) != std::string::npos) + throw llm::ApiError::invalid("replicate", "model must look like owner/name or owner/name:version"); + m.owner = head.substr(0, slash); + m.name = head.substr(slash + 1); + if (!CallTable::valid_segment(m.owner) || !CallTable::valid_segment(m.name) || + (colon != std::string::npos && !CallTable::valid_segment(m.version))) + throw llm::ApiError::invalid("replicate", "bad model reference"); + return m; +} + +PreparedCall ReplicateClient::plan_prediction(const ModelRef& m, const Json& input, const Json& options) const { + return prepare(m.version.empty() ? "models.predictions.create" : "predictions.create", args_for(m, input, options)); +} + +Json ReplicateClient::create_prediction(const ModelRef& m, const Json& input, const Json& options, bool confirm) { + return call_json(m.version.empty() ? "models.predictions.create" : "predictions.create", + args_for(m, input, options), confirm); +} + +Json ReplicateClient::create_deployment_prediction(const std::string& owner, const std::string& name, + const Json& input, const Json& options, bool confirm) { + CallArgs a; + a.path["deployment_owner"] = owner; + a.path["deployment_name"] = name; + Json body = Json::object(); + body.set("input", input); + if (options.is_object()) + for (const auto& [k, v] : options.as_object()) body.set(k, v); + a.body = body; + return call_json("deployments.predictions.create", a, confirm); +} + +Json ReplicateClient::get_prediction(const std::string& id) { return call_json("predictions.get", one("prediction_id", id)); } +Json ReplicateClient::cancel_prediction(const std::string& id, bool confirm) { + return call_json("predictions.cancel", one("prediction_id", id), confirm); +} +Json ReplicateClient::list_predictions() { return mark_truncated(call_json("predictions.list")); } +Json ReplicateClient::account() { return call_json("account.get"); } +Json ReplicateClient::hardware() { return call_json("hardware.list"); } +Json ReplicateClient::models_list() { return mark_truncated(call_json("models.list")); } + +Json ReplicateClient::model_get(const std::string& owner, const std::string& name) { + CallArgs a; + a.path["model_owner"] = owner; + a.path["model_name"] = name; + return call_json("models.get", a); +} + +Json ReplicateClient::versions_list(const std::string& owner, const std::string& name) { + CallArgs a; + a.path["model_owner"] = owner; + a.path["model_name"] = name; + return mark_truncated(call_json("models.versions.list", a)); +} + +Json ReplicateClient::deployments_list() { return mark_truncated(call_json("deployments.list")); } + +Json ReplicateClient::deployment_get(const std::string& owner, const std::string& name) { + CallArgs a; + a.path["deployment_owner"] = owner; + a.path["deployment_name"] = name; + return call_json("deployments.get", a); +} + +Json ReplicateClient::files_list() { return mark_truncated(call_json("files.list")); } +Json ReplicateClient::file_get(const std::string& id) { return call_json("files.get", one("file_id", id)); } +Json ReplicateClient::file_delete(const std::string& id, bool confirm) { + return call_json("files.delete", one("file_id", id), confirm); +} + +Json ReplicateClient::file_upload(const std::string& filename, const std::string& content_type, + const std::string& data, bool confirm) { + CallArgs a; + a.body = Json::object(); + a.body.set("filename", Json(filename)); + a.files.push_back(FilePart{"content", filename, content_type, data}); + return call_json("files.create", a, confirm); +} + +Json ReplicateClient::collections_list() { return mark_truncated(call_json("collections.list")); } + +Json ReplicateClient::search(const std::string& query) { + CallArgs a; + a.query["query"] = query; + return call_json("search", a); +} + +Json ReplicateClient::webhook_secret() { return call_json("webhooks.default.secret.get"); } +Json ReplicateClient::trainings_list() { return mark_truncated(call_json("trainings.list")); } + +// ---------------------------------------------------------------- webhooks + +std::string ReplicateWebhook::sign(const std::string& secret, const std::string& id, const std::string& timestamp, + const std::string& body) { + std::string b64 = secret.rfind("whsec_", 0) == 0 ? secret.substr(6) : secret; + std::string key; + if (b64.empty() || !security::crypto::Base64::decode(b64, key) || key.empty()) return ""; + if (body.size() > MediaCaps::kMaxWebhookBodyBytes) return ""; + auto mac = security::crypto::Hmac::sha256(key, id + "." + timestamp + "." + body); + return security::crypto::Base64::encode(reinterpret_cast(mac.data()), mac.size()); +} + +bool ReplicateWebhook::verify(const std::string& secret, const std::string& id, const std::string& timestamp, + const std::string& signature_header, const std::string& body, long now_seconds) { + if (id.empty() || timestamp.empty() || timestamp.size() > 12 || signature_header.size() > 1024) return false; + for (char c : timestamp) + if (c < '0' || c > '9') return false; + long ts = std::strtol(timestamp.c_str(), nullptr, 10); + long skew = now_seconds > ts ? now_seconds - ts : ts - now_seconds; + if (skew > MediaCaps::kWebhookToleranceSeconds) return false; + std::string expected = sign(secret, id, timestamp, body); + if (expected.empty()) return false; + std::istringstream in(signature_header); + std::string part; + while (in >> part) { + if (part.rfind("v1,", 0) != 0) continue; + std::string got = part.substr(3); + if (got.size() != expected.size()) continue; + unsigned char diff = 0; // constant time over the whole string + for (std::size_t i = 0; i < got.size(); ++i) diff |= static_cast(got[i] ^ expected[i]); + if (diff == 0) return true; + } + return false; +} + +// -------------------------------------------------------------------- jobs + +JobTicket ReplicateJobs::ticket_from(const Json& p, const std::string& model, MediaKind kind) const { + JobTicket t; + t.provider = "replicate"; + t.model = model; + t.kind = kind; + t.id = AdapterSupport::str(p, "id"); + if (t.id.empty()) throw llm::ApiError("replicate", 502, "BAD_RESPONSE", "no id in the prediction"); + return t; +} + +JobStatus ReplicateJobs::poll(const JobTicket& ticket, MediaKind kind) { + Json p = client_->get_prediction(ticket.id); + JobStatus s; + s.raw_state = AdapterSupport::str(p, "status"); + if (s.raw_state == "succeeded") { + s.state = JobState::kSucceeded; + if (const Json* o = p.find("output")) s.assets = OutputScanner::scan(*o, kind); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "succeeded without a downloadable output (non-URL output)"; + } + } else if (s.raw_state == "failed") { + s.state = JobState::kFailed; + s.error = AdapterSupport::str(p, "error"); + } else if (s.raw_state == "canceled" || s.raw_state == "aborted") { + s.state = JobState::kCancelled; + } else if (s.raw_state == "processing") { + s.state = JobState::kRunning; + } else { + s.state = JobState::kQueued; + } + return s; +} + +bool ReplicateJobs::cancel(const JobTicket& ticket, bool confirm) { + AdapterSupport::need_confirm(confirm, "replicate", "cancel"); + client_->cancel_prediction(ticket.id, true); + return true; +} + +Json ReplicateImageGenerator::input(const ImageRequest& r) const { + r.validate(); + Json in = Json::object(); + AdapterSupport::put(in, "prompt", r.prompt); + AdapterSupport::put(in, "negative_prompt", r.negative_prompt); + AdapterSupport::put_int(in, "width", r.width); + AdapterSupport::put_int(in, "height", r.height); + AdapterSupport::put(in, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put_seed(in, "seed", r.seed); + if (r.count > 1) in.set("num_outputs", Json(r.count)); + AdapterSupport::put(in, "image", r.source_url); + AdapterSupport::merge_extra(in, r.extra); + return in; +} + +Json ReplicateImageGenerator::plan(const ImageRequest& r) const { + if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)"); + return jobs_.client().plan_prediction(ReplicateClient::ModelRef::parse(r.model), input(r), Json::object()).summary(); +} + +JobTicket ReplicateImageGenerator::submit(const ImageRequest& r, bool confirm) { + if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)"); + auto ref = ReplicateClient::ModelRef::parse(r.model); + Json in = input(r); + AdapterSupport::need_confirm(confirm, "replicate", "create prediction"); + return jobs_.ticket_from(jobs_.client().create_prediction(ref, in, Json::object(), true), r.model, MediaKind::kImage); +} + +Json ReplicateVideoGenerator::input(const VideoRequest& r) const { + r.validate(); + Json in = Json::object(); + AdapterSupport::put(in, "prompt", r.prompt); + AdapterSupport::put(in, "negative_prompt", r.negative_prompt); + AdapterSupport::put_int(in, "duration", r.duration_seconds); + AdapterSupport::put(in, "aspect_ratio", r.aspect_ratio); + AdapterSupport::put(in, "resolution", r.resolution); + AdapterSupport::put_seed(in, "seed", r.seed); + AdapterSupport::put(in, "image", r.image_url); + AdapterSupport::put(in, "last_frame_image", r.end_image_url); + AdapterSupport::merge_extra(in, r.extra); + return in; +} + +Json ReplicateVideoGenerator::plan(const VideoRequest& r) const { + if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)"); + return jobs_.client().plan_prediction(ReplicateClient::ModelRef::parse(r.model), input(r), Json::object()).summary(); +} + +JobTicket ReplicateVideoGenerator::submit(const VideoRequest& r, bool confirm) { + if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)"); + auto ref = ReplicateClient::ModelRef::parse(r.model); + Json in = input(r); + AdapterSupport::need_confirm(confirm, "replicate", "create prediction"); + return jobs_.ticket_from(jobs_.client().create_prediction(ref, in, Json::object(), true), r.model, MediaKind::kVideo); +} + +} // namespace molo::media diff --git a/src/vendors/media/replicate/replicate.h b/src/vendors/media/replicate/replicate.h new file mode 100644 index 0000000..a56c77c --- /dev/null +++ b/src/vendors/media/replicate/replicate.h @@ -0,0 +1,129 @@ +#pragma once + +// Replicate: predictions (official models, versions, deployments), models, +// files, account, hardware, collections, trainings and webhook verification. +// Bearer auth, base https://api.replicate.com/v1. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/replicate/calls.h" + +namespace molo::media { + +class ReplicateClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.replicate.com/v1"; // the spec server url; table paths are relative to it + static constexpr const char* kKeyEnv = "REPLICATE_API_TOKEN"; + + explicit ReplicateClient(std::shared_ptr client) + : MediaService(std::move(client), ReplicateCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::bearer("replicate", kHost, kKeyEnv); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // "owner/name" (official model) or "owner/name:version" (pinned version). + struct ModelRef { + std::string owner, name, version; + static ModelRef parse(const std::string& ref); // throws ApiError::invalid + }; + + // Paid. Official model -> models.predictions.create; with version -> predictions.create. + Json create_prediction(const ModelRef& model, const Json& input, const Json& options, bool confirm); + PreparedCall plan_prediction(const ModelRef& model, const Json& input, const Json& options) const; + Json create_deployment_prediction(const std::string& owner, const std::string& name, const Json& input, + const Json& options, bool confirm); + Json get_prediction(const std::string& id); + Json cancel_prediction(const std::string& id, bool confirm); // write, no default + // First page only; "truncated" is set when the server has a next page. + Json list_predictions(); + + Json account(); + Json hardware(); + Json models_list(); + Json model_get(const std::string& owner, const std::string& name); + Json versions_list(const std::string& owner, const std::string& name); + Json deployments_list(); + Json deployment_get(const std::string& owner, const std::string& name); + Json files_list(); + Json file_get(const std::string& id); + Json file_upload(const std::string& filename, const std::string& content_type, const std::string& data, bool confirm); + Json file_delete(const std::string& id, bool confirm); // destructive, no default + Json collections_list(); + Json search(const std::string& query); + Json webhook_secret(); + Json trainings_list(); +}; + +// Standard Webhooks verification (id, timestamp, signature headers). +class ReplicateWebhook { + public: + // secret = the "whsec_..." value from webhooks.default.secret.get. + // signature_header = "v1, v1," (space separated). + // now_seconds: injected clock. Returns false on any mismatch, stale or + // malformed input; never throws on attacker-controlled data. + static bool verify(const std::string& secret, const std::string& webhook_id, const std::string& timestamp, + const std::string& signature_header, const std::string& body, long now_seconds); + // base64(HMAC-SHA256(key, id + "." + timestamp + "." + body)); "" on a bad secret. + static std::string sign(const std::string& secret, const std::string& webhook_id, const std::string& timestamp, + const std::string& body); +}; + +// kind-parametrised adapter core shared by the image and video generators. +class ReplicateJobs { + public: + explicit ReplicateJobs(std::shared_ptr c) : client_(std::move(c)) {} + JobStatus poll(const JobTicket& ticket, MediaKind kind); + bool cancel(const JobTicket& ticket, bool confirm); + JobTicket ticket_from(const Json& prediction, const std::string& model, MediaKind kind) const; + ReplicateClient& client() { return *client_; } + const ReplicateClient& client() const { return *client_; } + + private: + std::shared_ptr client_; +}; + +class ReplicateImageGenerator : public ImageGenerator { + public: + explicit ReplicateImageGenerator(std::shared_ptr c) : jobs_(std::move(c)) {} + std::string provider() const override { return "replicate"; } + bool configured() const override { return jobs_.client().configured(); } + std::vector models() const override { + return {"black-forest-labs/flux-schnell", "black-forest-labs/flux-dev", "black-forest-labs/flux-1.1-pro", + "stability-ai/stable-diffusion-3.5-large", "ideogram-ai/ideogram-v3-turbo", "recraft-ai/recraft-v3"}; + } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override { return jobs_.poll(ticket, MediaKind::kImage); } + bool cancel(const JobTicket& ticket, bool confirm) override { return jobs_.cancel(ticket, confirm); } + int suggested_poll_ms() const override { return 1500; } + + private: + Json input(const ImageRequest& request) const; + ReplicateJobs jobs_; +}; + +class ReplicateVideoGenerator : public VideoGenerator { + public: + explicit ReplicateVideoGenerator(std::shared_ptr c) : jobs_(std::move(c)) {} + std::string provider() const override { return "replicate"; } + bool configured() const override { return jobs_.client().configured(); } + std::vector models() const override { + return {"minimax/video-01", "luma/ray", "kwaivgi/kling-v1.6-standard", "wan-video/wan-2.1-t2v-480p"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override { return jobs_.poll(ticket, MediaKind::kVideo); } + bool cancel(const JobTicket& ticket, bool confirm) override { return jobs_.cancel(ticket, confirm); } + int suggested_poll_ms() const override { return 3000; } + + private: + Json input(const VideoRequest& request) const; + ReplicateJobs jobs_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/runway/calls.cpp b/src/vendors/media/runway/calls.cpp new file mode 100644 index 0000000..2e30bb1 --- /dev/null +++ b/src/vendors/media/runway/calls.cpp @@ -0,0 +1,91 @@ +#include "vendors/media/runway/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"text_to_image", "POST", "/v1/text_to_image", "", "json", "model,promptText,ratio,referenceImages,contentModeration,seed,background,outputCount,quality,outputFormat,grounding,edit,maskUri,preciseEdit,thinkingLevel,backgroundColor,colors", "", "paid"}, + {"text_to_video", "POST", "/v1/text_to_video", "", "json", "duration,model,promptText,ratio,contentModeration,outputFormat,proresProfile,seed,audio,negativePrompt,providerSettings,referenceAudio,references,referenceVideos,resolution,cfgScale,promptExpansionMode,draft,draftTaskId", "", "paid"}, + {"image_to_video", "POST", "/v1/image_to_video", "", "json", "duration,model,promptImage,promptText,ratio,contentModeration,outputFormat,proresProfile,seed,audio,negativePrompt,providerSettings,references,referenceAudio,resolution,cfgScale,referenceVideos,promptExpansionMode,draft", "", "paid"}, + {"video_to_video", "POST", "/v1/video_to_video", "", "json", "model,videoUri,contentModeration,keyframes,outputFormat,promptText,proresProfile,ratio,seed,targetAspectRatio,promptVideo,providerSettings,references,duration,referenceAudio,referenceVideos,resolution,audio,mode,draft", "", "paid"}, + {"image_upscale", "POST", "/v1/image_upscale", "", "json", "imageUri,model,flavor,scaleFactor,sharpen,smartGrain,ultraDetail", "", "paid"}, + {"video_upscale", "POST", "/v1/video_upscale", "", "json", "model,videoUri,creativity,flavor,fpsBoost,resolution,sharpen,smartGrain,targetFramerate", "", "paid"}, + {"video_to_hdr", "POST", "/v1/video_to_hdr", "", "json", "model,videoUri,outputFormat,proresProfile", "", "paid"}, + {"video_to_audio", "POST", "/v1/video_to_audio", "", "json", "duration,model,videoUri,format,negativePrompt,numVariants,preserveSpeech,promptText,seed", "", "paid"}, + {"character_performance", "POST", "/v1/character_performance", "", "json", "type,uri,publicFigureThreshold,character,model,reference,bodyControl,contentModeration,expressionIntensity,ratio,seed", "", "paid"}, + {"sound_effect", "POST", "/v1/sound_effect", "", "json", "model,promptText,loudnessRate,outputFormat,pitchRate,referenceAudios,sampleRate,speechRate,duration,format,negativePrompt,numVariants,seed,loop", "", "paid"}, + {"speech_to_speech", "POST", "/v1/speech_to_speech", "", "json", "type,uri,presetId,media,model,voice,removeBackgroundNoise", "", "paid"}, + {"text_to_speech", "POST", "/v1/text_to_speech", "", "json", "model,promptText,loudnessRate,outputFormat,pitchRate,sampleRate,speechRate,voice,languageCode,nextText,previousText,seed,similarityBoost,speed,stability,style,useSpeakerBoost,applyTextNormalization", "", "paid"}, + {"voice_dubbing", "POST", "/v1/voice_dubbing", "", "json", "audioUri,model,targetLang,disableVoiceCloning,dropBackgroundAudio,numSpeakers", "", "paid"}, + {"voice_isolation", "POST", "/v1/voice_isolation", "", "json", "audioUri,model", "", "paid"}, + {"avatar_videos", "POST", "/v1/avatar_videos", "", "json", "presetId,type,avatarId,audio,text,voice,avatar,model,speech", "", "paid"}, + {"generate.image", "POST", "/v1/generate/image", "", "json", "promptText,aspectRatio,contentModeration,outputCount,referenceImages,resolution,seed,configId,input", "", "paid"}, + {"generate.video", "POST", "/v1/generate/video", "", "json", "aspectRatio,audio,contentModeration,duration,keyframes,negativePrompt,promptText,referenceAudio,referenceImages,referenceVideos,resolution,seed,configId,input", "", "paid"}, + {"generate.audio", "POST", "/v1/generate/audio", "", "json", "promptText,type,duration,loop,referenceAudios,voice,configId,input", "", "paid"}, + {"recipes.ad_localization", "POST", "/v1/recipes/ad_localization", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"recipes.marketing_stock_image", "POST", "/v1/recipes/marketing_stock_image", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"recipes.multi_shot_video", "POST", "/v1/recipes/multi_shot_video", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"recipes.product_ad", "POST", "/v1/recipes/product_ad", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"recipes.product_campaign_image", "POST", "/v1/recipes/product_campaign_image", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"recipes.product_swap", "POST", "/v1/recipes/product_swap", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"recipes.product_ugc", "POST", "/v1/recipes/product_ugc", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"}, + {"tasks.get", "GET", "/v1/tasks/{id}", "", "none", "", "", "read"}, + {"tasks.delete", "DELETE", "/v1/tasks/{id}", "", "none", "", "", "delete"}, + {"organization.get", "GET", "/v1/organization", "", "none", "", "", "read"}, + {"organization.usage", "POST", "/v1/organization/usage", "", "json", "beforeDate,startDate", "", "read"}, + {"uploads.create", "POST", "/v1/uploads", "", "json", "file,fileMetadata", "", "write"}, + {"avatars.create", "POST", "/v1/avatars", "", "json", "presetId,type,id,name,personality,referenceImage,voice,documentIds,imageProcessing,startScript,endDate,startDate", "", "write"}, + {"avatars.list", "GET", "/v1/avatars", "", "none", "", "", "read"}, + {"avatars.get", "GET", "/v1/avatars/{id}", "", "none", "", "", "read"}, + {"avatars.update", "PATCH", "/v1/avatars/{id}", "", "json", "presetId,type,id,name,personality,referenceImage,voice,documentIds,imageProcessing,startScript,endDate,startDate", "", "write"}, + {"avatars.delete", "DELETE", "/v1/avatars/{id}", "", "none", "", "", "delete"}, + {"avatar_usage", "GET", "/v1/avatar_usage", "", "none", "", "", "read"}, + {"avatar_conversations.list", "GET", "/v1/avatar_conversations", "", "none", "", "", "read"}, + {"avatar_conversations.get", "GET", "/v1/avatar_conversations/{id}", "", "none", "", "", "read"}, + {"avatar_conversations.delete", "DELETE", "/v1/avatar_conversations/{id}", "", "none", "", "", "delete"}, + {"documents.create", "POST", "/v1/documents", "", "json", "content,name", "", "write"}, + {"documents.list", "GET", "/v1/documents", "", "none", "", "", "read"}, + {"documents.get", "GET", "/v1/documents/{id}", "", "none", "", "", "read"}, + {"documents.update", "PATCH", "/v1/documents/{id}", "", "json", "content,name", "", "write"}, + {"documents.delete", "DELETE", "/v1/documents/{id}", "", "none", "", "", "delete"}, + {"voices.create", "POST", "/v1/voices", "", "json", "audio,type,model,prompt,from,name,description", "", "write"}, + {"voices.preview", "POST", "/v1/voices/preview", "", "json", "audio,type,model,prompt,from,name,description", "", "paid"}, + {"voices.list", "GET", "/v1/voices", "", "none", "", "", "read"}, + {"voices.get", "GET", "/v1/voices/{id}", "", "none", "", "", "read"}, + {"voices.update", "PATCH", "/v1/voices/{id}", "", "json", "audio,type,model,prompt,from,name,description", "", "write"}, + {"voices.delete", "DELETE", "/v1/voices/{id}", "", "none", "", "", "delete"}, + {"realtime_sessions.create", "POST", "/v1/realtime_sessions", "", "json", "presetId,type,avatarId,signedUrl,token,roomName,url,agentIdentity,description,name,parameters,timeoutSeconds,avatar,model,integration,livekit,maxDuration,personality,startScript,tools", "", "paid"}, + {"realtime_sessions.get", "GET", "/v1/realtime_sessions/{id}", "", "none", "", "", "read"}, + {"realtime_sessions.delete", "DELETE", "/v1/realtime_sessions/{id}", "", "none", "", "", "delete"}, + {"routers.create", "POST", "/v1/routers", "", "json", "fallback,maxCreditsPerGeneration,models,optimizeFor,schemaVersion,slug,description,name,settings", "", "write"}, + {"routers.list", "GET", "/v1/routers", "", "none", "", "", "read"}, + {"routers.get", "GET", "/v1/routers/{id}", "", "none", "", "", "read"}, + {"routers.update", "PATCH", "/v1/routers/{id}", "", "json", "fallback,maxCreditsPerGeneration,models,optimizeFor,schemaVersion,slug,description,name,settings", "", "write"}, + {"routers.delete", "DELETE", "/v1/routers/{id}", "", "none", "", "", "delete"}, + {"routers.requests", "GET", "/v1/routers/{id}/requests", "", "none", "", "", "read"}, + {"workflows.list", "GET", "/v1/workflows", "", "none", "", "", "read"}, + {"workflows.get", "GET", "/v1/workflows/{id}", "", "none", "", "", "read"}, + {"workflows.invoke", "POST", "/v1/workflows/{id}", "", "json", "type,value,uri,nodeOutputs", "", "paid"}, + {"workflow_invocations.get", "GET", "/v1/workflow_invocations/{id}", "", "none", "", "", "read"}, + {"audit_logs.list", "GET", "/v1/organization/webapp/audit_logs", "", "none", "", "", "read"}, + {"audit_logs.get", "GET", "/v1/organization/webapp/audit_logs/{id}", "", "none", "", "", "read"}, + {"webapp_usage.list", "GET", "/v1/organization/webapp/usage", "", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"TaskRetrieveResponse", "id,status,createdAt,output,failure,failureCode,progress"}, +}; +// FIELDS-END + +const CallTable& RunwayCalls::table() { + static const CallTable t("runway", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t RunwayCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* RunwayCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/runway/calls.h b/src/vendors/media/runway/calls.h new file mode 100644 index 0000000..83818e4 --- /dev/null +++ b/src/vendors/media/runway/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct RunwayCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/runway/runway.cpp b/src/vendors/media/runway/runway.cpp new file mode 100644 index 0000000..f0f0f26 --- /dev/null +++ b/src/vendors/media/runway/runway.cpp @@ -0,0 +1,163 @@ +#include "vendors/media/runway/runway.h" + +#include "vendors/media/common/adapter_support.h" +#include "vendors/media/common/output_scanner.h" + +namespace molo::media { + +namespace { + +CallArgs with_body(const Json& body) { + CallArgs a; + a.body = body; + return a; +} + +CallArgs by_id(const std::string& id) { + CallArgs a; + a.path["id"] = id; + return a; +} + +std::string ratio_of(const std::string& aspect, int width, int height) { + if (!aspect.empty()) return aspect; + if (width && height) return std::to_string(width) + ":" + std::to_string(height); + return ""; +} + +} // namespace + +PreparedCall RunwayClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); } +Json RunwayClient::submit(const std::string& key, const Json& body, bool confirm) { + return call_json(key, with_body(body), confirm); +} +Json RunwayClient::task(const std::string& id) { return call_json("tasks.get", by_id(id)); } +Json RunwayClient::cancel_or_delete_task(const std::string& id, bool confirm) { + return call_json("tasks.delete", by_id(id), confirm); +} +Json RunwayClient::organization() { return call_json("organization.get"); } + +Json RunwayClient::organization_usage(const Json& body) { + PreparedCall c = prepare("organization.usage", with_body(body)); + c.effect = Effect::kRead; // a usage query: the POST creates nothing and costs nothing + return parse_json("runway", execute(c, false)); +} + +JobTicket RunwayJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) { + JobTicket t; + t.provider = "runway"; + t.model = model; + t.kind = kind; + t.id = AdapterSupport::str(reply, "id"); + if (t.id.empty()) throw llm::ApiError("runway", 502, "BAD_RESPONSE", "no id in the task reply"); + return t; +} + +JobStatus RunwayJobs::status_of(const Json& task, MediaKind kind) { + JobStatus s; + s.raw_state = AdapterSupport::str(task, "status"); + if (s.raw_state == "SUCCEEDED") { + s.state = JobState::kSucceeded; + if (const Json* o = task.find("output")) s.assets = OutputScanner::scan(*o, kind); + if (s.assets.empty()) { + s.state = JobState::kFailed; + s.error = "succeeded without output URLs"; + } + } else if (s.raw_state == "FAILED") { + s.state = JobState::kFailed; + s.error = AdapterSupport::str(task, "failure"); + std::string code = AdapterSupport::str(task, "failureCode"); + if (!code.empty()) s.error += " (" + code + ")"; + } else if (s.raw_state == "CANCELLED") { + s.state = JobState::kCancelled; + } else if (s.raw_state == "RUNNING") { + s.state = JobState::kRunning; + if (const Json* p = task.find("progress"); p && p->is_number()) s.progress = p->as_number(); + } else { + s.state = JobState::kQueued; // PENDING, THROTTLED + } + return s; +} + +JobStatus RunwayJobs::poll(RunwayClient& client, const JobTicket& ticket) { + return status_of(client.task(ticket.id), ticket.kind); +} + +Json RunwayImageGenerator::body(const ImageRequest& r) const { + r.validate(); + if (r.count != 1) throw llm::ApiError::invalid("runway", "one image per task (count must be 1)"); + Json b = Json::object(); + b.set("model", Json(r.model.empty() ? "gen4_image" : r.model)); + AdapterSupport::put(b, "promptText", r.prompt); + AdapterSupport::put(b, "ratio", ratio_of(r.aspect_ratio, r.width, r.height)); + AdapterSupport::put_seed(b, "seed", r.seed); + if (!r.source_url.empty()) { + Json refs = Json::array(); + Json one = Json::object(); + one.set("uri", Json(r.source_url)); + one.set("tag", Json("source")); + refs.push(one); + b.set("referenceImages", refs); + } + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json RunwayImageGenerator::plan(const ImageRequest& request) const { + return client_->plan("text_to_image", body(request)).summary(); +} + +JobTicket RunwayImageGenerator::submit(const ImageRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "runway", "generate"); + return RunwayJobs::ticket_from(client_->submit("text_to_image", b, true), b.find("model")->as_string(), MediaKind::kImage); +} + +JobStatus RunwayImageGenerator::poll(const JobTicket& t) { return RunwayJobs::poll(*client_, t); } + +bool RunwayImageGenerator::cancel(const JobTicket& t, bool confirm) { + AdapterSupport::need_confirm(confirm, "runway", "cancel task"); + client_->cancel_or_delete_task(t.id, true); + return true; +} + +std::string RunwayVideoGenerator::key_for(const VideoRequest& r) { + return r.image_url.empty() ? "text_to_video" : "image_to_video"; +} + +Json RunwayVideoGenerator::body(const VideoRequest& r) const { + r.validate(); + Json b = Json::object(); + b.set("model", Json(r.model.empty() ? (r.image_url.empty() ? "gen4.5" : "gen4_turbo") : r.model)); + AdapterSupport::put(b, "promptText", r.prompt); + AdapterSupport::put(b, "ratio", r.aspect_ratio); + AdapterSupport::put_int(b, "duration", r.duration_seconds); + AdapterSupport::put_seed(b, "seed", r.seed); + AdapterSupport::put(b, "negativePrompt", r.negative_prompt); + AdapterSupport::put(b, "resolution", r.resolution); + AdapterSupport::put(b, "promptImage", r.image_url); + if (!r.end_image_url.empty()) + throw llm::ApiError::invalid("runway", "end frames use per-model promptImage arrays: pass them in extra"); + AdapterSupport::merge_extra(b, r.extra); + return b; +} + +Json RunwayVideoGenerator::plan(const VideoRequest& request) const { + return client_->plan(key_for(request), body(request)).summary(); +} + +JobTicket RunwayVideoGenerator::submit(const VideoRequest& request, bool confirm) { + Json b = body(request); + AdapterSupport::need_confirm(confirm, "runway", "generate"); + return RunwayJobs::ticket_from(client_->submit(key_for(request), b, true), b.find("model")->as_string(), MediaKind::kVideo); +} + +JobStatus RunwayVideoGenerator::poll(const JobTicket& t) { return RunwayJobs::poll(*client_, t); } + +bool RunwayVideoGenerator::cancel(const JobTicket& t, bool confirm) { + AdapterSupport::need_confirm(confirm, "runway", "cancel task"); + client_->cancel_or_delete_task(t.id, true); + return true; +} + +} // namespace molo::media diff --git a/src/vendors/media/runway/runway.h b/src/vendors/media/runway/runway.h new file mode 100644 index 0000000..a1e63d3 --- /dev/null +++ b/src/vendors/media/runway/runway.h @@ -0,0 +1,93 @@ +#pragma once + +// Runway developer API (https://api.dev.runwayml.com): text / image / video +// generation, tasks, uploads, organization, avatars, voices, documents, +// routers, workflows, realtime sessions. Bearer auth plus the required +// X-Runway-Version header. The key is RUNWAY_API_KEY (the official SDK reads +// RUNWAYML_API_SECRET; both are accepted). Runway publishes no OpenAPI +// document: the table is checked against the official SDK source. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/runway/calls.h" + +namespace molo::media { + +class RunwayClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.dev.runwayml.com"; + static constexpr const char* kKeyEnv = "RUNWAY_API_KEY"; + static constexpr const char* kKeyEnvSdk = "RUNWAYML_API_SECRET"; + static constexpr const char* kApiVersion = "2024-11-06"; // SDK default X-Runway-Version + + explicit RunwayClient(std::shared_ptr client) + : MediaService(std::move(client), RunwayCalls::table()) {} + static llm::ApiClientConfig config() { + llm::ApiClientConfig c = ClientFactory::bearer("runway", kHost, kKeyEnv); + c.default_headers["X-Runway-Version"] = kApiVersion; + return c; + } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv, kKeyEnvSdk}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // Paid generation rows (text_to_image, text_to_video, image_to_video, ...). Returns {id, ...}. + Json submit(const std::string& key, const Json& body, bool confirm); + PreparedCall plan(const std::string& key, const Json& body) const; + Json task(const std::string& id); + Json cancel_or_delete_task(const std::string& id, bool confirm); // no default + Json organization(); + Json organization_usage(const Json& body); // POST but a query: effect read +}; + +class RunwayImageGenerator : public ImageGenerator { + public: + explicit RunwayImageGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "runway"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"gen4_image", "gen4_image_turbo", "gemini_2.5_flash", "ideogram_4_5", "gpt_image_2", "seedream5_pro"}; + } + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 3000; } + + private: + Json body(const ImageRequest& request) const; + std::shared_ptr client_; +}; + +class RunwayVideoGenerator : public VideoGenerator { + public: + explicit RunwayVideoGenerator(std::shared_ptr c) : client_(std::move(c)) {} + std::string provider() const override { return "runway"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override { + return {"gen4.5", "gen4_turbo", "veo3.1", "veo3.1_fast", "kling3.0_pro", "seedance2", "hailuo3"}; + } + Json plan(const VideoRequest& request) const override; + JobTicket submit(const VideoRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket& ticket, bool confirm) override; + int suggested_poll_ms() const override { return 5000; } + + static std::string key_for(const VideoRequest& request); + + private: + Json body(const VideoRequest& request) const; + std::shared_ptr client_; +}; + +struct RunwayJobs { + static JobStatus poll(RunwayClient& client, const JobTicket& ticket); + static JobStatus status_of(const Json& task, MediaKind kind); + static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind); +}; + +} // namespace molo::media diff --git a/src/vendors/media/stability/calls.cpp b/src/vendors/media/stability/calls.cpp new file mode 100644 index 0000000..77eb2b9 --- /dev/null +++ b/src/vendors/media/stability/calls.cpp @@ -0,0 +1,61 @@ +#include "vendors/media/stability/calls.h" + +namespace molo::media { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// key, http method, path, spec operation id, body kind, body fields, query params, effect) +static const CallRow kRows[] = { + {"3d.stable-fast-3d", "POST", "/v2beta/3d/stable-fast-3d", "", "multipart", "image,texture_resolution,foreground_ratio,remesh,vertex_count", "", "paid"}, + {"3d.stable-point-aware-3d", "POST", "/v2beta/3d/stable-point-aware-3d", "", "multipart", "image,texture_resolution,foreground_ratio,remesh,target_type,target_count,guidance_scale,seed", "", "paid"}, + {"results.id", "GET", "/v2beta/results/{id}", "", "none", "", "", "read"}, + {"audio.stable-audio-2.text-to-audio", "POST", "/v2beta/audio/stable-audio-2/text-to-audio", "", "multipart", "prompt,duration,seed,steps,cfg_scale,model,output_format", "", "paid"}, + {"audio.stable-audio-2.audio-to-audio", "POST", "/v2beta/audio/stable-audio-2/audio-to-audio", "", "multipart", "prompt,duration,seed,steps,cfg_scale,model,output_format,audio,strength", "", "paid"}, + {"audio.stable-audio-2.inpaint", "POST", "/v2beta/audio/stable-audio-2/inpaint", "", "multipart", "prompt,duration,seed,steps,output_format,audio,mask_start,mask_end", "", "paid"}, + {"audio.stable-audio.text-to-audio", "POST", "/v2beta/audio/stable-audio/text-to-audio", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format", "", "paid"}, + {"audio.stable-audio.audio-to-audio", "POST", "/v2beta/audio/stable-audio/audio-to-audio", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format,strength,audio", "", "paid"}, + {"audio.stable-audio.inpaint", "POST", "/v2beta/audio/stable-audio/inpaint", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format,mask_start,mask_end,audio", "", "paid"}, + {"audio.results.id", "GET", "/v2beta/audio/results/{id}", "fetchAudioResult", "none", "", "", "read"}, + {"stable-image.upscale.conservative", "POST", "/v2beta/stable-image/upscale/conservative", "", "multipart", "image,prompt,negative_prompt,seed,output_format,creativity", "", "paid"}, + {"stable-image.upscale.creative", "POST", "/v2beta/stable-image/upscale/creative", "", "multipart", "image,prompt,negative_prompt,output_format,seed,creativity,style_preset", "", "paid"}, + {"stable-image.upscale.creative.result.id", "GET", "/v2beta/stable-image/upscale/creative/result/{id}", "", "none", "", "", "read"}, + {"stable-image.upscale.fast", "POST", "/v2beta/stable-image/upscale/fast", "", "multipart", "image,output_format", "", "paid"}, + {"stable-image.edit.erase", "POST", "/v2beta/stable-image/edit/erase", "", "multipart", "image,mask,grow_mask,seed,output_format", "", "paid"}, + {"stable-image.edit.inpaint", "POST", "/v2beta/stable-image/edit/inpaint", "", "multipart", "image,prompt,negative_prompt,mask,grow_mask,seed,output_format,style_preset", "", "paid"}, + {"stable-image.edit.outpaint", "POST", "/v2beta/stable-image/edit/outpaint", "", "multipart", "image,left,right,up,down,creativity,prompt,seed,output_format,style_preset", "", "paid"}, + {"stable-image.edit.search-and-replace", "POST", "/v2beta/stable-image/edit/search-and-replace", "", "multipart", "image,prompt,search_prompt,negative_prompt,grow_mask,seed,output_format,style_preset", "", "paid"}, + {"stable-image.edit.search-and-recolor", "POST", "/v2beta/stable-image/edit/search-and-recolor", "", "multipart", "image,prompt,select_prompt,negative_prompt,grow_mask,seed,output_format,style_preset", "", "paid"}, + {"stable-image.edit.remove-background", "POST", "/v2beta/stable-image/edit/remove-background", "", "multipart", "image,output_format", "", "paid"}, + {"stable-image.edit.replace-background-and-relight", "POST", "/v2beta/stable-image/edit/replace-background-and-relight", "", "multipart", "subject_image,background_reference,background_prompt,foreground_prompt,negative_prompt,preserve_original_subject,original_background_depth,keep_original_background,light_source_direction,light_reference,light_source_strength,seed,output_format", "", "paid"}, + {"stable-image.generate.ultra", "POST", "/v2beta/stable-image/generate/ultra", "", "multipart", "prompt,negative_prompt,aspect_ratio,seed,output_format,image,style_preset,strength", "", "paid"}, + {"stable-image.generate.core", "POST", "/v2beta/stable-image/generate/core", "", "multipart", "prompt,aspect_ratio,negative_prompt,seed,style_preset,output_format", "", "paid"}, + {"stable-image.generate.sd3", "POST", "/v2beta/stable-image/generate/sd3", "", "multipart", "prompt,mode,image,strength,aspect_ratio,model,seed,output_format,style_preset,negative_prompt,cfg_scale", "", "paid"}, + {"stable-image.control.sketch", "POST", "/v2beta/stable-image/control/sketch", "", "multipart", "prompt,image,control_strength,negative_prompt,seed,output_format,style_preset", "", "paid"}, + {"stable-image.control.structure", "POST", "/v2beta/stable-image/control/structure", "", "multipart", "prompt,image,control_strength,negative_prompt,seed,output_format,style_preset", "", "paid"}, + {"stable-image.control.style", "POST", "/v2beta/stable-image/control/style", "", "multipart", "prompt,image,negative_prompt,aspect_ratio,fidelity,seed,output_format,style_preset", "", "paid"}, + {"stable-image.control.style-transfer", "POST", "/v2beta/stable-image/control/style-transfer", "", "multipart", "init_image,style_image,prompt,negative_prompt,seed,style_strength,composition_fidelity,change_strength,output_format", "", "paid"}, + {"v1.generation.id.text-to-image", "POST", "/v1/generation/{engine_id}/text-to-image", "textToImage", "json", "height,width,text_prompts,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"}, + {"v1.generation.id.image-to-image", "POST", "/v1/generation/{engine_id}/image-to-image", "imageToImage", "multipart", "text_prompts,init_image,init_image_mode,image_strength,step_schedule_start,step_schedule_end,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"}, + {"v1.generation.id.image-to-image.masking", "POST", "/v1/generation/{engine_id}/image-to-image/masking", "masking", "multipart", "init_image,mask_source,mask_image,text_prompts,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"}, + {"v1.engines.list", "GET", "/v1/engines/list", "listEngines", "none", "", "", "read"}, + {"v1.user.account", "GET", "/v1/user/account", "userAccount", "none", "", "", "read"}, + {"v1.user.balance", "GET", "/v1/user/balance", "userBalance", "none", "", "", "read"}, +}; +// CALLS-END + +// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read) +static const char* const kFields[][2] = { + {"AsyncGenerationResponse", "id"}, + {"ImageJSONResponse", "image,finish_reason,seed"}, + {"AudioJSONResponse", "audio,finish_reason,seed"}, +}; +// FIELDS-END + +const CallTable& StabilityCalls::table() { + static const CallTable t("stability", kRows, sizeof(kRows) / sizeof(kRows[0])); + return t; +} + +std::size_t StabilityCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); } +const char* const* StabilityCalls::field_row(std::size_t i) { return kFields[i]; } + +} // namespace molo::media diff --git a/src/vendors/media/stability/calls.h b/src/vendors/media/stability/calls.h new file mode 100644 index 0000000..fc27f42 --- /dev/null +++ b/src/vendors/media/stability/calls.h @@ -0,0 +1,19 @@ +#pragma once + +// The ONE table of REST calls for this provider. tools/check_spec.py checks +// every row against the published spec. Code builds every path from here. + +#include + +#include "vendors/media/common/call_table.h" + +namespace molo::media { + +struct StabilityCalls { + static const CallTable& table(); + // Response fields our parsers read, per schema (checked by check_spec.py). + static std::size_t field_count(); + static const char* const* field_row(std::size_t index); +}; + +} // namespace molo::media diff --git a/src/vendors/media/stability/stability.cpp b/src/vendors/media/stability/stability.cpp new file mode 100644 index 0000000..43ccde1 --- /dev/null +++ b/src/vendors/media/stability/stability.cpp @@ -0,0 +1,141 @@ +#include "vendors/media/stability/stability.h" + +#include "vendors/media/common/adapter_support.h" + +namespace molo::media { + +namespace { + +CallArgs form(const Json& fields, const std::vector& files) { + CallArgs a; + a.body = fields; + a.files = files; + a.headers["Accept"] = "application/json"; + return a; +} + +CallArgs by_id(const std::string& id) { + CallArgs a; + a.path["id"] = id; + a.headers["Accept"] = "application/json"; + return a; +} + +} // namespace + +PreparedCall StabilityClient::plan(const std::string& key, const Json& fields, + const std::vector& files) const { + return prepare(key, form(fields, files)); +} + +Json StabilityClient::submit(const std::string& key, const Json& fields, const std::vector& files, + bool confirm) { + return call_json(key, form(fields, files), confirm); +} + +Json StabilityClient::result(const std::string& id) { return call_json("results.id", by_id(id)); } +Json StabilityClient::audio_result(const std::string& id) { return call_json("audio.results.id", by_id(id)); } +Json StabilityClient::creative_upscale_result(const std::string& id) { + return call_json("stable-image.upscale.creative.result.id", by_id(id)); +} +Json StabilityClient::engines() { return call_json("v1.engines.list"); } +Json StabilityClient::account() { return call_json("v1.user.account"); } +Json StabilityClient::balance() { return call_json("v1.user.balance"); } + +Json StabilityClient::v1_text_to_image(const std::string& engine_id, const Json& body, bool confirm) { + CallArgs a; + a.path["engine_id"] = engine_id; + a.body = body; + a.headers["Accept"] = "application/json"; + return call_json("v1.generation.id.text-to-image", a, confirm); +} + +std::vector StabilityImageGenerator::models() const { + return {"core", "ultra", "sd3.5-large", "sd3.5-large-turbo", "sd3.5-medium", "sd3.5-flash"}; +} + +std::string StabilityImageGenerator::key_for(const std::string& model) { + if (model.empty() || model == "core") return "stable-image.generate.core"; + if (model == "ultra") return "stable-image.generate.ultra"; + if (model.rfind("sd3", 0) == 0) return "stable-image.generate.sd3"; + throw llm::ApiError::invalid("stability", "unknown model '" + model + "' (core, ultra, sd3*)"); +} + +Json StabilityImageGenerator::fields(const ImageRequest& r, std::string& key) const { + r.validate(); + key = key_for(r.model); + if (r.count != 1) throw llm::ApiError::invalid("stability", "one image per call (count must be 1)"); + if (r.width || r.height) throw llm::ApiError::invalid("stability", "use aspect_ratio, not width / height"); + if (!r.source_url.empty()) + throw llm::ApiError::invalid("stability", "source images are uploaded as files: use StabilityClient::submit with a FilePart"); + Json f = Json::object(); + AdapterSupport::put(f, "prompt", r.prompt); + AdapterSupport::put(f, "negative_prompt", r.negative_prompt); + AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio); + if (r.seed >= 0) f.set("seed", Json(static_cast(r.seed > 4294967294L ? 4294967294L : r.seed))); + if (key == "stable-image.generate.sd3" && r.model != "sd3") f.set("model", Json(r.model)); + f.set("output_format", Json("png")); + AdapterSupport::merge_extra(f, r.extra); + return f; +} + +Json StabilityImageGenerator::plan(const ImageRequest& request) const { + std::string key; + Json f = fields(request, key); + return client_->plan(key, f, {}).summary(); +} + +JobTicket StabilityImageGenerator::submit(const ImageRequest& request, bool confirm) { + std::string key; + Json f = fields(request, key); + AdapterSupport::need_confirm(confirm, "stability", "generate"); + Json out = client_->submit(key, f, {}, true); + JobTicket t; + t.provider = "stability"; + t.model = request.model.empty() ? "core" : request.model; + t.kind = MediaKind::kImage; + if (const Json* id = out.find("id"); id && id->is_string()) { + t.id = id->as_string(); + return t; + } + std::string b64 = AdapterSupport::str(out, "image"); + if (b64.empty()) throw llm::ApiError("stability", 502, "BAD_RESPONSE", "no image and no id in the reply"); + MediaAsset a; + a.kind = MediaKind::kImage; + a.data_base64 = b64; + a.mime = "image/" + (f.find("output_format") ? f.find("output_format")->as_string() : std::string("png")); + a.label = "seed=" + AdapterSupport::str(out, "seed") + " finish_reason=" + AdapterSupport::str(out, "finish_reason"); + t.id = "immediate"; + t.immediate = true; + t.assets.push_back(a); + return t; +} + +JobStatus StabilityImageGenerator::poll(const JobTicket& ticket) { + JobStatus s; + if (ticket.immediate) { + s.state = JobState::kSucceeded; + s.raw_state = "immediate"; + s.assets = ticket.assets; + return s; + } + Json out = client_->result(ticket.id); + std::string b64 = AdapterSupport::str(out, "image"); + if (b64.empty()) { + s.state = JobState::kRunning; + s.raw_state = AdapterSupport::str(out, "status"); + if (s.raw_state.empty()) s.raw_state = "in-progress"; + return s; + } + MediaAsset a; + a.kind = MediaKind::kImage; + a.data_base64 = b64; + a.mime = "image/png"; + a.label = "seed=" + AdapterSupport::str(out, "seed") + " finish_reason=" + AdapterSupport::str(out, "finish_reason"); + s.state = JobState::kSucceeded; + s.raw_state = "complete"; + s.assets.push_back(a); + return s; +} + +} // namespace molo::media diff --git a/src/vendors/media/stability/stability.h b/src/vendors/media/stability/stability.h new file mode 100644 index 0000000..81b1654 --- /dev/null +++ b/src/vendors/media/stability/stability.h @@ -0,0 +1,67 @@ +#pragma once + +// Stability AI: v2beta Stable Image (generate, edit, upscale, control), 3D, +// Stable Audio, and the v1 engines / account surface. Every v2beta call is +// multipart/form-data; we always ask for JSON (base64 images) so errors and +// results parse the same way. + +#include "vendors/media/common/client_factory.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" +#include "vendors/media/stability/calls.h" + +namespace molo::media { + +class StabilityClient : public MediaService { + public: + static constexpr const char* kHost = "https://api.stability.ai"; + static constexpr const char* kKeyEnv = "STABILITY_API_KEY"; + + explicit StabilityClient(std::shared_ptr client) + : MediaService(std::move(client), StabilityCalls::table()) {} + static llm::ApiClientConfig config() { return ClientFactory::bearer("stability", kHost, kKeyEnv); } + static std::shared_ptr from_env() { + return std::make_shared(ClientFactory::real(config(), ClientFactory::env({kKeyEnv}))); + } + static std::shared_ptr with(std::unique_ptr t, const std::string& key) { + return std::make_shared(ClientFactory::with(std::move(t), config(), key)); + } + + // Any v2beta POST row (stable-image.*, 3d.*, audio.*). Paid: confirm required. + // Returns {image|audio, finish_reason, seed} or, for async rows, {id}. + Json submit(const std::string& key, const Json& fields, const std::vector& files, bool confirm); + PreparedCall plan(const std::string& key, const Json& fields, const std::vector& files) const; + + // Polls: HTTP 202 {id,status} while running, 200 with the result when done. + Json result(const std::string& id); // /v2beta/results/{id} + Json audio_result(const std::string& id); // /v2beta/audio/results/{id} + Json creative_upscale_result(const std::string& id); // legacy creative upscale path + + // v1 reads. + Json engines(); + Json account(); + Json balance(); + // v1 text-to-image (JSON body, artifacts[] with base64). + Json v1_text_to_image(const std::string& engine_id, const Json& body, bool confirm); +}; + +class StabilityImageGenerator : public ImageGenerator { + public: + explicit StabilityImageGenerator(std::shared_ptr client) : client_(std::move(client)) {} + std::string provider() const override { return "stability"; } + bool configured() const override { return client_->configured(); } + std::vector models() const override; + Json plan(const ImageRequest& request) const override; + JobTicket submit(const ImageRequest& request, bool confirm) override; + JobStatus poll(const JobTicket& ticket) override; + bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the API + + // model -> table key (ultra, core, sd3*), "" model = core. + static std::string key_for(const std::string& model); + + private: + Json fields(const ImageRequest& request, std::string& key) const; + std::shared_ptr client_; +}; + +} // namespace molo::media diff --git a/src/vendors/media/tools/check_spec.py b/src/vendors/media/tools/check_spec.py new file mode 100644 index 0000000..fe9adfd --- /dev/null +++ b/src/vendors/media/tools/check_spec.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +"""Check the media call tables against what each vendor publishes. + +For every provider the checker reads the rows between CALLS-BEGIN / CALLS-END +in src/vendors/media//calls.cpp (and the response fields between +FIELDS-BEGIN / FIELDS-END) and compares them with the vendor's own source: + + openapi Stability (v2beta + v1), Black Forest Labs, Replicate, fal (queue + spec from the docs, Platform API), Ideogram v2, Recraft, Higgsfield + sdk Runway and Luma: the official npm package (resources/*.js for the + paths, resources/*.d.ts for the request fields) + docs Leonardo v2 (the OpenAPI document embedded in three reference + pages), MiniMax (the OpenAPI embedded in the API reference pages) + probe Kling (no machine-readable source exists): the path must answer + 401/400/403 unauthenticated, never 404. Reported as UNVERIFIED. + +It fails (exit 1) when + row a declared (method, path) is not an operation of the source + field a request body field or query parameter we send is not in the source + kind the declared body kind disagrees with the source content types + schema a response field we read is not reachable from the named schema + coverage a source operation is neither in the table nor in EXCLUSIONS + +Python 3 standard library only. No credentials. Network required. + + python3 -I src/vendors/media/tools/check_spec.py [provider ...] +""" +import io +import json +import os +import re +import sys +import tarfile +import urllib.error +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.normpath(os.path.join(HERE, "..")) +STR = re.compile(r'"((?:[^"\\]|\\.)*)"') +UA = {"User-Agent": "molo-media-spec-check/1"} + +# (provider, source operation) -> reason. Everything else in a source must be in the table. +EXCLUSIONS = { + "stability": { + ("POST", "/v2alpha/generation/stable-image/upscale"): "alpha endpoint, superseded by v2beta upscale", + ("GET", "/v2alpha/generation/stable-image/upscale/result/{id}"): "alpha endpoint", + ("POST", "/v2alpha/generation/stable-image/inpaint"): "alpha endpoint, superseded by v2beta edit/inpaint", + }, + "fal": { + ("GET", "/{owner}/{app}/requests/{request_id}/status/stream"): "server-sent events; ApiClient streams POST only (gap)", + }, +} +# fal Platform API operations outside model inference: out of scope for package media. +FAL_PLATFORM_SKIP_PREFIXES = ("/workflows", "/assets", "/storage", "/serverless", "/compute", "/keys", "/account", + "/organization") +# Rows whose path is generic by design (the vendor says its OpenAPI file is supplementary). +GENERIC_ROWS = {"higgsfield": {"model.submit"}} +# Rows checked by documentation prose only (the reference page was removed). Listed, never silent. +PROSE_ONLY = {"leonardo": {"v1.generations.create", "v1.generations.get", "v1.me"}} + +problems = [] +notes = [] +cache = {} + + +def fail(provider, kind, msg): + problems.append("%s: %s: %s" % (provider, kind, msg)) + + +def fetch(url, binary=False): + if url not in cache: + req = urllib.request.Request(url, headers=UA) + with urllib.request.urlopen(req, timeout=90) as r: + cache[url] = r.read() + data = cache[url] + return data if binary else data.decode("utf-8", "replace") + + +def rows_of(provider): + text = open(os.path.join(ROOT, provider, "calls.cpp")).read() + a, b = text.index("// CALLS-BEGIN"), text.index("// CALLS-END") + rows = [] + for line in text[a:b].splitlines()[2:]: + s = line.strip() + if s.startswith("{") and '"' in s: + f = STR.findall(s) + assert len(f) == 8, (provider, s) + rows.append(dict(zip(("key", "method", "path", "op", "kind", "body", "query", "effect"), f))) + c, d = text.index("// FIELDS-BEGIN"), text.index("// FIELDS-END") + fields = [] + for line in text[c:d].splitlines()[1:]: + s = line.strip() + if s.startswith("{") and '"' in s: + fields.append(STR.findall(s)) + for r in rows: + r["body"] = [x for x in r["body"].split(",") if x] + r["query"] = [x for x in r["query"].split(",") if x] + return rows, fields + + +# ---------------------------------------------------------------- OpenAPI ---- +class Oas: + def __init__(self, doc): + self.d = doc + + def ref(self, node): + n = 0 + while isinstance(node, dict) and "$ref" in node and n < 25: + cur = self.d + for part in node["$ref"].lstrip("#/").split("/"): + cur = cur[part.replace("~1", "/").replace("~0", "~")] + node = cur + n += 1 + return node + + def props(self, schema, depth=0): + schema = self.ref(schema) + out = [] + if not isinstance(schema, dict) or depth > 6: + return out + for name in schema.get("properties") or {}: + out.append(name) + for key in ("allOf", "oneOf", "anyOf"): + for sub in schema.get(key) or []: + out.extend(self.props(sub, depth + 1)) + return out + + def reachable(self, schema, depth=0, seen=None): + """Every property name reachable from a schema (for response fields).""" + seen = seen if seen is not None else set() + out = set() + if depth > 6: + return out + if isinstance(schema, dict) and "$ref" in schema: + if schema["$ref"] in seen: + return out + seen.add(schema["$ref"]) + schema = self.ref(schema) + if not isinstance(schema, dict): + return out + for name, sub in (schema.get("properties") or {}).items(): + out.add(name) + out |= self.reachable(sub, depth + 1, seen) + for key in ("allOf", "oneOf", "anyOf"): + for sub in schema.get(key) or []: + out |= self.reachable(sub, depth + 1, seen) + if "items" in schema: + out |= self.reachable(schema["items"], depth + 1, seen) + ap = schema.get("additionalProperties") + if isinstance(ap, dict): + out |= self.reachable(ap, depth + 1, seen) + return out + + def ops(self): + for path, item in self.d["paths"].items(): + for m, op in item.items(): + if m in ("get", "post", "put", "patch", "delete"): + yield m.upper(), path, op + + def op(self, method, path): + item = self.d["paths"].get(path) + return item.get(method.lower()) if item else None + + def params(self, path, op, where): + out = [] + for p in (self.d["paths"][path].get("parameters") or []) + (op.get("parameters") or []): + p = self.ref(p) + if p.get("in") == where: + out.append(p["name"]) + return out + + def body(self, op): + rb = self.ref(op.get("requestBody") or {}) + content = rb.get("content") or {} + kinds, fields = [], [] + for ct, k in (("application/json", "json"), ("multipart/form-data", "multipart"), + ("application/x-www-form-urlencoded", "form")): + if ct in content: + kinds.append(k) + fields.extend(self.props(content[ct].get("schema") or {})) + return kinds, fields + + +def check_rows_against_oas(provider, rows, oas, strip="", only=lambda r: True, path_filter=lambda p: True): + """Compare table rows with one OpenAPI document. Returns the (method, path) set covered.""" + covered = set() + for r in rows: + if not only(r): + continue + if r["key"] in GENERIC_ROWS.get(provider, ()): + notes.append("%s: row %s is generic by design (not in the spec)" % (provider, r["key"])) + continue + path = r["path"][len(strip):] if strip and r["path"].startswith(strip) else r["path"] + op = oas.op(r["method"], path) + if op is None: + fail(provider, "row", "%s %s (%s) is not in the spec" % (r["method"], r["path"], r["key"])) + continue + covered.add((r["method"], path)) + if r["op"] and op.get("operationId") != r["op"]: + fail(provider, "row", "%s declares operation id %s, spec says %s" % (r["key"], r["op"], op.get("operationId"))) + kinds, fields = oas.body(op) + want = r["kind"] + have = "none" if not kinds else (kinds[0] if len(kinds) == 1 else "either") + if want != have: + fail(provider, "kind", "%s declares body kind %s, spec has %s" % (r["key"], want, have)) + for f in r["body"]: + if f != "*" and f not in fields: + fail(provider, "field", "%s sends body field %s, absent from the spec" % (r["key"], f)) + q = oas.params(path, op, "query") + for f in r["query"]: + if f not in q: + fail(provider, "field", "%s sends query parameter %s, absent from the spec" % (r["key"], f)) + return covered + + +def check_coverage(provider, oas, covered, strip="", path_filter=lambda p: True): + excl = EXCLUSIONS.get(provider, {}) + n = 0 + for m, p, _op in oas.ops(): + if not path_filter(p): + continue + n += 1 + if (m, p) in covered or (m, p) in excl: + continue + fail(provider, "coverage", "spec operation %s %s is neither in the table nor excluded" % (m, p)) + return n + + +def check_fields(provider, fields, oas_list): + for schema, flist in fields: + want = [x for x in flist.split(",") if x] + ok = False + for oas in oas_list: + schemas = oas.d.get("components", {}).get("schemas", {}) + if schema in schemas: + have = oas.reachable({"$ref": "#/components/schemas/" + schema}) + missing = [f for f in want if f not in have] + for f in missing: + fail(provider, "schema", "response field %s is not reachable from schema %s" % (f, schema)) + ok = True + break + if not ok: + fail(provider, "schema", "schema %s not found in the spec" % schema) + + +def summarize(provider, rows, ops, kind): + nb = sum(len(r["body"]) + len(r["query"]) for r in rows) + print(" %-11s %-8s rows=%-3d fields=%-4d source ops=%s" % (provider, kind, len(rows), nb, ops)) + + +# ------------------------------------------------------------- providers ---- +def stability(): + rows, fields = rows_of("stability") + v2 = Oas(json.loads(fetch("https://docs.stability.ai/openapi.json"))) + v1 = Oas(json.loads(fetch("https://docs.stability.ai/openapi-v1.json"))) + c2 = check_rows_against_oas("stability", rows, v2, only=lambda r: r["path"].startswith("/v2beta")) + c1 = check_rows_against_oas("stability", rows, v1, only=lambda r: r["path"].startswith("/v1")) + n = check_coverage("stability", v2, c2) + check_coverage("stability", v1, c1) + check_fields("stability", fields, [v2, v1]) + summarize("stability", rows, n, "openapi") + + +def bfl(): + rows, fields = rows_of("bfl") + oas = Oas(json.loads(fetch("https://api.bfl.ai/openapi.json"))) + cov = check_rows_against_oas("bfl", rows, oas) + n = check_coverage("bfl", oas, cov) + check_fields("bfl", fields, [oas]) + summarize("bfl", rows, n, "openapi") + + +def replicate(): + rows, fields = rows_of("replicate") + oas = Oas(json.loads(fetch("https://api.replicate.com/openapi.json"))) + servers = [s.get("url") for s in oas.d.get("servers", [])] + if servers != ["https://api.replicate.com/v1"]: + fail("replicate", "row", "spec server url changed: %s (kHost assumes https://api.replicate.com/v1)" % servers) + cov = check_rows_against_oas("replicate", rows, oas) + n = check_coverage("replicate", oas, cov) + check_fields("replicate", fields, [oas]) + summarize("replicate", rows, n, "openapi") + + +def fal(): + rows, fields = rows_of("fal") + queue = Oas(json.loads(fetch("https://fal.ai/docs/api-reference/inference-api/openapi/queue.json"))) + plat = Oas(json.loads(fetch("https://api.fal.ai/v1/openapi.json"))) + cq = check_rows_against_oas("fal", rows, queue, only=lambda r: r["key"].startswith("queue.")) + cp = check_rows_against_oas("fal", rows, plat, strip="/v1", only=lambda r: r["key"].startswith("platform.")) + n = check_coverage("fal", queue, cq) + check_coverage( + "fal", plat, cp, path_filter=lambda p: not p.startswith(FAL_PLATFORM_SKIP_PREFIXES)) + skipped = [p for _m, p, _o in plat.ops() if p.startswith(FAL_PLATFORM_SKIP_PREFIXES)] + notes.append("fal: %d Platform API operations outside model inference are out of scope (assets, storage, " + "serverless, compute, keys, account, organization, workflows)" % len(skipped)) + check_fields("fal", fields, [queue, plat]) + summarize("fal", rows, n, "openapi") + + +def ideogram(): + rows, fields = rows_of("ideogram") + oas = Oas(json.loads(fetch("https://developer.ideogram.ai/v2/openapi.json"))) + cov = check_rows_against_oas("ideogram", rows, oas) + n = check_coverage("ideogram", oas, cov) + check_fields("ideogram", fields, [oas]) + summarize("ideogram", rows, n, "openapi") + + +def recraft(): + rows, fields = rows_of("recraft") + oas = Oas(json.loads(fetch("https://www.recraft.ai/docs/api-reference/openapi.json"))) + cov = check_rows_against_oas("recraft", rows, oas) + n = check_coverage("recraft", oas, cov) + check_fields("recraft", fields, [oas]) + summarize("recraft", rows, n, "openapi") + + +def higgsfield(): + rows, fields = rows_of("higgsfield") + oas = Oas(json.loads(fetch("https://docs.higgsfield.ai/docs/openapi.json"))) + cov = check_rows_against_oas("higgsfield", rows, oas) + n = check_coverage("higgsfield", oas, cov) + check_fields("higgsfield", fields, [oas]) + summarize("higgsfield", rows, n, "openapi") + + +# ------------------------------------------------------- docs-embedded ---- +def md_openapi(url): + text = fetch(url) + m = re.search(r"```json\n(.*?)\n```", text, re.S) + return Oas(json.loads(m.group(1))) + + +def leonardo(): + rows, fields = rows_of("leonardo") + pages = {"createGeneration": "creategeneration", "createSyncGeneration": "createsyncgeneration", "getModels": "getmodels"} + docs = [md_openapi("https://docs.leonardo.ai/reference/%s.md" % p) for p in pages.values()] + covered = set() + n = 0 + for oas in docs: + for m, p, _op in oas.ops(): + n += 1 + covered.add((m, "/v2" + p)) + prose = PROSE_ONLY["leonardo"] + for r in rows: + if r["key"] in prose: + continue + hit = None + for oas in docs: + op = oas.op(r["method"], r["path"][len("/v2"):]) + if op is not None: + hit = (oas, op) + if hit is None: + fail("leonardo", "row", "%s %s (%s) is not in the v2 reference" % (r["method"], r["path"], r["key"])) + continue + oas, op = hit + if r["op"] and op.get("operationId") != r["op"]: + fail("leonardo", "row", "%s operation id %s != %s" % (r["key"], r["op"], op.get("operationId"))) + kinds, have = oas.body(op) + if r["body"] and not (set(r["body"]) <= set(have) | {"model", "parameters", "public"}): + fail("leonardo", "field", "%s body fields %s" % (r["key"], r["body"])) + for key in sorted(prose): + notes.append("leonardo: row %s is UNVERIFIED against a spec (v1 reference pages were removed); it is " + "documented in the quick start (POST /v1/generations) and FAQ prose (GET generation by id)" % key) + quick = fetch("https://docs.leonardo.ai/docs/getting-started.md") + if "api/rest/v1/generations" not in quick: + fail("leonardo", "row", "quick start no longer shows POST /api/rest/v1/generations") + summarize("leonardo", rows, n, "docs") + + +MINIMAX_PAGES = ["video-generation-t2v", "video-generation-i2v", "video-generation-fl2v", "video-generation-s2v", + "video-generation-query", "video-generation-v2-create", "video-generation-v2-query", + "video-generation-v2-list", "video-generation-v2-delete"] +MINIMAX_FILE_PAGE = "file-management-retrieve" +MINIMAX_UNSUPPORTED_PAGES = { + "video-generation-v2-h3-context-ir": "H3-Context-IR task (gap)", + "video-generation-v2-regeneration": "video regeneration task (gap)", + "video-agent-create": "Video Agent templates (gap)", + "video-agent-query": "Video Agent templates (gap)", + "video-generation-download": "download helper page, covered by files.retrieve", +} + + +def minimax(): + rows, _fields = rows_of("minimax") + base = "https://platform.minimax.io/docs/api-reference/" + texts = {p: fetch(base + p + ".md") for p in MINIMAX_PAGES + [MINIMAX_FILE_PAGE]} + heads = set() + page_text = {} + for p, t in texts.items(): + for m in re.finditer(r"^````yaml \S+ (get|post|put|patch|delete) (/\S+)$", t, re.M | re.I): + heads.add((m.group(1).upper(), m.group(2))) + page_text.setdefault((m.group(1).upper(), m.group(2)), "") + page_text[(m.group(1).upper(), m.group(2))] += t + for r in rows: + key = (r["method"], r["path"]) + if key not in heads: + fail("minimax", "row", "%s %s (%s) is not an operation in the API reference pages" % (r["method"], r["path"], r["key"])) + continue + t = page_text[key] + for f in r["body"]: + if not re.search(r"^\s+%s:\s*$" % re.escape(f), t, re.M): + fail("minimax", "field", "%s body field %s not found in the reference" % (r["key"], f)) + for f in r["query"]: + if not re.search(r"- name: %s\b" % re.escape(f), t): + fail("minimax", "field", "%s query parameter %s not found in the reference" % (r["key"], f)) + declared = {(r["method"], r["path"]) for r in rows} + for op in sorted(heads): + if op not in declared and op not in EXCLUSIONS["minimax"]: + fail("minimax", "coverage", "reference operation %s %s is neither in the table nor excluded" % op) + index = fetch("https://platform.minimax.io/docs/llms.txt") + for page in re.findall(r"api-reference/((?:video-generation|video-agent)[a-z0-9-]*)\.md", index): + if page in texts or page in MINIMAX_UNSUPPORTED_PAGES: + continue + fail("minimax", "coverage", "index lists reference page %s that is neither checked nor excluded" % page) + for page, why in sorted(MINIMAX_UNSUPPORTED_PAGES.items()): + notes.append("minimax: reference page %s not implemented: %s" % (page, why)) + summarize("minimax", rows, len(heads), "docs") + + +# -------------------------------------------------------------------- SDK ---- +def npm_tarball(pkg): + meta = json.loads(fetch("https://registry.npmjs.org/%s/latest" % pkg)) + data = fetch(meta["dist"]["tarball"], binary=True) + files = {} + with tarfile.open(fileobj=io.BytesIO(data)) as tf: + for m in tf.getmembers(): + if m.isfile() and m.name.startswith("package/") and m.size < 4_000_000: + files[m.name[len("package/"):]] = tf.extractfile(m).read().decode("utf-8", "replace") + return meta["version"], files + + +def sdk_ops(files, prefix, js_ext=".js"): + """(METHOD, normalized path) -> resource file, from `_client.(...)` calls.""" + out = {} + pat = re.compile(r"_client\.(get|post|put|patch|delete|getAPIList)\(\s*(?:\(0,\s*path_1\.path\)\s*`([^`]*)`|'([^']*)'|`([^`]*)`)") + for name, text in files.items(): + if not name.startswith(prefix) or not name.endswith(js_ext) or name.endswith(".d.ts"): + continue + for m in pat.finditer(text): + verb = m.group(1) + p = m.group(2) or m.group(3) or m.group(4) + p = re.sub(r"\$\{[^}]*\}", "{id}", p) + out[("GET" if verb == "getAPIList" else verb.upper(), p)] = name + return out + + +def norm(path): + return re.sub(r"\{[^}]*\}", "{id}", path) + + +def runway_fields(text): + out = [] + for m in re.finditer(r"export declare namespace (\w+Params) \{\n(.*?)\n\}\n", text, re.S): + out += re.findall(r"^ (\w+)\??:", m.group(2), re.M) + for m in re.finditer(r"export interface (\w+Params) \{\n(.*?)\n\}\n", text, re.S): + out += re.findall(r"^ (\w+)\??:", m.group(2), re.M) + return set(out) + + +def luma_fields(text): + out = [] + for m in re.finditer(r"export interface (\w+Params) \{\n(.*?)\n\}\n", text, re.S): + out += re.findall(r"^ (\w+)\??:", m.group(2), re.M) + return set(out) + + +def sdk_check(provider, pkg, rows, prefix, base_strip, field_fn): + version, files = npm_tarball(pkg) + ops = sdk_ops(files, prefix) + notes.append("%s: official SDK %s %s (%d operations found)" % (provider, pkg, version, len(ops))) + declared = set() + for r in rows: + path = r["path"] + key = (r["method"], norm(path)) + declared.add(key) + if key not in ops: + fail(provider, "row", "%s %s (%s) is not called by the SDK" % (r["method"], path, r["key"])) + continue + if r["body"] and r["body"] != ["*"]: + js = ops[key] + dts = files.get(js[:-3] + ".d.ts", "") + have = field_fn(dts) + for f in r["body"]: + if f not in have: + fail(provider, "field", "%s sends body field %s, absent from %s" % (r["key"], f, js[:-3] + ".d.ts")) + for key in sorted(ops): + if key not in declared and key not in EXCLUSIONS.get(provider, {}): + fail(provider, "coverage", "SDK operation %s %s is neither in the table nor excluded" % key) + return version, ops, files + + +def runway(): + rows, fields = rows_of("runway") + version, ops, files = sdk_check("runway", "@runwayml/sdk", rows, "resources/", "", runway_fields) + tasks = files.get("resources/tasks.d.ts", "") + for schema, flist in fields: + for f in [x for x in flist.split(",") if x]: + if not re.search(r"\b%s\??:" % re.escape(f), tasks): + fail("runway", "schema", "task field %s not found in resources/tasks.d.ts" % f) + for line in ("X-Runway-Version",): + if not any(line in t for n, t in files.items() if n.endswith("client.js")): + fail("runway", "row", "SDK no longer sends %s" % line) + m = re.search(r"runwayVersion = '([^']+)'", files.get("client.js", "")) + notes.append("runway: SDK default X-Runway-Version %s (table assumes 2024-11-06)" % (m.group(1) if m else "?")) + if m and m.group(1) != "2024-11-06": + fail("runway", "row", "SDK default X-Runway-Version is %s, RunwayClient::kApiVersion says 2024-11-06" % m.group(1)) + summarize("runway", rows, len(ops), "sdk") + + +def luma(): + rows, fields = rows_of("luma") + version, ops, files = sdk_check("luma", "lumaai", rows, "resources/", "", luma_fields) + gen = files.get("resources/generations/generations.d.ts", "") + for schema, flist in fields: + for f in [x for x in flist.split(",") if x]: + if not any(re.search(r"\b%s\??:" % re.escape(f), t) for n, t in files.items() if n.endswith(".d.ts")): + fail("luma", "schema", "field %s not found in the SDK types" % f) + if "dream-machine/v1" not in files.get("index.js", ""): + fail("luma", "row", "SDK base URL is no longer .../dream-machine/v1") + if "'queued' | 'dreaming' | 'completed' | 'failed'" not in gen: + fail("luma", "schema", "generation states changed in the SDK types") + summarize("luma", rows, len(ops), "sdk") + + +# ------------------------------------------------------------------ probe ---- +def kling(): + rows, _f = rows_of("kling") + bad = 0 + for r in rows: + url = "https://api-singapore.klingai.com" + r["path"].replace("{id}", "0") + data = b"{}" if r["method"] == "POST" else None + req = urllib.request.Request(url, data=data, method=r["method"], headers=dict(UA, **{"Content-Type": "application/json"})) + try: + urllib.request.urlopen(req, timeout=30) + code = 200 + except urllib.error.HTTPError as e: + code = e.code + except Exception as e: # network down: do not pretend + fail("kling", "row", "probe of %s failed: %s" % (r["path"], e)) + continue + if code == 404: + fail("kling", "row", "%s %s answered 404 on the documented host" % (r["method"], r["path"])) + else: + bad += 0 + notes.append("kling: UNVERIFIED. No spec or SDK source exists; %d paths only probed (not 404 unauthenticated). " + "Request and response fields come from community sources." % len(rows)) + summarize("kling", rows, "-", "probe") + + +CHECKS = {"stability": stability, "bfl": bfl, "replicate": replicate, "fal": fal, "ideogram": ideogram, + "recraft": recraft, "leonardo": leonardo, "runway": runway, "luma": luma, "minimax": minimax, + "kling": kling, "higgsfield": higgsfield} + + +def main(): + wanted = sys.argv[1:] or list(CHECKS) + print("media call tables against vendor sources") + for name in wanted: + if name not in CHECKS: + print("unknown provider", name) + return 2 + try: + CHECKS[name]() + except Exception as e: # a vendor source that cannot be read is a failure, loudly + fail(name, "source", "%s: %s" % (type(e).__name__, e)) + for n in notes: + print(" note:", n) + if problems: + print("\nFAILED (%d)" % len(problems)) + for p in problems: + print(" ", p) + return 1 + print("\nOK: every declared row, field and schema matches its source; every source operation is covered or excluded") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/example/ai/media/page_test.cpp b/tests/example/ai/media/page_test.cpp new file mode 100644 index 0000000..d50d6d4 --- /dev/null +++ b/tests/example/ai/media/page_test.cpp @@ -0,0 +1,47 @@ +// Routes: GET /ai/media (page), /status, /calls (no credential needed) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (const char* v : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "RUNWAYML_API_SECRET", "LUMA_API_KEY", + "REPLICATE_API_TOKEN", "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY", + "MINIMAX_API_KEY", "KLING_ACCESS_KEY", "KLING_SECRET_KEY", "HF_API_KEY_ID", "HF_API_KEY_SECRET"}) + unsetenv(v); + Fixture f; + auto page = f.client->get("/ai/media"); + check(page.status == 200, "page 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "html"); + check(contains(page.body, "Media generation") && contains(page.body, "viewport"), "title and mobile viewport"); + check(contains(page.body, "/embedded/ai_media.js"), "script wired"); + check(f.client->post("/ai/media", "{}", "application/json").status == 405, "post page is 405"); + + auto st = f.client->get("/ai/media/status"); + check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without keys"); + check(!contains(st.body, "\"configured\":true"), "no provider is configured"); + for (const char* env : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "LUMA_API_KEY", "REPLICATE_API_TOKEN", + "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY"}) + check(contains(st.body, env), std::string("status names ") + env); + check(contains(st.body, "Midjourney") && contains(st.body, "No public API"), "Midjourney is listed as unavailable with the reason"); + check(contains(st.body, "Pika"), "Pika is listed as unavailable"); + check(contains(st.body, "\"evidence\":\"probe-only\"") && contains(st.body, "\"evidence\":\"openapi\""), "evidence levels shown"); + check(contains(st.body, "flux-2-pro") && contains(st.body, "fal-ai/flux/dev"), "models are listed"); + check(contains(st.body, "\"max_poll_attempts\":240"), "caps are shown"); + check(f.client->post("/ai/media/status", "{}", "application/json").status == 405, "post status is 405"); + + auto all = f.client->get("/ai/media/calls"); + check(all.status == 200 && contains(all.body, "\"provider\":\"stability\"") && contains(all.body, "\"provider\":\"higgsfield\""), + "calls lists every provider"); + auto bfl = f.client->get("/ai/media/calls?provider=bfl"); + check(bfl.status == 200 && contains(bfl.body, "/v1/flux-2-pro") && contains(bfl.body, "\"effect\":\"paid\"") && + contains(bfl.body, "/v1/credits"), + "bfl call table"); + auto fal = f.client->get("/ai/media/calls?provider=fal"); + check(fal.status == 200 && contains(fal.body, "queue.fal.run") == false && contains(fal.body, "/{owner}/{app}/requests/{request_id}/status"), + "fal call table"); + check(f.client->get("/ai/media/calls?provider=midjourney").status == 422, "unknown provider is 422"); + return finish("ai/media/page"); +} diff --git a/tests/example/ai/media/routes_test.cpp b/tests/example/ai/media/routes_test.cpp new file mode 100644 index 0000000..75973f7 --- /dev/null +++ b/tests/example/ai/media/routes_test.cpp @@ -0,0 +1,128 @@ +// Routes: POST /ai/media/* (405, 422, dry runs, 503 without a credential) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + for (const char* v : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "RUNWAYML_API_SECRET", "LUMA_API_KEY", + "REPLICATE_API_TOKEN", "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY", + "MINIMAX_API_KEY", "KLING_ACCESS_KEY", "KLING_SECRET_KEY", "HF_API_KEY_ID", "HF_API_KEY_SECRET"}) + unsetenv(v); + Fixture f; + auto post = [&](const std::string& route, const std::string& body) { + return f.client->post("/ai/media/" + route, body, "application/json"); + }; + const char* posts[] = {"image", "video", "poll", "wait", "call", "account", "fetch", "webhook"}; + for (const char* r : posts) { + std::string route = std::string("/ai/media/") + r; + check(f.client->get(route).status == 405, route + " get is 405"); + check(post(r, "not json").status == 422, route + " bad json is 422"); + check(post(r, "[1]").status == 422, route + " non-object is 422"); + check(post(r, "{}").status == 422, route + " empty body is 422"); + } + check(f.client->get("/ai/media/job").status == 405, "job get is 405"); + check(f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "not json").status == 422, "job bad json is 422"); + + // 422: input rules run before anything else + check(post("image", "{\"provider\":\"midjourney\",\"prompt\":\"x\"}").status == 422, "midjourney has no API"); + check(post("image", "{\"provider\":\"minimax\",\"prompt\":\"x\"}").status == 422, "minimax has no image generation"); + check(post("video", "{\"provider\":\"stability\",\"prompt\":\"x\"}").status == 422, "stability has no video generation"); + check(post("image", "{\"provider\":\"bfl\"}").status == 422, "prompt is required"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":0}").status == 422, "count 0"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":9}").status == 422, "count over 8"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"width\":512}").status == 422, "width without height"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"width\":9000,\"height\":9000}").status == 422, "dimension over the cap"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"extra\":{\"bogus\":1}}").status == 422, "field not in the call table"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"model\":\"flux-9000\"}").status == 422, "model not in the call table"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":2}").status == 422, "bfl takes one image per call"); + check(post("image", "{\"provider\":\"stability\",\"prompt\":\"x\",\"width\":512,\"height\":512}").status == 422, "stability sizes by aspect ratio"); + check(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\"}").status == 422, "fal needs an endpoint id"); + check(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\",\"model\":\"a/../b\"}").status == 422, "bad endpoint id"); + check(post("image", "{\"provider\":\"replicate\",\"prompt\":\"x\",\"model\":\"nope\"}").status == 422, "replicate needs owner/name"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"seed\":\"a\"}").status == 422, "seed must be a number"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"confirm\":\"yes\"}").status == 422, "confirm must be a boolean"); + check(post("video", "{\"provider\":\"runway\"}").status == 422, "video needs a prompt or an image"); + check(post("video", "{\"provider\":\"runway\",\"prompt\":\"x\",\"duration_seconds\":999}").status == 422, "duration over the cap"); + check(post("video", "{\"provider\":\"minimax\",\"prompt\":\"x\",\"aspect_ratio\":\"16:9\"}").status == 422, "minimax v1 has no aspect ratio"); + check(post("poll", "{\"ticket\":{\"provider\":\"bfl\"}}").status == 422, "ticket needs an id"); + check(post("poll", "{\"ticket\":{\"provider\":\"nope\",\"id\":\"1\"}}").status == 422, "ticket with an unknown provider"); + check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_attempts\":31}").status == 422, "demo attempt cap"); + check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_wait_seconds\":61}").status == 422, "demo wait cap"); + check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_attempts\":0}").status == 422, "attempts under 1"); + check(post("call", "{\"provider\":\"bfl\",\"key\":\"nope\"}").status == 422, "unknown call key"); + check(post("call", "{\"provider\":\"bfl\",\"key\":\"get_result\",\"query\":{\"nope\":\"1\"}}").status == 422, "unknown query parameter"); + check(post("call", "{\"provider\":\"replicate\",\"key\":\"predictions.get\",\"path\":{\"prediction_id\":\"../x\"}}").status == 422, "path traversal"); + check(post("call", "{\"provider\":\"replicate\",\"key\":\"predictions.get\"}").status == 422, "missing path argument"); + check(post("call", "{\"provider\":\"bfl\",\"key\":\"credits\",\"body\":{\"x\":1}}").status == 422, "a GET takes no body"); + check(post("call", "{\"provider\":\"stability\",\"key\":\"stable-image.upscale.fast\",\"files\":[{\"field\":\"image\",\"data_base64\":\"@@\"}]}").status == 422, + "bad base64 file"); + check(post("call", "{\"provider\":\"stability\",\"key\":\"stable-image.upscale.fast\",\"files\":[{\"field\":\"picture\",\"data_base64\":\"aGk=\"}]}").status == 422, + "file field not in the table"); + check(post("fetch", "{\"url\":\"http://example.com/a.png\"}").status == 422, "fetch needs https"); + check(post("fetch", "{\"url\":\"https://user:pw@example.com/a.png\"}").status == 422, "fetch refuses embedded credentials"); + check(post("webhook", "{\"secret\":\"s\",\"webhook_id\":\"i\",\"timestamp\":\"1\",\"signature\":\"v1,x\"}").status == 422, "webhook needs a payload"); + + // dry runs: paid or resource-changing routes show the exact request + auto dry = post("image", "{\"provider\":\"bfl\",\"prompt\":\"a cat\",\"width\":1024,\"height\":768}"); + check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "/v1/flux-2-pro") && + contains(dry.body, "\"effect\":\"paid\"") && contains(dry.body, "\"prompt\":\"a cat\""), + "image is a dry run without confirm"); + check(contains(post("image", "{\"provider\":\"stability\",\"prompt\":\"a cat\",\"model\":\"ultra\",\"aspect_ratio\":\"16:9\"}").body, + "/v2beta/stable-image/generate/ultra"), + "stability dry run shows the endpoint"); + check(contains(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\",\"model\":\"fal-ai/flux/dev\"}").body, "\"target\":\"/fal-ai/flux/dev\""), + "fal dry run shows the queue target"); + check(contains(post("image", "{\"provider\":\"replicate\",\"prompt\":\"x\",\"model\":\"black-forest-labs/flux-schnell\"}").body, + "/models/black-forest-labs/flux-schnell/predictions"), + "replicate dry run shows the prediction target"); + check(contains(post("image", "{\"provider\":\"leonardo\",\"prompt\":\"x\",\"model\":\"flux-dev\"}").body, "\"public\":false"), + "leonardo never publishes by default"); + check(contains(post("image", "{\"provider\":\"ideogram\",\"prompt\":\"x\",\"model\":\"ideogram-4\"}").body, "/v2/image/generate/ideogram-4"), + "ideogram dry run"); + check(contains(post("image", "{\"provider\":\"recraft\",\"prompt\":\"x\",\"width\":1024,\"height\":1024}").body, "\"size\":\"1024x1024\""), + "recraft dry run"); + check(contains(post("video", "{\"provider\":\"runway\",\"prompt\":\"waves\",\"duration_seconds\":5}").body, "/v1/text_to_video"), + "runway video dry run"); + check(contains(post("video", "{\"provider\":\"luma\",\"prompt\":\"waves\",\"duration_seconds\":5}").body, "\"duration\":\"5s\""), + "luma video dry run"); + check(contains(post("video", "{\"provider\":\"kling\",\"prompt\":\"waves\"}").body, "/v1/videos/text2video"), "kling video dry run"); + check(contains(post("video", "{\"provider\":\"higgsfield\",\"prompt\":\"waves\",\"model\":\"minimax/hailuo-2.3/standard/text-to-video\"}").body, + "/minimax/hailuo-2.3/standard/text-to-video"), + "higgsfield video dry run"); + check(contains(post("call", "{\"provider\":\"bfl\",\"key\":\"flux-dev\",\"body\":{\"prompt\":\"x\"}}").body, "\"dry_run\":true"), + "a paid table call is a dry run"); + check(contains(post("call", "{\"provider\":\"replicate\",\"key\":\"files.delete\",\"path\":{\"file_id\":\"f1\"}}").body, "\"effect\":\"delete\""), + "a destructive table call is a dry run"); + check(contains(post("call", "{\"provider\":\"bfl\",\"key\":\"delete_finetune\",\"body\":{\"finetune_id\":\"my-ft\"}}").body, "\"dry_run\":true"), + "delete_finetune is a dry run"); + auto cancel = f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "{\"ticket\":{\"provider\":\"luma\",\"id\":\"g1\",\"kind\":\"video\"}}"); + check(cancel.status == 200 && contains(cancel.body, "\"dry_run\":true"), "cancel is a dry run"); + + // pure routes + auto hook = post("webhook", + "{\"secret\":\"whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw\",\"webhook_id\":\"msg_p5jXN8AQM9LWM0D4loKWxJek\"," + "\"timestamp\":\"1614265330\",\"signature\":\"v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=\"," + "\"payload\":\"{\\\"test\\\": 2432232314}\"}"); + check(hook.status == 200 && contains(hook.body, "\"valid\":false"), "a vector from 2021 is stale now, so it is rejected"); + + // 503: reads and confirmed calls need the credential + auto acct = post("account", "{\"provider\":\"bfl\"}"); + check(acct.status == 503 && contains(acct.body, "media_not_configured"), "account without key is 503"); + check(post("account", "{\"provider\":\"minimax\"}").status == 422, "minimax has no free account read"); + check(post("account", "{\"provider\":\"midjourney\"}").status == 422, "unknown provider account"); + check(post("call", "{\"provider\":\"bfl\",\"key\":\"credits\"}").status == 503, "read call without key is 503"); + check(post("call", "{\"provider\":\"fal\",\"key\":\"platform.meta\"}").status == 503, "fal platform read without key is 503"); + check(post("call", "{\"provider\":\"bfl\",\"key\":\"flux-dev\",\"body\":{\"prompt\":\"x\"},\"confirm\":true}").status == 503, + "confirmed paid call without key is 503"); + check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed image without key is 503"); + check(post("video", "{\"provider\":\"runway\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed video without key is 503"); + check(post("video", "{\"provider\":\"kling\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "kling without keys is 503"); + check(post("poll", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"}}").status == 503, "poll without key is 503"); + check(post("wait", "{\"ticket\":{\"provider\":\"fal\",\"id\":\"1\",\"meta\":{\"endpoint\":\"fal-ai/flux/dev\"}}}").status == 503, "wait without key is 503"); + check(f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "{\"ticket\":{\"provider\":\"luma\",\"id\":\"g1\"},\"confirm\":true}").status == 503, + "confirmed cancel without key is 503"); + return finish("ai/media/routes"); +} diff --git a/tests/src/vendors/media/support.h b/tests/src/vendors/media/support.h new file mode 100644 index 0000000..0fb0976 --- /dev/null +++ b/tests/src/vendors/media/support.h @@ -0,0 +1,57 @@ +#pragma once + +// Shared rig for the media tests. A MockTransport only RECORDS the requests +// the real clients build; the few replies it serves are labelled FIXTURE +// (verbatim vendor documentation) or EMPTY (no content, the assertion is on +// our request). Nothing here verifies vendor behavior: see +// src/vendors/media/SPEC.md "Unverified". + +#include +#include + +#include "clients/transports/mock_transport.h" +#include "framework.h" +#include "llm/error.h" +#include "vendors/media/common/job.h" +#include "vendors/media/common/media_service.h" + +namespace media_test { + +using molo::clients::ClientResponse; +using molo::clients::HeaderMap; +using molo::clients::Method; +using molo::clients::transports::MockExpectation; +using molo::clients::transports::MockTransport; +using molo::llm::ApiError; +using molo::media::Json; + +inline ClientResponse reply(const std::string& body, int status = 200, + const std::string& type = "application/json") { + HeaderMap h; + h.set("Content-Type", type); + return ClientResponse(status, h, body); +} + +// EMPTY: a 200 with `{}`; used only to let the real client finish a request +// whose OUTGOING shape the test inspects. +inline ClientResponse empty_ok() { return reply("{}"); } + +inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; } + +// Allows a path to answer several times. +inline void serve(MockTransport* t, Method m, const std::string& path, ClientResponse r, int times = 8) { + MockExpectation e; + e.method = m; + e.path = path; + e.response = std::move(r); + e.times = times; + t->expect(e); +} + +struct Mock { + std::unique_ptr owned = std::make_unique(); + MockTransport* raw = owned.get(); + std::unique_ptr take() { return std::move(owned); } +}; + +} // namespace media_test diff --git a/tests/src/vendors/media/test_adapters.cpp b/tests/src/vendors/media/test_adapters.cpp new file mode 100644 index 0000000..ce5fe75 --- /dev/null +++ b/tests/src/vendors/media/test_adapters.cpp @@ -0,0 +1,361 @@ +// ImageGenerator / VideoGenerator adapters: the exact request each one would +// send (plan), loud refusal of options a provider cannot honour, and the +// rule that submit() without confirm makes no network call. +#include "support.h" +#include "vendors/media/bfl/bfl.h" +#include "vendors/media/common/registry.h" +#include "vendors/media/fal/fal.h" +#include "vendors/media/higgsfield/higgsfield.h" +#include "vendors/media/ideogram/ideogram.h" +#include "vendors/media/kling/kling.h" +#include "vendors/media/leonardo/leonardo.h" +#include "vendors/media/luma/luma.h" +#include "vendors/media/minimax/minimax.h" +#include "vendors/media/recraft/recraft.h" +#include "vendors/media/replicate/replicate.h" +#include "vendors/media/runway/runway.h" +#include "vendors/media/stability/stability.h" + +using namespace molo::media; +using namespace media_test; + +namespace { + +ImageRequest image(const std::string& model = "") { + ImageRequest r; + r.prompt = "a lighthouse at dusk"; + r.model = model; + return r; +} + +VideoRequest video(const std::string& model = "") { + VideoRequest r; + r.prompt = "waves rolling in"; + r.model = model; + return r; +} + +std::string target(const Json& plan) { return plan.find("target")->as_string(); } +std::string body_of(const Json& plan) { return plan.find("body") ? plan.find("body")->dump() : ""; } + +} // namespace + +TEST(media_adapter, stability_picks_the_endpoint_from_the_model) { + ASSERT_EQ(StabilityImageGenerator::key_for(""), std::string("stable-image.generate.core")); + ASSERT_EQ(StabilityImageGenerator::key_for("ultra"), std::string("stable-image.generate.ultra")); + ASSERT_EQ(StabilityImageGenerator::key_for("sd3.5-large"), std::string("stable-image.generate.sd3")); + ASSERT_THROW(StabilityImageGenerator::key_for("dall-e"), ApiError); + Mock m; + StabilityImageGenerator g(StabilityClient::with(m.take(), "KEY")); + ImageRequest r = image("sd3.5-large"); + r.aspect_ratio = "16:9"; + r.seed = 42; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/v2beta/stable-image/generate/sd3")); + ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"16:9\"")); + ASSERT_TRUE(has(body_of(p), "\"model\":\"sd3.5-large\"")); + ASSERT_TRUE(has(body_of(p), "\"seed\":42")); + ImageRequest wh = image(); + wh.width = 512; + wh.height = 512; + ASSERT_THROW(g.plan(wh), ApiError); // stability sizes by aspect ratio only + ImageRequest many = image(); + many.count = 2; + ASSERT_THROW(g.plan(many), ApiError); + ImageRequest src = image(); + src.source_url = "https://example.com/in.png"; + ASSERT_THROW(g.plan(src), ApiError); + ASSERT_THROW(g.submit(image(), false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, bfl_models_and_options) { + Mock m; + BflImageGenerator g(BflClient::with(m.take(), "KEY")); + ImageRequest r = image("flux-kontext-pro"); + r.aspect_ratio = "1:1"; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/v1/flux-kontext-pro")); + ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"1:1\"")); + ImageRequest wh = image("flux-kontext-pro"); // kontext takes aspect_ratio, not width / height + wh.width = 1024; + wh.height = 1024; + ASSERT_THROW(g.plan(wh), ApiError); + ImageRequest def = image(); + def.width = 1024; + def.height = 768; + ASSERT_EQ(target(g.plan(def)), std::string("/v1/flux-2-pro")); // default model + ImageRequest many = image(); + many.count = 3; + ASSERT_THROW(g.plan(many), ApiError); + ImageRequest unknown = image("flux-9000"); + ASSERT_THROW(g.plan(unknown), ApiError); // not in the table + ASSERT_THROW(g.submit(def, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); + BflVideoGenerator v(BflClient::with(std::make_unique(), "KEY")); + VideoRequest vr = video(); + vr.duration_seconds = 5; + vr.aspect_ratio = "16:9"; + Json vp = v.plan(vr); + ASSERT_EQ(target(vp), std::string("/v1/flux-3-video")); + ASSERT_TRUE(has(body_of(vp), "\"duration\":5")); +} + +TEST(media_adapter, replicate_needs_owner_name_and_maps_common_fields_into_input) { + Mock m; + ReplicateImageGenerator g(ReplicateClient::with(m.take(), "KEY")); + ImageRequest r = image("black-forest-labs/flux-schnell"); + r.aspect_ratio = "3:2"; + r.count = 2; + r.seed = 9; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/models/black-forest-labs/flux-schnell/predictions")); + ASSERT_TRUE(has(body_of(p), "\"num_outputs\":2")); + ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"3:2\"")); + ASSERT_THROW(g.plan(image()), ApiError); + ASSERT_THROW(g.plan(image("flux-schnell")), ApiError); + ASSERT_THROW(g.submit(r, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); + ReplicateVideoGenerator v(ReplicateClient::with(std::make_unique(), "KEY")); + VideoRequest vr = video("minimax/video-01"); + vr.image_url = "https://example.com/a.png"; + Json vp = v.plan(vr); + ASSERT_TRUE(has(body_of(vp), "\"image\":\"https://example.com/a.png\"")); +} + +TEST(media_adapter, fal_takes_any_endpoint_id) { + Mock m; + FalImageGenerator g(FalQueue::with(m.take(), "id:secret")); + ImageRequest r = image("fal-ai/flux/dev"); + r.width = 1024; + r.height = 576; + r.count = 2; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/fal-ai/flux/dev")); + ASSERT_TRUE(has(body_of(p), "\"image_size\":{\"height\":576,\"width\":1024}")); + ASSERT_TRUE(has(body_of(p), "\"num_images\":2")); + ASSERT_THROW(g.plan(image()), ApiError); + ASSERT_THROW(g.submit(r, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); + FalVideoGenerator v(FalQueue::with(std::make_unique(), "id:secret")); + VideoRequest vr = video("fal-ai/kling-video/v2.1/master/text-to-video"); + vr.duration_seconds = 5; + Json vp = v.plan(vr); + ASSERT_EQ(target(vp), std::string("/fal-ai/kling-video/v2.1/master/text-to-video")); + ASSERT_TRUE(has(body_of(vp), "\"duration\":\"5\"")); +} + +TEST(media_adapter, ideogram_models_map_to_rows_and_edit_inputs_are_files) { + Mock m; + IdeogramImageGenerator g(IdeogramClient::with(m.take(), "KEY")); + ASSERT_EQ(IdeogramImageGenerator::key_for(""), std::string("image.generate.ideogram-3")); + ImageRequest r = image("ideogram-4"); + r.count = 2; + r.width = 1024; + r.height = 1024; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/v2/image/generate/ideogram-4")); + ASSERT_TRUE(has(body_of(p), "\"resolution\":\"1024x1024\"")); + ImageRequest src = image(); + src.source_url = "https://example.com/in.png"; + ASSERT_THROW(g.plan(src), ApiError); + ASSERT_THROW(g.plan(image("ideogram-99")), ApiError); + ASSERT_THROW(g.submit(image(), false), ApiError); + IdeogramVideoGenerator v(IdeogramClient::with(std::make_unique(), "KEY")); + ASSERT_THROW(v.plan(video()), ApiError); // a video model is required + Json vp = v.plan(video("kling-3-standard-text-to-video")); + ASSERT_EQ(target(vp), std::string("/v2/video/generate/kling-3-standard-text-to-video")); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, recraft_sizes_by_wxh) { + Mock m; + RecraftImageGenerator g(RecraftClient::with(m.take(), "KEY")); + ImageRequest r = image("recraftv3"); + r.width = 1024; + r.height = 1024; + r.count = 2; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/v1/images/generations")); + ASSERT_TRUE(has(body_of(p), "\"size\":\"1024x1024\"")); + ASSERT_TRUE(has(body_of(p), "\"n\":2")); + ImageRequest ar = image(); + ar.aspect_ratio = "16:9"; + ASSERT_THROW(g.plan(ar), ApiError); + ASSERT_THROW(g.submit(r, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, leonardo_wraps_parameters_and_never_publishes) { + Mock m; + LeonardoImageGenerator g(LeonardoClient::with(m.take(), "KEY")); + ImageRequest r = image("flux-dev"); + r.width = 1024; + r.height = 1024; + r.count = 4; + Json p = g.plan(r); + ASSERT_EQ(target(p), std::string("/v2/generations")); + ASSERT_TRUE(has(body_of(p), "\"model\":\"flux-dev\"")); + ASSERT_TRUE(has(body_of(p), "\"public\":false")); + ASSERT_TRUE(has(body_of(p), "\"quantity\":4")); + ASSERT_THROW(g.plan(image()), ApiError); // model required + ImageRequest ar = image("flux-dev"); + ar.aspect_ratio = "1:1"; + ASSERT_THROW(g.plan(ar), ApiError); + ASSERT_THROW(g.submit(r, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, runway_chooses_text_or_image_to_video) { + Mock m; + RunwayVideoGenerator g(RunwayClient::with(m.take(), "KEY")); + VideoRequest t = video(); + t.duration_seconds = 5; + t.aspect_ratio = "1280:720"; + ASSERT_EQ(target(g.plan(t)), std::string("/v1/text_to_video")); + ASSERT_TRUE(has(body_of(g.plan(t)), "\"model\":\"gen4.5\"")); + VideoRequest i = video(); + i.image_url = "https://example.com/a.png"; + ASSERT_EQ(target(g.plan(i)), std::string("/v1/image_to_video")); + ASSERT_TRUE(has(body_of(g.plan(i)), "\"model\":\"gen4_turbo\"")); + ASSERT_TRUE(has(body_of(g.plan(i)), "\"promptImage\"")); + RunwayImageGenerator ig(RunwayClient::with(std::make_unique(), "KEY")); + ImageRequest r = image(); + r.width = 1280; + r.height = 720; + Json p = ig.plan(r); + ASSERT_EQ(target(p), std::string("/v1/text_to_image")); + ASSERT_TRUE(has(body_of(p), "\"ratio\":\"1280:720\"")); + ASSERT_TRUE(has(body_of(p), "\"promptText\"")); + ASSERT_THROW(g.submit(t, false), ApiError); + ASSERT_THROW(ig.cancel(JobTicket{}, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, luma_durations_are_strings_and_keyframes_nest_frames) { + Mock m; + LumaVideoGenerator g(LumaClient::with(m.take(), "KEY")); + VideoRequest v = video(); + v.duration_seconds = 5; + v.image_url = "https://example.com/start.png"; + v.end_image_url = "https://example.com/end.png"; + Json p = g.plan(v); + ASSERT_EQ(target(p), std::string("/generations/video")); + ASSERT_TRUE(has(body_of(p), "\"duration\":\"5s\"")); + ASSERT_TRUE(has(body_of(p), "\"frame0\":{\"type\":\"image\",\"url\":\"https://example.com/start.png\"}")); + ASSERT_TRUE(has(body_of(p), "\"frame1\"")); + ASSERT_TRUE(has(body_of(p), "\"model\":\"ray-2\"")); + LumaImageGenerator ig(LumaClient::with(std::make_unique(), "KEY")); + ImageRequest wh = image(); + wh.width = 512; + wh.height = 512; + ASSERT_THROW(ig.plan(wh), ApiError); + ASSERT_EQ(target(ig.plan(image())), std::string("/generations/image")); + ASSERT_THROW(g.cancel(JobTicket{}, false), ApiError); // cancel deletes: confirm required + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, minimax_refuses_options_v1_does_not_have) { + Mock m; + MinimaxVideoGenerator g(MinimaxClient::with(m.take(), "KEY")); + VideoRequest v = video(); + v.duration_seconds = 6; + v.resolution = "768p"; + Json p = g.plan(v); + ASSERT_EQ(target(p), std::string("/v1/video_generation")); + ASSERT_TRUE(has(body_of(p), "\"resolution\":\"768P\"")); + ASSERT_TRUE(has(body_of(p), "\"model\":\"MiniMax-Hailuo-2.3\"")); + VideoRequest ar = video(); + ar.aspect_ratio = "16:9"; + ASSERT_THROW(g.plan(ar), ApiError); + VideoRequest sd = video(); + sd.seed = 5; + ASSERT_THROW(g.plan(sd), ApiError); + ASSERT_THROW(g.submit(v, false), ApiError); + ASSERT_FALSE(g.cancel(JobTicket{}, true)); // not supported, says so without a call + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, kling_and_higgsfield) { + auto kc = std::make_shared("AK", "SK", [] { return std::make_unique(); }); + KlingVideoGenerator k(kc); + VideoRequest v = video(); + v.duration_seconds = 5; + Json p = k.plan(v); + ASSERT_EQ(target(p), std::string("/v1/videos/text2video")); + ASSERT_TRUE(has(body_of(p), "\"model_name\":\"kling-v1-6\"")); + ASSERT_TRUE(has(body_of(p), "\"duration\":\"5\"")); + VideoRequest i = video(); + i.image_url = "https://example.com/a.png"; + ASSERT_EQ(target(k.plan(i)), std::string("/v1/videos/image2video")); + VideoRequest sd = video(); + sd.seed = 1; + ASSERT_THROW(k.plan(sd), ApiError); + ASSERT_THROW(k.submit(v, false), ApiError); + + Mock m; + HiggsfieldImageGenerator h(HiggsfieldClient::with(m.take(), "id", "secret")); + Json hp = h.plan(image()); + ASSERT_EQ(target(hp), std::string("/higgsfield-ai/soul/standard")); + ImageRequest wh = image(); + wh.width = 512; + wh.height = 512; + ASSERT_THROW(h.plan(wh), ApiError); + HiggsfieldVideoGenerator hv(HiggsfieldClient::with(std::make_unique(), "id", "secret")); + ASSERT_THROW(hv.plan(video()), ApiError); // model path required + ASSERT_EQ(target(hv.plan(video("kling-video/v2.5-turbo/pro/text-to-video"))), + std::string("/kling-video/v2.5-turbo/pro/text-to-video")); + ASSERT_THROW(h.cancel(JobTicket{}, false), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_adapter, registry_builds_a_generator_for_every_declared_capability) { + MediaRegistry reg; + for (const ProviderInfo& p : MediaRegistry::providers()) { + if (p.image) { + auto g = reg.image(p.id); + ASSERT_EQ(g->provider(), p.id); + ASSERT_FALSE(g->models().empty()); + } else { + ASSERT_THROW(reg.image(p.id), ApiError); + } + if (p.video) { + auto g = reg.video(p.id); + ASSERT_EQ(g->provider(), p.id); + ASSERT_FALSE(g->models().empty()); + } else { + ASSERT_THROW(reg.video(p.id), ApiError); + } + ASSERT_TRUE(reg.service(p.id) != nullptr); + } + ASSERT_THROW(reg.image("midjourney"), ApiError); + ASSERT_THROW(reg.service("pika"), ApiError); +} + +TEST(media_adapter, every_default_model_resolves_to_a_row_in_its_table) { + // A model named by models() must plan cleanly where the adapter derives the row from it. + MediaRegistry reg; + ImageRequest r = image(); + for (const std::string& model : reg.image("bfl")->models()) { + r.model = model; + r.width = 0; + r.height = 0; + if (model.rfind("flux-kontext", 0) == 0) r.aspect_ratio = "1:1"; + else r.aspect_ratio = ""; + if (model == "flux-3-image") continue; // takes images[], aspect_ratio: covered by the table test + r.aspect_ratio = ""; + reg.image("bfl")->plan(r); + } + for (const std::string& model : reg.image("ideogram")->models()) { + r.model = model; + r.aspect_ratio = ""; + r.width = 0; + r.height = 0; + reg.image("ideogram")->plan(r); + } + for (const std::string& model : reg.video("ideogram")->models()) { + VideoRequest v = video(model); + reg.video("ideogram")->plan(v); + } + for (const std::string& model : reg.video("higgsfield")->models()) reg.video("higgsfield")->plan(video(model)); +} diff --git a/tests/src/vendors/media/test_call_tables.cpp b/tests/src/vendors/media/test_call_tables.cpp new file mode 100644 index 0000000..5051a73 --- /dev/null +++ b/tests/src/vendors/media/test_call_tables.cpp @@ -0,0 +1,154 @@ +// Call tables: structure of every provider table, CallTable rules, registry. +#include + +#include "support.h" +#include "vendors/media/common/registry.h" + +using namespace molo::media; +using namespace media_test; + +namespace { + +struct Expect { + const char* id; + std::size_t rows; // rows declared from the published spec / SDK / docs (see SPEC.md) +}; + +const Expect kExpected[] = { + {"stability", 34}, {"bfl", 30}, {"replicate", 36}, {"fal", 16}, {"ideogram", 70}, {"recraft", 22}, + {"leonardo", 6}, {"runway", 66}, {"luma", 13}, {"minimax", 7}, {"kling", 6}, {"higgsfield", 10}, +}; + +} // namespace + +TEST(media_tables, every_provider_has_the_declared_row_count) { + for (const Expect& e : kExpected) { + const std::string id = e.id; + const CallTable& t = MediaRegistry::table(id); + ASSERT_EQ(t.size(), e.rows); + ASSERT_EQ(t.vendor(), std::string(e.id)); + } + ASSERT_EQ(MediaRegistry::providers().size(), sizeof(kExpected) / sizeof(kExpected[0])); +} + +TEST(media_tables, rows_are_well_formed) { + for (const Expect& e : kExpected) { + const std::string id = e.id; + const CallTable& t = MediaRegistry::table(id); + std::set seen; + for (std::size_t i = 0; i < t.size(); ++i) { + const CallRow& r = t.at(i); + ASSERT_TRUE(seen.insert(r.key).second); + std::string method = r.http_method; + std::string kind = r.body_kind; + // a body-less method never declares body fields + if (method == "GET" || method == "DELETE") ASSERT_TRUE(kind == "none"); + if (kind == "none") ASSERT_EQ(std::string(r.body_fields), std::string("")); + if (kind != "none") ASSERT_TRUE(method != "GET"); + // destructive rows are DELETE or an explicit delete_* call + if (CallTable::effect_of(r) == Effect::kDelete) + ASSERT_TRUE(method == "DELETE" || std::string(r.key).find("delete") != std::string::npos); + // every GET reads + if (method == "GET") ASSERT_TRUE(CallTable::effect_of(r) == Effect::kRead); + // placeholders are non-empty and unique within a path + std::set ph; + for (const std::string& p : CallTable::placeholders(r.path)) { + ASSERT_FALSE(p.empty()); + ASSERT_TRUE(ph.insert(p).second); + } + } + } +} + +TEST(media_tables, paid_rows_exist_for_every_generating_provider) { + for (const Expect& e : kExpected) { + const std::string id = e.id; + const CallTable& t = MediaRegistry::table(id); + std::size_t paid = 0; + for (std::size_t i = 0; i < t.size(); ++i) + if (CallTable::effect_of(t.at(i)) == Effect::kPaid) ++paid; + ASSERT_TRUE(paid >= 1); + } +} + +TEST(media_tables, rejects_a_malformed_table) { + static const CallRow dup[] = {{"a", "GET", "/x", "", "none", "", "", "read"}, {"a", "GET", "/y", "", "none", "", "", "read"}}; + ASSERT_THROW(CallTable("t", dup, 2), std::logic_error); + static const CallRow badm[] = {{"a", "FETCH", "/x", "", "none", "", "", "read"}}; + ASSERT_THROW(CallTable("t", badm, 1), std::logic_error); + static const CallRow badk[] = {{"a", "GET", "/x", "", "blob", "", "", "read"}}; + ASSERT_THROW(CallTable("t", badk, 1), std::logic_error); + static const CallRow bade[] = {{"a", "GET", "/x", "", "none", "", "", "free"}}; + ASSERT_THROW(CallTable("t", bade, 1), std::logic_error); + static const CallRow badp[] = {{"a", "GET", "x", "", "none", "", "", "read"}}; + ASSERT_THROW(CallTable("t", badp, 1), std::logic_error); +} + +TEST(media_tables, path_rendering_validates_every_argument) { + static const CallRow rows[] = { + {"one", "GET", "/a/{id}/b/{name}", "", "none", "", "", "read"}, + {"multi", "GET", "/{endpoint_id}/x", "", "none", "", "", "read"}, + }; + CallTable t("t", rows, 2); + ASSERT_EQ(t.render_path(t.get("one"), {{"id", "abc-1"}, {"name", "n.v"}}), std::string("/a/abc-1/b/n.v")); + ASSERT_THROW(t.render_path(t.get("one"), {{"id", "abc"}}), ApiError); // missing + ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a"}, {"name", "b"}, {"x", "c"}}), ApiError); // extra + ASSERT_THROW(t.render_path(t.get("one"), {{"id", "../x"}, {"name", "b"}}), ApiError); // traversal + ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a/b"}, {"name", "b"}}), ApiError); // slash in a segment + ASSERT_THROW(t.render_path(t.get("one"), {{"id", ".."}, {"name", "b"}}), ApiError); + ASSERT_THROW(t.render_path(t.get("one"), {{"id", ""}, {"name", "b"}}), ApiError); + ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a b"}, {"name", "b"}}), ApiError); + ASSERT_THROW(t.render_path(t.get("one"), {{"id", std::string(257, 'a')}, {"name", "b"}}), ApiError); + ASSERT_EQ(t.render_path(t.get("multi"), {{"endpoint_id", "fal-ai/flux/dev"}}), std::string("/fal-ai/flux/dev/x")); + ASSERT_THROW(t.render_path(t.get("multi"), {{"endpoint_id", "a//b"}}), ApiError); + ASSERT_THROW(t.render_path(t.get("multi"), {{"endpoint_id", "a/b/c/d/e/f/g"}}), ApiError); // 7 segments + ASSERT_THROW(t.get("nope"), ApiError); +} + +TEST(media_tables, segment_alphabet_matches_the_documented_id_shapes) { + ASSERT_TRUE(CallTable::valid_segment("cneqzikepnug6xezperrr4z55o")); // Replicate file id + ASSERT_TRUE(CallTable::valid_segment("d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff")); // Higgsfield uuid + ASSERT_TRUE(CallTable::valid_segment("abc_DEF-123==")); // URL-safe base64 with padding + ASSERT_FALSE(CallTable::valid_segment("a?b")); + ASSERT_FALSE(CallTable::valid_segment("a#b")); + ASSERT_FALSE(CallTable::valid_segment("a%2fb")); +} + +TEST(media_registry, lists_unsupported_products_with_a_reason) { + const auto& u = MediaRegistry::unsupported(); + ASSERT_EQ(u.size(), static_cast(2)); + ASSERT_EQ(u[0].name, std::string("Midjourney")); + ASSERT_TRUE(has(u[0].reason, "No public API")); + ASSERT_EQ(u[1].name, std::string("Pika")); + for (const auto& x : u) ASSERT_TRUE(x.reason.size() > 40); +} + +TEST(media_registry, every_provider_declares_credentials_evidence_and_a_console) { + for (const ProviderInfo& p : MediaRegistry::providers()) { + ASSERT_FALSE(p.env.empty()); + ASSERT_TRUE(p.image || p.video); + ASSERT_TRUE(p.console_url.rfind("https://", 0) == 0); + ASSERT_TRUE(p.note.size() > 10); + if (!p.account_key.empty()) { + const CallRow& r = MediaRegistry::table(p.id).get(p.account_key); + ASSERT_TRUE(CallTable::effect_of(r) == Effect::kRead); // the credential proof never costs money + } + } + ASSERT_EQ(std::string(MediaRegistry::evidence_name(MediaRegistry::find("kling")->evidence)), std::string("probe-only")); + ASSERT_EQ(std::string(MediaRegistry::evidence_name(MediaRegistry::find("bfl")->evidence)), std::string("openapi")); + ASSERT_TRUE(MediaRegistry::find("midjourney") == nullptr); + ASSERT_THROW(MediaRegistry::table("midjourney"), ApiError); +} + +TEST(media_registry, required_environment_variables_are_the_documented_ones) { + auto env = [](const char* id) { return MediaRegistry::find(id)->env; }; + ASSERT_EQ(env("stability")[0], std::string("STABILITY_API_KEY")); + ASSERT_EQ(env("bfl")[0], std::string("BFL_API_KEY")); + ASSERT_EQ(env("runway")[0], std::string("RUNWAY_API_KEY")); + ASSERT_EQ(env("luma")[0], std::string("LUMA_API_KEY")); + ASSERT_EQ(env("replicate")[0], std::string("REPLICATE_API_TOKEN")); + ASSERT_EQ(env("fal")[0], std::string("FAL_KEY")); + ASSERT_EQ(env("ideogram")[0], std::string("IDEOGRAM_API_KEY")); + ASSERT_EQ(env("recraft")[0], std::string("RECRAFT_API_TOKEN")); + ASSERT_EQ(env("leonardo")[0], std::string("LEONARDO_API_KEY")); +} diff --git a/tests/src/vendors/media/test_jobs.cpp b/tests/src/vendors/media/test_jobs.cpp new file mode 100644 index 0000000..d81421e --- /dev/null +++ b/tests/src/vendors/media/test_jobs.cpp @@ -0,0 +1,414 @@ +// The neutral job seam: request validation, JobRunner caps (with a fake +// generator: local logic only), output scanning, status mappers, downloader +// rules. Inputs to the mappers are hand-built (LOCAL-LOGIC): they exercise our +// code, they are not claims about what a vendor returns. +#include + +#include "support.h" +#include "vendors/media/bfl/bfl.h" +#include "vendors/media/common/caps.h" +#include "vendors/media/common/downloader.h" +#include "vendors/media/common/job_runner.h" +#include "vendors/media/common/output_scanner.h" +#include "vendors/media/higgsfield/higgsfield.h" +#include "vendors/media/kling/kling.h" +#include "vendors/media/leonardo/leonardo.h" +#include "vendors/media/luma/luma.h" +#include "vendors/media/minimax/minimax.h" +#include "vendors/media/runway/runway.h" + +using namespace molo::media; +using namespace media_test; + +namespace { + +Json obj(std::initializer_list> items) { + Json o = Json::object(); + for (const auto& [k, v] : items) o.set(k, v); + return o; +} + +Json arr(std::initializer_list items) { + Json a = Json::array(); + for (const Json& v : items) a.push(v); + return a; +} + +// Local fake: finishes after `finish_after` polls (0 = never). +class FakeGenerator : public ImageGenerator { + public: + explicit FakeGenerator(int finish_after, JobState end = JobState::kSucceeded) : finish_after_(finish_after), end_(end) {} + std::string provider() const override { return "fake"; } + bool configured() const override { return true; } + std::vector models() const override { return {"m"}; } + Json plan(const ImageRequest&) const override { return Json::object(); } + JobTicket submit(const ImageRequest&, bool) override { return JobTicket{}; } + JobStatus poll(const JobTicket&) override { + ++polls; + JobStatus s; + if (finish_after_ > 0 && polls >= finish_after_) { + s.state = end_; + if (end_ == JobState::kFailed) s.error = "boom"; + MediaAsset a; + a.url = "https://example.com/out.png"; + if (end_ == JobState::kSucceeded) s.assets.push_back(a); + } else { + s.state = JobState::kRunning; + } + return s; + } + bool cancel(const JobTicket&, bool) override { return false; } + int suggested_poll_ms() const override { return 100; } // below the floor on purpose + int polls = 0; + + private: + int finish_after_; + JobState end_; +}; + +struct Clockwork { + long now = 0; + std::vector slept; + JobRunner::Sleeper sleeper() { + return [this](int ms) { + slept.push_back(ms); + now += ms; + }; + } + JobRunner::Clock clock() { + return [this] { return now; }; + } +}; + +JobTicket ticket(const std::string& id = "j1") { + JobTicket t; + t.provider = "fake"; + t.id = id; + return t; +} + +} // namespace + +// ------------------------------------------------------------------ JobRunner + +TEST(media_runner, polls_until_terminal_and_applies_the_interval_floor_and_backoff) { + Clockwork w; + FakeGenerator g(5); + WaitPolicy p; + p.backoff = 2.0; + p.max_interval_ms = 3000; + JobRunner r(p, w.sleeper(), w.clock()); + JobStatus s = r.wait(g, ticket()); + ASSERT_TRUE(s.state == JobState::kSucceeded); + ASSERT_EQ(g.polls, 5); + // suggestion 100 ms is lifted to the 500 ms floor, then doubles, capped at 3000 + ASSERT_EQ(w.slept.size(), static_cast(4)); + ASSERT_EQ(w.slept[0], 500); + ASSERT_EQ(w.slept[1], 1000); + ASSERT_EQ(w.slept[2], 2000); + ASSERT_EQ(w.slept[3], 3000); +} + +TEST(media_runner, a_failed_job_is_a_status_not_an_exception) { + Clockwork w; + FakeGenerator g(2, JobState::kFailed); + JobRunner r({}, w.sleeper(), w.clock()); + JobStatus s = r.wait(g, ticket()); + ASSERT_TRUE(s.state == JobState::kFailed); + ASSERT_EQ(s.error, std::string("boom")); +} + +TEST(media_runner, attempt_cap_throws_loudly_and_does_not_cancel) { + Clockwork w; + FakeGenerator g(0); + WaitPolicy p; + p.max_attempts = 3; + JobRunner r(p, w.sleeper(), w.clock()); + try { + r.wait(g, ticket("job-77")); + FAIL("expected WAIT_EXPIRED"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("WAIT_EXPIRED")); + ASSERT_EQ(e.http_status(), 504); + ASSERT_TRUE(has(e.message(), "job-77")); + ASSERT_TRUE(has(e.message(), "NOT cancelled")); + } + ASSERT_EQ(g.polls, 3); +} + +TEST(media_runner, total_wait_cap_stops_before_sleeping_past_the_deadline) { + Clockwork w; + FakeGenerator g(0); + WaitPolicy p; + p.max_wait_seconds = 5; + p.max_attempts = 200; + p.first_interval_ms = 2000; + p.backoff = 1.0; + JobRunner r(p, w.sleeper(), w.clock()); + ASSERT_THROW(r.wait(g, ticket()), ApiError); + long total = 0; + for (int s : w.slept) total += s; + ASSERT_TRUE(total <= 5000); + ASSERT_TRUE(g.polls <= 4); +} + +TEST(media_runner, immediate_tickets_need_no_poll) { + Clockwork w; + FakeGenerator g(0); + JobTicket t = ticket(); + t.immediate = true; + MediaAsset a; + a.data_base64 = "AAAA"; + t.assets.push_back(a); + JobStatus s = JobRunner({}, w.sleeper(), w.clock()).wait(g, t); + ASSERT_TRUE(s.state == JobState::kSucceeded); + ASSERT_EQ(g.polls, 0); + ASSERT_EQ(s.assets.size(), static_cast(1)); +} + +TEST(media_runner, policy_values_outside_the_caps_are_rejected) { + auto bad = [](auto edit) { + WaitPolicy p; + edit(p); + return p; + }; + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_attempts = 0; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_attempts = MediaCaps::kMaxPollAttempts + 1; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_wait_seconds = MediaCaps::kMaxWaitSeconds + 1; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_wait_seconds = 0; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.backoff = 0.9; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.backoff = 3.5; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_interval_ms = MediaCaps::kMinPollIntervalMs - 1; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_interval_ms = MediaCaps::kMaxPollIntervalMs + 1; })), ApiError); + ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.first_interval_ms = -1; })), ApiError); + JobRunner ok{WaitPolicy{}}; + ASSERT_EQ(ok.policy().max_attempts, 120); +} + +// -------------------------------------------------------------------- requests + +TEST(media_requests, image_validation) { + ImageRequest r; + ASSERT_THROW(r.validate(), ApiError); // no prompt + r.prompt = "a cat"; + r.validate(); + r.count = 0; + ASSERT_THROW(r.validate(), ApiError); + r.count = MediaCaps::kMaxImagesPerRequest + 1; + ASSERT_THROW(r.validate(), ApiError); + r.count = 1; + r.width = 512; + ASSERT_THROW(r.validate(), ApiError); // width without height + r.height = MediaCaps::kMaxDimension + 1; + ASSERT_THROW(r.validate(), ApiError); + r.height = 512; + r.validate(); + r.seed = -2; + ASSERT_THROW(r.validate(), ApiError); + r.seed = 4294967296L; + ASSERT_THROW(r.validate(), ApiError); + r.seed = 7; + r.extra = Json(5); + ASSERT_THROW(r.validate(), ApiError); + r.extra = Json::object(); + r.prompt = std::string(MediaCaps::kMaxPromptChars + 1, 'x'); + ASSERT_THROW(r.validate(), ApiError); +} + +TEST(media_requests, video_validation) { + VideoRequest v; + ASSERT_THROW(v.validate(), ApiError); + v.image_url = "https://example.com/a.png"; // image-to-video needs no prompt + v.validate(); + v.duration_seconds = MediaCaps::kMaxVideoSeconds + 1; + ASSERT_THROW(v.validate(), ApiError); + v.duration_seconds = 5; + v.validate(); + v.duration_seconds = -1; + ASSERT_THROW(v.validate(), ApiError); +} + +TEST(media_requests, ticket_and_status_json) { + JobTicket t; + t.provider = "fal"; + t.id = "r1"; + t.model = "fal-ai/flux/dev"; + t.kind = MediaKind::kVideo; + t.meta = obj({{"endpoint", Json("fal-ai/flux/dev")}}); + JobTicket back = JobTicket::from_json(t.to_json()); + ASSERT_EQ(back.provider, std::string("fal")); + ASSERT_EQ(back.id, std::string("r1")); + ASSERT_TRUE(back.kind == MediaKind::kVideo); + ASSERT_EQ(back.meta.find("endpoint")->as_string(), std::string("fal-ai/flux/dev")); + ASSERT_THROW(JobTicket::from_json(Json(1)), ApiError); + ASSERT_THROW(JobTicket::from_json(obj({{"provider", Json("x")}})), ApiError); + JobStatus s; + s.state = JobState::kRunning; + s.progress = 0.5; + ASSERT_TRUE(has(s.to_json().dump(), "\"state\":\"running\"")); + ASSERT_FALSE(s.terminal()); + s.state = JobState::kCancelled; + ASSERT_TRUE(s.terminal()); +} + +// -------------------------------------------------------------- OutputScanner + +TEST(media_scanner, finds_urls_in_the_shapes_providers_use) { + // fal style: images[].url and video.url + auto fal = OutputScanner::scan(obj({{"images", arr({obj({{"url", Json("https://v3.fal.media/a.png")}})})}, + {"video", obj({{"url", Json("https://v3.fal.media/b.mp4")}})}}), + MediaKind::kImage); + ASSERT_EQ(fal.size(), static_cast(2)); + ASSERT_TRUE(fal[0].kind == MediaKind::kImage); + ASSERT_EQ(fal[0].mime, std::string("image/png")); + ASSERT_TRUE(fal[1].kind == MediaKind::kVideo); + // Replicate style: a bare string, or an array + ASSERT_EQ(OutputScanner::scan(Json("https://replicate.delivery/x/out.webp"), MediaKind::kImage).size(), static_cast(1)); + ASSERT_EQ(OutputScanner::scan(arr({Json("https://a/1.png"), Json("https://a/2.png")}), MediaKind::kImage).size(), static_cast(2)); + // non-URL strings are ignored + ASSERT_EQ(OutputScanner::scan(arr({Json("hello"), Json("ftp://x/y.png")}), MediaKind::kImage).size(), static_cast(0)); +} + +TEST(media_scanner, skips_request_echoes_and_control_links_and_dedupes) { + Json doc = obj({{"input", obj({{"image", Json("https://in/source.png")}})}, + {"urls", obj({{"get", Json("https://api.replicate.com/v1/predictions/p1")}})}, + {"status_url", Json("https://api.higgsfield.ai/requests/x/status")}, + {"output", arr({Json("https://out/a.png"), Json("https://out/a.png")})}}); + auto found = OutputScanner::scan(doc, MediaKind::kImage); + ASSERT_EQ(found.size(), static_cast(1)); + ASSERT_EQ(found[0].url, std::string("https://out/a.png")); +} + +TEST(media_scanner, data_urls_become_inline_assets_and_caps_hold) { + auto d = OutputScanner::scan(Json("data:image/png;base64,iVBORw0KGgo="), MediaKind::kOther); + ASSERT_EQ(d.size(), static_cast(1)); + ASSERT_EQ(d[0].mime, std::string("image/png")); + ASSERT_EQ(d[0].data_base64, std::string("iVBORw0KGgo=")); + ASSERT_TRUE(d[0].kind == MediaKind::kImage); + Json many = Json::array(); + for (int i = 0; i < 100; ++i) many.push(Json("https://x/" + std::to_string(i) + ".png")); + ASSERT_EQ(OutputScanner::scan(many, MediaKind::kImage).size(), MediaCaps::kMaxAssetsPerJob); + Json deep = Json("https://deep/out.png"); + for (int i = 0; i < 20; ++i) deep = obj({{"k", deep}}); + ASSERT_EQ(OutputScanner::scan(deep, MediaKind::kImage).size(), static_cast(0)); // depth cap + ASSERT_TRUE(OutputScanner::kind_from_name("https://x/a.GLB?sig=1", MediaKind::kOther) == MediaKind::kModel3d); + ASSERT_TRUE(OutputScanner::kind_from_name("https://x/a.wav", MediaKind::kOther) == MediaKind::kAudio); +} + +// ------------------------------------------------------------ status mappers + +TEST(media_status, bfl_words) { + ASSERT_TRUE(BflImageGenerator::state_of("Ready") == JobState::kSucceeded); + ASSERT_TRUE(BflImageGenerator::state_of("Pending") == JobState::kQueued); + ASSERT_TRUE(BflImageGenerator::state_of("Generating") == JobState::kRunning); + for (const char* w : {"Error", "Request Moderated", "Content Moderated", "Task not found", "No Card Generated"}) + ASSERT_TRUE(BflImageGenerator::state_of(w) == JobState::kFailed); + ASSERT_TRUE(BflImageGenerator::state_of("something new") == JobState::kRunning); // unknown = keep polling, bounded by caps +} + +TEST(media_status, runway_tasks) { + JobStatus ok = RunwayJobs::status_of(obj({{"status", Json("SUCCEEDED")}, {"output", arr({Json("https://r/v.mp4")})}}), MediaKind::kVideo); + ASSERT_TRUE(ok.state == JobState::kSucceeded); + ASSERT_TRUE(ok.assets[0].kind == MediaKind::kVideo); + JobStatus empty = RunwayJobs::status_of(obj({{"status", Json("SUCCEEDED")}}), MediaKind::kVideo); + ASSERT_TRUE(empty.state == JobState::kFailed); + JobStatus run = RunwayJobs::status_of(obj({{"status", Json("RUNNING")}, {"progress", Json(0.4)}}), MediaKind::kVideo); + ASSERT_TRUE(run.state == JobState::kRunning); + ASSERT_TRUE(run.progress > 0.39 && run.progress < 0.41); + JobStatus fail = RunwayJobs::status_of(obj({{"status", Json("FAILED")}, {"failure", Json("moderation")}, {"failureCode", Json("SAFETY")}}), MediaKind::kVideo); + ASSERT_EQ(fail.error, std::string("moderation (SAFETY)")); + ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("CANCELLED")}}), MediaKind::kVideo).state == JobState::kCancelled); + ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("THROTTLED")}}), MediaKind::kVideo).state == JobState::kQueued); + ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("PENDING")}}), MediaKind::kVideo).state == JobState::kQueued); +} + +TEST(media_status, luma_generations_ignore_the_progress_preview) { + Json done = obj({{"state", Json("completed")}, + {"assets", obj({{"video", Json("https://l/final.mp4")}, {"progress_video", Json("https://l/progress.mp4")}})}}); + JobStatus s = LumaJobs::status_of(done, MediaKind::kVideo); + ASSERT_TRUE(s.state == JobState::kSucceeded); + ASSERT_EQ(s.assets.size(), static_cast(1)); + ASSERT_EQ(s.assets[0].url, std::string("https://l/final.mp4")); + ASSERT_TRUE(LumaJobs::status_of(obj({{"state", Json("dreaming")}}), MediaKind::kVideo).state == JobState::kRunning); + ASSERT_TRUE(LumaJobs::status_of(obj({{"state", Json("queued")}}), MediaKind::kVideo).state == JobState::kQueued); + JobStatus f = LumaJobs::status_of(obj({{"state", Json("failed")}, {"failure_reason", Json("nope")}}), MediaKind::kVideo); + ASSERT_EQ(f.error, std::string("nope")); +} + +TEST(media_status, word_maps_of_minimax_higgsfield_kling) { + ASSERT_TRUE(MinimaxVideoGenerator::state_of("Success") == JobState::kSucceeded); + ASSERT_TRUE(MinimaxVideoGenerator::state_of("Fail") == JobState::kFailed); + ASSERT_TRUE(MinimaxVideoGenerator::state_of("Processing") == JobState::kRunning); + ASSERT_TRUE(MinimaxVideoGenerator::state_of("Queueing") == JobState::kQueued); + ASSERT_TRUE(MinimaxVideoGenerator::state_of("Preparing") == JobState::kQueued); + for (const char* w : {"completed"}) ASSERT_TRUE(HiggsfieldJobs::state_of(w) == JobState::kSucceeded); + ASSERT_TRUE(HiggsfieldJobs::state_of("nsfw") == JobState::kFailed); + ASSERT_TRUE(HiggsfieldJobs::state_of("failed") == JobState::kFailed); + ASSERT_TRUE(HiggsfieldJobs::state_of("canceled") == JobState::kCancelled); + ASSERT_TRUE(HiggsfieldJobs::state_of("in_progress") == JobState::kRunning); + ASSERT_TRUE(HiggsfieldJobs::state_of("queued") == JobState::kQueued); + ASSERT_TRUE(KlingVideoGenerator::state_of("succeed") == JobState::kSucceeded); + ASSERT_TRUE(KlingVideoGenerator::state_of("failed") == JobState::kFailed); + ASSERT_TRUE(KlingVideoGenerator::state_of("processing") == JobState::kRunning); + ASSERT_TRUE(KlingVideoGenerator::state_of("submitted") == JobState::kQueued); +} + +TEST(media_status, leonardo_generation_shape_is_read_from_generations_by_pk) { + Json done = obj({{"generations_by_pk", obj({{"status", Json("COMPLETE")}, + {"generated_images", arr({obj({{"url", Json("https://cdn.leonardo.ai/a.jpg")}, {"id", Json("i1")}})})}})}}); + JobStatus s = LeonardoJobs::status_of(done, MediaKind::kImage); + ASSERT_TRUE(s.state == JobState::kSucceeded); + ASSERT_EQ(s.assets.size(), static_cast(1)); + ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("PENDING")}})}}), MediaKind::kImage).state == JobState::kRunning); + ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("FAILED")}})}}), MediaKind::kImage).state == JobState::kFailed); + ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("COMPLETE")}})}}), MediaKind::kImage).state == JobState::kFailed); +} + +// ----------------------------------------------------------------- downloader + +TEST(media_downloader, url_rules) { + Downloader::validate_url("https://cdn.example.com/a.png?sig=abc"); + for (const char* bad : {"", "http://example.com/a.png", "ftp://x/y", "https://", "https:///path", "https://u:p@example.com/a", + "https://exa mple.com/a", "javascript:alert(1)", "file:///etc/passwd", "data:image/png;base64,AA=="}) + ASSERT_THROW(Downloader::validate_url(bad), ApiError); + ASSERT_THROW(Downloader::validate_url("https://x/" + std::string(MediaCaps::kMaxUrlChars, 'a')), ApiError); + ASSERT_THROW(Downloader::validate_url("https://x/a\nb"), ApiError); +} + +TEST(media_downloader, inline_assets_decode_without_network) { + Mock m; + MockTransport* t = m.raw; + Downloader d(m.take()); + MediaAsset a; + a.data_base64 = "aGVsbG8="; + a.mime = "image/png"; + FetchedAsset f = d.fetch(a); + ASSERT_EQ(f.bytes, std::string("hello")); + ASSERT_EQ(f.content_type, std::string("image/png")); + MediaAsset bad; + bad.data_base64 = "@@@"; + ASSERT_THROW(d.fetch(bad), ApiError); + ASSERT_THROW(d.fetch(MediaAsset{}), ApiError); + MediaAsset insecure; + insecure.url = "http://example.com/x.png"; + ASSERT_THROW(d.fetch(insecure), ApiError); + ASSERT_EQ(t->send_count(), static_cast(0)); +} + +TEST(media_downloader, sends_no_credential_and_maps_failures) { + Mock m; + serve(m.raw, Method::kGet, "/missing.png", reply("not found", 404, "text/plain")); + MockTransport* t = m.raw; + Downloader d(m.take()); + try { + d.fetch_url("https://cdn.example.com/missing.png"); + FAIL("expected a download failure"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("DOWNLOAD_FAILED")); + ASSERT_EQ(e.http_status(), 404); + } + auto seen = t->seen(); + ASSERT_EQ(seen.size(), static_cast(1)); + ASSERT_FALSE(seen[0].headers.get("Authorization").has_value()); + ASSERT_FALSE(seen[0].headers.get("x-key").has_value()); +} diff --git a/tests/src/vendors/media/test_live.cpp b/tests/src/vendors/media/test_live.cpp new file mode 100644 index 0000000..c7b3a5c --- /dev/null +++ b/tests/src/vendors/media/test_live.cpp @@ -0,0 +1,106 @@ +// Opt-in live checks. MOLO_LIVE_NET=1 enables the network; a provider's key +// (via llm::Config) enables its read-only calls (account, balance, models). +// Anything that spends money additionally needs MOLO_LIVE_PAID=1 and is +// limited to a free price quote where the provider offers one (Ideogram +// dry_run) - this file never submits a generation. Without the variables each +// test prints a loud SKIP. Nothing is created or deleted. +#include +#include + +#include "clients/transports/http_transport.h" +#include "support.h" +#include "vendors/media/common/downloader.h" +#include "vendors/media/common/registry.h" +#include "vendors/media/bfl/bfl.h" +#include "vendors/media/fal/fal.h" +#include "vendors/media/higgsfield/higgsfield.h" +#include "vendors/media/ideogram/ideogram.h" +#include "vendors/media/kling/kling.h" +#include "vendors/media/leonardo/leonardo.h" +#include "vendors/media/luma/luma.h" +#include "vendors/media/minimax/minimax.h" +#include "vendors/media/recraft/recraft.h" +#include "vendors/media/replicate/replicate.h" +#include "vendors/media/runway/runway.h" +#include "vendors/media/stability/stability.h" + +using namespace molo::media; +using namespace media_test; + +namespace { + +// Read once at startup: other suites in this binary unset MOLO_LIVE_NET while they run. +const bool kLiveNet = std::getenv("MOLO_LIVE_NET") != nullptr; +const bool kLivePaid = std::getenv("MOLO_LIVE_PAID") != nullptr; +bool net() { return kLiveNet; } + +void skip(const std::string& name, const std::string& why) { std::cout << "SKIP " << name << " (" << why << ")\n"; } + +// Unauthenticated probe: a deliberately invalid key must be refused (the host +// resolves, TLS works, the documented path exists). `accept` lists the HTTP +// statuses seen on 2026-10-25: BFL answers 422 for a malformed key, MiniMax +// answers 200 with base_resp 1004 (turned into an ApiError by the client). +template +void expect_refused(const std::string& name, Fn call, std::initializer_list accept = {401, 403}) { + try { + call(); + FAIL(name << ": an invalid key was accepted"); + } catch (const ApiError& e) { + std::cout << " " << name << ": " << e.http_status() << " " << e.type() << "\n"; + bool ok = false; + for (int a : accept) ok = ok || e.http_status() == a; + ASSERT_TRUE(ok); + } +} + +std::unique_ptr real() { return std::make_unique(); } + +} // namespace + +TEST(media_live, invalid_keys_are_refused_by_every_provider) { + if (!net()) return skip("media_live.invalid_keys_are_refused", "set MOLO_LIVE_NET=1"); + const std::string bad = "molo-invalid-probe"; + expect_refused("stability", [&] { StabilityClient::with(real(), bad)->balance(); }); + expect_refused("bfl", [&] { BflClient::with(real(), bad)->credits(); }, {401, 403, 422}); + expect_refused("replicate", [&] { ReplicateClient::with(real(), bad)->account(); }); + expect_refused("fal.platform", [&] { FalPlatform::with(real(), bad)->pricing("fal-ai/flux/dev"); }); + expect_refused("fal.queue", [&] { FalQueue::with(real(), bad)->status(FalEndpoint::parse("fal-ai/flux/dev"), "00000000-0000-0000-0000-000000000000"); }); + expect_refused("ideogram", [&] { IdeogramClient::with(real(), bad)->api_keys(); }); + expect_refused("recraft", [&] { RecraftClient::with(real(), bad)->me(); }); + expect_refused("leonardo", [&] { LeonardoClient::with(real(), bad)->models(); }); + expect_refused("runway", [&] { RunwayClient::with(real(), bad)->organization(); }); + expect_refused("luma", [&] { LumaClient::with(real(), bad)->credits(); }); + expect_refused("minimax", [&] { MinimaxClient::with(real(), bad)->query_video("1"); }, {400}); + expect_refused("kling", [&] { std::make_shared("AK-invalid-probe", "SK-invalid-probe")->get("text2video.get", "0"); }); + expect_refused("higgsfield", [&] { HiggsfieldClient::with(real(), "a", "b")->status("00000000-0000-0000-0000-000000000000"); }); +} + +TEST(media_live, read_only_account_calls_for_every_configured_provider) { + MediaRegistry reg; + int ran = 0; + for (const ProviderInfo& p : MediaRegistry::providers()) { + const std::string name = "media_live.account." + p.id; + if (!net()) { + skip(name, "set MOLO_LIVE_NET=1"); + continue; + } + if (!MediaRegistry::configured(p.id)) { + skip(name, "set " + p.env[0]); + continue; + } + if (p.account_key.empty()) { + skip(name, p.id + " has no free account read"); + continue; + } + Json out = reg.account(p.id); + std::cout << " " << p.id << " " << p.account_key << " -> " << out.dump().substr(0, 120) << "\n"; + ASSERT_TRUE(out.is_object()); + ++ran; + } + std::cout << " account reads run: " << ran << "\n"; +} + +TEST(media_live, paid_calls_are_not_exercised_here) { + if (!kLivePaid) return skip("media_live.paid", "set MOLO_LIVE_PAID=1 (and read this file: nothing paid runs even then)"); + std::cout << " MOLO_LIVE_PAID is set; this file still submits no generation. Use the demo with confirm:true by hand.\n"; +} diff --git a/tests/src/vendors/media/test_prepare.cpp b/tests/src/vendors/media/test_prepare.cpp new file mode 100644 index 0000000..ea823ac --- /dev/null +++ b/tests/src/vendors/media/test_prepare.cpp @@ -0,0 +1,430 @@ +// Requests built from the call tables: exact targets, bodies, headers, field +// validation and the confirmation gate. No reply is scripted except where a +// fixture is labelled; prepare() itself never touches a transport. +#include "support.h" +#include "vendors/media/common/caps.h" +#include "vendors/media/bfl/bfl.h" +#include "vendors/media/fal/fal.h" +#include "vendors/media/higgsfield/higgsfield.h" +#include "vendors/media/ideogram/ideogram.h" +#include "vendors/media/kling/kling.h" +#include "vendors/media/leonardo/leonardo.h" +#include "vendors/media/luma/luma.h" +#include "vendors/media/minimax/minimax.h" +#include "vendors/media/recraft/recraft.h" +#include "vendors/media/replicate/replicate.h" +#include "vendors/media/runway/runway.h" +#include "vendors/media/stability/stability.h" + +using namespace molo::media; +using namespace media_test; + +namespace { +Json obj(std::initializer_list> items) { + Json o = Json::object(); + for (const auto& [k, v] : items) o.set(k, v); + return o; +} +} // namespace + +// ------------------------------------------------------------------ Stability + +TEST(media_prepare, stability_generate_is_multipart_with_json_accept) { + Mock m; + auto c = StabilityClient::with(m.take(), "KEY"); + PreparedCall p = c->plan("stable-image.generate.core", obj({{"prompt", Json("a cat")}, {"output_format", Json("png")}}), {}); + ASSERT_EQ(p.method, std::string("POST")); + ASSERT_EQ(p.target, std::string("/v2beta/stable-image/generate/core")); + ASSERT_TRUE(has(p.content_type, "multipart/form-data; boundary=")); + ASSERT_TRUE(has(p.body, "name=\"prompt\"")); + ASSERT_TRUE(has(p.body, "a cat")); + ASSERT_EQ(p.headers.at("Accept"), std::string("application/json")); + ASSERT_TRUE(p.effect == Effect::kPaid); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_prepare, stability_files_must_use_a_declared_field_and_fit_the_cap) { + Mock m; + auto c = StabilityClient::with(m.take(), "KEY"); + FilePart img{"image", "in.png", "image/png", "\x89PNG-bytes"}; + PreparedCall p = c->plan("stable-image.upscale.fast", obj({{"output_format", Json("webp")}}), {img}); + ASSERT_TRUE(has(p.body, "filename=\"in.png\"")); + ASSERT_TRUE(has(p.body, "Content-Type: image/png")); + ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"picture", "a.png", "image/png", "x"}}), ApiError); + ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"image", "a.png", "image/png", ""}}), ApiError); + ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), + {FilePart{"image", "a.png", "image/png", std::string(MediaCaps::kMaxFileBytes + 1, 'x')}}), ApiError); + ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"image", "a\".png", "image/png", "x"}}), ApiError); +} + +TEST(media_prepare, stability_rejects_unknown_fields_and_unknown_calls) { + Mock m; + auto c = StabilityClient::with(m.take(), "KEY"); + ASSERT_THROW(c->plan("stable-image.generate.core", obj({{"prompt", Json("x")}, {"bogus", Json(1)}}), {}), ApiError); + ASSERT_THROW(c->plan("stable-image.generate.nothing", Json::object(), {}), ApiError); +} + +TEST(media_prepare, stability_result_ids_are_validated_before_any_traffic) { + Mock m; + auto c = StabilityClient::with(m.take(), "KEY"); + ASSERT_THROW(c->result("../etc"), ApiError); + ASSERT_THROW(c->result("a/b"), ApiError); + ASSERT_THROW(c->audio_result(""), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_prepare, stability_v1_text_to_image_is_json_with_an_engine_in_the_path) { + Mock m; + auto c = StabilityClient::with(m.take(), "KEY"); + CallArgs a; + a.path["engine_id"] = "stable-diffusion-xl-1024-v1-0"; + a.body = obj({{"samples", Json(1)}, {"steps", Json(30)}}); + PreparedCall p = c->prepare("v1.generation.id.text-to-image", a); + ASSERT_EQ(p.target, std::string("/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image")); + ASSERT_EQ(p.content_type, std::string("application/json")); + ASSERT_EQ(p.body, std::string("{\"samples\":1,\"steps\":30}")); +} + +// ------------------------------------------------------------------------ BFL + +TEST(media_prepare, bfl_body_is_json_and_query_is_encoded) { + Mock m; + auto c = BflClient::with(m.take(), "KEY"); + PreparedCall p = c->plan("flux-2-pro", obj({{"prompt", Json("x")}, {"width", Json(1024)}, {"height", Json(768)}, {"seed", Json(7)}})); + ASSERT_EQ(p.target, std::string("/v1/flux-2-pro")); + ASSERT_EQ(p.body, std::string("{\"height\":768,\"prompt\":\"x\",\"seed\":7,\"width\":1024}")); + CallArgs a; + a.query["id"] = "a b&c"; + PreparedCall g = c->prepare("get_result", a); + ASSERT_EQ(g.target, std::string("/v1/get_result?id=a%20b%26c")); + ASSERT_EQ(g.body, std::string("")); + CallArgs bad; + bad.query["nope"] = "1"; + ASSERT_THROW(c->prepare("get_result", bad), ApiError); +} + +TEST(media_prepare, bfl_sends_the_x_key_header_and_never_a_bearer) { + Mock m; + serve(m.raw, Method::kGet, "/v1/credits", empty_ok()); + MockTransport* t = m.raw; + auto c = BflClient::with(m.take(), "SECRET-KEY"); + c->credits(); + auto seen = t->seen(); + ASSERT_EQ(seen.size(), static_cast(1)); + ASSERT_EQ(*seen[0].headers.get("x-key"), std::string("SECRET-KEY")); + ASSERT_FALSE(seen[0].headers.get("Authorization").has_value()); +} + +// ------------------------------------------------------------------ Replicate + +TEST(media_prepare, replicate_official_model_and_pinned_version_use_different_rows) { + Mock m; + auto c = ReplicateClient::with(m.take(), "KEY"); + Json in = obj({{"prompt", Json("a cat")}}); + PreparedCall a = c->plan_prediction(ReplicateClient::ModelRef::parse("black-forest-labs/flux-schnell"), in, Json::object()); + ASSERT_EQ(a.target, std::string("/models/black-forest-labs/flux-schnell/predictions")); + ASSERT_EQ(a.body, std::string("{\"input\":{\"prompt\":\"a cat\"}}")); + PreparedCall b = c->plan_prediction(ReplicateClient::ModelRef::parse("owner/name:abc123"), in, Json::object()); + ASSERT_EQ(b.target, std::string("/predictions")); + ASSERT_EQ(b.body, std::string("{\"input\":{\"prompt\":\"a cat\"},\"version\":\"abc123\"}")); + Json hook = obj({{"webhook", Json("https://example.com/h")}, {"webhook_events_filter", Json::array()}}); + PreparedCall w = c->plan_prediction(ReplicateClient::ModelRef::parse("o/n"), in, hook); + ASSERT_TRUE(has(w.body, "\"webhook\":\"https://example.com/h\"")); + ASSERT_THROW(c->plan_prediction(ReplicateClient::ModelRef::parse("o/n"), in, obj({{"nope", Json(1)}})), ApiError); +} + +TEST(media_prepare, replicate_model_references_are_validated) { + using R = ReplicateClient::ModelRef; + ASSERT_THROW(R::parse("justaname"), ApiError); + ASSERT_THROW(R::parse("a/b/c"), ApiError); + ASSERT_THROW(R::parse("a/../b"), ApiError); + ASSERT_THROW(R::parse("a/b:v/x"), ApiError); + ASSERT_THROW(R::parse("/b"), ApiError); + R ok = R::parse("o/n:v1"); + ASSERT_EQ(ok.owner, std::string("o")); + ASSERT_EQ(ok.version, std::string("v1")); +} + +TEST(media_prepare, replicate_uses_the_v1_base_and_bearer_auth) { + Mock m; + serve(m.raw, Method::kGet, "/v1/account", empty_ok()); + MockTransport* t = m.raw; + auto c = ReplicateClient::with(m.take(), "tok"); + c->account(); + auto seen = t->seen(); + ASSERT_EQ(seen[0].target, std::string("/v1/account")); + ASSERT_EQ(*seen[0].headers.get("Authorization"), std::string("Bearer tok")); +} + +// ------------------------------------------------------------------------ fal + +TEST(media_prepare, fal_endpoint_ids_split_into_owner_app_path) { + FalEndpoint e = FalEndpoint::parse("fal-ai/flux/dev"); + ASSERT_EQ(e.owner, std::string("fal-ai")); + ASSERT_EQ(e.app, std::string("flux")); + ASSERT_EQ(e.path, std::string("dev")); + ASSERT_EQ(e.app_id(), std::string("fal-ai/flux")); + FalEndpoint two = FalEndpoint::parse("fal-ai/veo3"); + ASSERT_EQ(two.path, std::string("")); + FalEndpoint deep = FalEndpoint::parse("fal-ai/kling-video/v2.1/master/text-to-video"); + ASSERT_EQ(deep.path, std::string("v2.1/master/text-to-video")); + ASSERT_THROW(FalEndpoint::parse("only"), ApiError); + ASSERT_THROW(FalEndpoint::parse("a/../b"), ApiError); + ASSERT_THROW(FalEndpoint::parse("a//b"), ApiError); + ASSERT_THROW(FalEndpoint::parse("a/b/c/d/e/f"), ApiError); +} + +TEST(media_prepare, fal_submit_carries_priority_header_and_webhook_query) { + Mock m; + auto q = FalQueue::with(m.take(), "id:secret"); + FalSubmitOptions o; + o.priority = "high"; + o.webhook = "https://example.com/h"; + o.max_queue_length = 3; + PreparedCall p = q->plan(FalEndpoint::parse("fal-ai/flux/dev"), obj({{"prompt", Json("x")}}), o); + ASSERT_EQ(p.method, std::string("POST")); + ASSERT_EQ(p.target, std::string("/fal-ai/flux/dev?fal_max_queue_length=3&fal_webhook=https%3A%2F%2Fexample.com%2Fh")); + ASSERT_EQ(p.headers.at("X-Fal-Queue-Priority"), std::string("high")); + ASSERT_EQ(p.body, std::string("{\"prompt\":\"x\"}")); + PreparedCall two = q->plan(FalEndpoint::parse("fal-ai/veo3"), Json::object(), FalSubmitOptions{}); + ASSERT_EQ(two.target, std::string("/fal-ai/veo3")); + FalSubmitOptions bad; + bad.priority = "urgent"; + ASSERT_THROW(q->plan(FalEndpoint::parse("fal-ai/veo3"), Json::object(), bad), ApiError); +} + +TEST(media_prepare, fal_status_result_and_cancel_use_owner_app_only) { + Mock m; + auto q = FalQueue::with(m.take(), "k"); + ASSERT_THROW(q->status(FalEndpoint::parse("fal-ai/flux/dev"), "../x"), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); + // FIXTURE: queue.json cancelQueueRequest example 202 (verbatim, https://fal.ai/docs/api-reference/inference-api/openapi/queue.json) + serve(m.raw, Method::kPut, "/fal-ai/flux/requests/019a1b2c/cancel", reply("{\"status\": \"CANCELLATION_REQUESTED\"}", 202)); + serve(m.raw, Method::kGet, "/fal-ai/flux/requests/019a1b2c/status", empty_ok()); + Json r = q->cancel(FalEndpoint::parse("fal-ai/flux/dev"), "019a1b2c", true); + ASSERT_EQ(r.find("status")->as_string(), std::string("CANCELLATION_REQUESTED")); + q->status(FalEndpoint::parse("fal-ai/flux/dev"), "019a1b2c", true); + auto seen = m.raw->seen(); + ASSERT_EQ(seen[0].method, std::string("PUT")); + ASSERT_TRUE(has(seen[1].target, "/status?logs=1")); +} + +TEST(media_prepare, fal_sends_the_key_prefix_on_both_hosts) { + Mock m; + serve(m.raw, Method::kGet, "/v1/meta", empty_ok()); + MockTransport* t = m.raw; + auto p = FalPlatform::with(m.take(), "id:secret"); + p->meta(); + ASSERT_EQ(*t->seen()[0].headers.get("Authorization"), std::string("Key id:secret")); + ASSERT_EQ(FalQueue::header_value(""), std::string("")); +} + +// ------------------------------------------------------------------- Ideogram + +TEST(media_prepare, ideogram_quote_is_a_free_dry_run_and_generate_is_paid) { + Mock m; + auto c = IdeogramClient::with(m.take(), "KEY"); + Json f = obj({{"prompt", Json("a cat")}, {"num_images", Json(1)}}); + PreparedCall q = c->plan("image.generate.ideogram-3", f, {}, true); + ASSERT_EQ(q.target, std::string("/v2/image/generate/ideogram-3?dry_run=true")); + ASSERT_TRUE(has(q.content_type, "multipart/form-data")); // ideogram-3 declares multipart only + ASSERT_TRUE(q.effect == Effect::kPaid); + PreparedCall j = c->plan("image.generate.ideogram-4", f, {}, false); + ASSERT_EQ(j.content_type, std::string("application/json")); + ASSERT_EQ(j.target, std::string("/v2/image/generate/ideogram-4")); + ASSERT_THROW(c->plan("image.generate.ideogram-3", obj({{"nope", Json(1)}}), {}), ApiError); + ASSERT_EQ(m.raw->send_count(), static_cast(0)); +} + +TEST(media_prepare, ideogram_uses_the_api_key_header) { + Mock m; + serve(m.raw, Method::kGet, "/v2/generations/abc", empty_ok()); + MockTransport* t = m.raw; + auto c = IdeogramClient::with(m.take(), "KEY"); + c->generation("abc"); + ASSERT_EQ(*t->seen()[0].headers.get("Api-Key"), std::string("KEY")); +} + +// --------------------------------------------------------------------- Recraft + +TEST(media_prepare, recraft_json_by_default_multipart_with_files) { + Mock m; + auto c = RecraftClient::with(m.take(), "KEY"); + PreparedCall j = c->plan("images.generations", obj({{"prompt", Json("x")}, {"n", Json(2)}}), {}); + ASSERT_EQ(j.target, std::string("/v1/images/generations")); + ASSERT_EQ(j.body, std::string("{\"n\":2,\"prompt\":\"x\"}")); + ASSERT_THROW(c->plan("images.generations", obj({{"prompt", Json("x")}}), {FilePart{"image", "a.png", "image/png", "x"}}), + ApiError); // generations is JSON-only + ASSERT_THROW(c->style("../x"), ApiError); + ASSERT_THROW(c->delete_style("s1", false), ApiError); // gate +} + +// --------------------------------------------------------------------- Leonardo + +TEST(media_prepare, leonardo_v2_create_wraps_model_and_parameters) { + Mock m; + auto c = LeonardoClient::with(m.take(), "KEY"); + Json b = LeonardoJobs::body("flux-dev", obj({{"prompt", Json("x")}}), false); + PreparedCall p = c->plan(b); + ASSERT_EQ(p.target, std::string("/v2/generations")); + ASSERT_EQ(p.body, std::string("{\"model\":\"flux-dev\",\"parameters\":{\"prompt\":\"x\"},\"public\":false}")); + ASSERT_THROW(c->generation("../x"), ApiError); +} + +// ---------------------------------------------------------------------- Runway + +TEST(media_prepare, runway_sends_the_version_header_and_accepts_both_key_names) { + Mock m; + serve(m.raw, Method::kGet, "/v1/organization", empty_ok()); + MockTransport* t = m.raw; + auto c = RunwayClient::with(m.take(), "KEY"); + c->organization(); + ASSERT_EQ(*t->seen()[0].headers.get("X-Runway-Version"), std::string(RunwayClient::kApiVersion)); + ASSERT_EQ(*t->seen()[0].headers.get("Authorization"), std::string("Bearer KEY")); + ASSERT_EQ(std::string(RunwayClient::kKeyEnv), std::string("RUNWAY_API_KEY")); + ASSERT_EQ(std::string(RunwayClient::kKeyEnvSdk), std::string("RUNWAYML_API_SECRET")); +} + +TEST(media_prepare, runway_task_bodies_are_checked_against_the_sdk_field_union) { + Mock m; + auto c = RunwayClient::with(m.take(), "KEY"); + PreparedCall p = c->plan("text_to_video", obj({{"model", Json("gen4.5")}, {"promptText", Json("x")}, {"duration", Json(5)}})); + ASSERT_EQ(p.target, std::string("/v1/text_to_video")); + ASSERT_THROW(c->plan("text_to_video", obj({{"promt", Json("typo")}})), ApiError); + ASSERT_THROW(c->task("a/b"), ApiError); + ASSERT_THROW(c->cancel_or_delete_task("t1", false), ApiError); +} + +// ------------------------------------------------------------------------ Luma + +TEST(media_prepare, luma_paths_and_the_dream_machine_base) { + Mock m; + serve(m.raw, Method::kGet, "/dream-machine/v1/credits", empty_ok()); + MockTransport* t = m.raw; + auto c = LumaClient::with(m.take(), "KEY"); + PreparedCall p = c->plan("generations.create_video", obj({{"model", Json("ray-2")}, {"prompt", Json("x")}})); + ASSERT_EQ(p.target, std::string("/generations/video")); + c->credits(); + ASSERT_EQ(t->seen()[0].target, std::string("/dream-machine/v1/credits")); + CallArgs a; + a.path["id"] = "g1"; + ASSERT_EQ(c->prepare("generations.upscale", a).target, std::string("/generations/g1/upscale")); + ASSERT_THROW(c->remove("g1", false), ApiError); +} + +// --------------------------------------------------------------------- MiniMax + +TEST(media_prepare, minimax_v1_and_v2_rows) { + Mock m; + auto c = MinimaxClient::with(m.take(), "KEY"); + PreparedCall p = c->plan_video(obj({{"model", Json("MiniMax-Hailuo-2.3")}, {"prompt", Json("x")}, {"duration", Json(6)}})); + ASSERT_EQ(p.target, std::string("/v1/video_generation")); + CallArgs q; + q.query["task_id"] = "t1"; + ASSERT_EQ(c->prepare("video.query", q).target, std::string("/v1/query/video_generation?task_id=t1")); + CallArgs d; + d.path["task_id"] = "t1"; + ASSERT_EQ(c->prepare("v2.video.delete", d).target, std::string("/v2/video_generation/t1")); + ASSERT_THROW(c->delete_video_v2("t1", false), ApiError); + Json ok = obj({{"base_resp", obj({{"status_code", Json(0)}, {"status_msg", Json("success")}})}}); + MinimaxClient::check_base_resp(ok); // no throw + Json bad = obj({{"base_resp", obj({{"status_code", Json(1004)}, {"status_msg", Json("login fail")}})}}); + ASSERT_THROW(MinimaxClient::check_base_resp(bad), ApiError); +} + +// ------------------------------------------------------------------ Higgsfield + +TEST(media_prepare, higgsfield_typed_rows_for_known_paths_and_a_generic_row_for_the_rest) { + ASSERT_EQ(HiggsfieldClient::row_for("higgsfield-ai/soul/standard"), std::string("higgsfield-ai.soul.standard")); + ASSERT_EQ(HiggsfieldClient::row_for("vendor/new-model/v9"), std::string("model.submit")); + Mock m; + auto c = HiggsfieldClient::with(m.take(), "id", "secret"); + PreparedCall typed = c->plan("higgsfield-ai/soul/standard", obj({{"prompt", Json("x")}})); + ASSERT_EQ(typed.target, std::string("/higgsfield-ai/soul/standard")); + ASSERT_THROW(c->plan("higgsfield-ai/soul/standard", obj({{"nope", Json(1)}})), ApiError); // typed row validates fields + PreparedCall generic = c->plan("vendor/new-model/v9", obj({{"anything", Json(1)}})); + ASSERT_EQ(generic.target, std::string("/vendor/new-model/v9")); + ASSERT_THROW(c->plan("vendor/../x", Json::object()), ApiError); + ASSERT_EQ(HiggsfieldClient::header_value("id", "secret"), std::string("Key id:secret")); + ASSERT_EQ(HiggsfieldClient::header_value("id", ""), std::string("")); +} + +TEST(media_prepare, higgsfield_submit_and_status_over_the_wire) { + Mock m; + // FIXTURE: docs.higgsfield.ai quickstart, submit reply (verbatim) + const char* kSubmit = + "{\"status\":\"queued\",\"request_id\":\"d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff\"," + "\"status_url\":\"https://api.higgsfield.ai/requests/d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff/status\"," + "\"cancel_url\":\"https://api.higgsfield.ai/requests/d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff/cancel\"}"; + serve(m.raw, Method::kPost, "/higgsfield-ai/soul/standard", reply(kSubmit)); + MockTransport* t = m.raw; + auto c = HiggsfieldClient::with(m.take(), "id", "secret"); + Json out = c->submit("higgsfield-ai/soul/standard", obj({{"prompt", Json("Editorial portrait in soft daylight")}}), true); + JobTicket ticket = HiggsfieldJobs::ticket_from(out, "higgsfield-ai/soul/standard", MediaKind::kImage); + ASSERT_EQ(ticket.id, std::string("d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff")); + auto seen = t->seen(); + ASSERT_EQ(*seen[0].headers.get("Authorization"), std::string("Key id:secret")); + ASSERT_EQ(seen[0].body, std::string("{\"prompt\":\"Editorial portrait in soft daylight\"}")); +} + +// ----------------------------------------------------------------------- Kling + +TEST(media_prepare, kling_rows_and_envelope) { + Mock m; + auto c = std::make_shared("AK", "SK", [] { return std::make_unique(); }); + PreparedCall p = c->plan("text2video.create", obj({{"model_name", Json("kling-v1-6")}, {"prompt", Json("x")}})); + ASSERT_EQ(p.target, std::string("/v1/videos/text2video")); + ASSERT_TRUE(c->configured()); + KlingClient::check_envelope(obj({{"code", Json(0)}})); + ASSERT_THROW(KlingClient::check_envelope(obj({{"code", Json(1001)}, {"message", Json("Authorization is empty")}})), ApiError); + auto none = std::make_shared("", "", [] { return std::make_unique(); }); + ASSERT_FALSE(none->configured()); +} + +// ------------------------------------------------------- confirmation gate + +TEST(media_gate, non_read_calls_need_confirm_and_make_no_traffic_without_it) { + Mock m; + MockTransport* t = m.raw; + auto bfl = BflClient::with(m.take(), "KEY"); + ASSERT_THROW(bfl->submit("flux-2-pro", obj({{"prompt", Json("x")}}), false), ApiError); + ASSERT_THROW(bfl->delete_finetune("my-ft", false), ApiError); + ASSERT_EQ(t->send_count(), static_cast(0)); + Mock m2; + MockTransport* t2 = m2.raw; + auto rep = ReplicateClient::with(m2.take(), "KEY"); + ASSERT_THROW(rep->cancel_prediction("p1", false), ApiError); + ASSERT_THROW(rep->file_delete("f1", false), ApiError); + ASSERT_THROW(rep->file_upload("a.png", "image/png", "x", false), ApiError); + ASSERT_THROW(rep->create_prediction(ReplicateClient::ModelRef::parse("o/n"), Json::object(), Json::object(), false), ApiError); + ASSERT_EQ(t2->send_count(), static_cast(0)); +} + +TEST(media_gate, missing_credentials_fail_before_any_traffic) { + Mock m; + MockTransport* t = m.raw; + auto c = BflClient::with(m.take(), ""); + ASSERT_FALSE(c->configured()); + ASSERT_THROW(c->credits(), molo::llm::NotConfiguredError); + ASSERT_THROW(c->submit("flux-2-pro", obj({{"prompt", Json("x")}}), true), molo::llm::NotConfiguredError); + ASSERT_EQ(t->send_count(), static_cast(0)); + ASSERT_FALSE(FalQueue::with(std::make_unique(), "")->configured()); + ASSERT_FALSE(HiggsfieldClient::with(std::make_unique(), "id", "")->configured()); +} + +TEST(media_gate, headers_and_query_are_bounded) { + Mock m; + auto c = BflClient::with(m.take(), "KEY"); + CallArgs a; + a.headers["Bad Name"] = "v"; + ASSERT_THROW(c->prepare("credits", a), ApiError); + CallArgs b; + b.headers["X-A"] = "line\r\nbreak"; + ASSERT_THROW(c->prepare("credits", b), ApiError); + CallArgs d; + d.query["id"] = std::string(2000, 'x'); + ASSERT_THROW(c->prepare("get_result", d), ApiError); + CallArgs e; + e.body = obj({{"x", Json(1)}}); + ASSERT_THROW(c->prepare("credits", e), ApiError); // a GET takes no body +} diff --git a/tests/src/vendors/media/test_signing.cpp b/tests/src/vendors/media/test_signing.cpp new file mode 100644 index 0000000..fe9d579 --- /dev/null +++ b/tests/src/vendors/media/test_signing.cpp @@ -0,0 +1,61 @@ +// Webhook signature verification (Replicate, Standard Webhooks scheme) and the +// Kling JWT. Both use vectors produced outside this code base. +#include "support.h" +#include "vendors/media/kling/kling.h" +#include "vendors/media/replicate/replicate.h" + +using namespace molo::media; +using namespace media_test; + +namespace { +// Standard Webhooks published test vector (github.com/standard-webhooks): secret, id, timestamp, payload. +const char* kSecret = "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw"; +const char* kId = "msg_p5jXN8AQM9LWM0D4loKWxJek"; +const char* kTimestamp = "1614265330"; +const char* kPayload = "{\"test\": 2432232314}"; +const char* kSignature = "g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE="; +} // namespace + +TEST(media_signing, webhook_signature_matches_the_standard_webhooks_vector) { + ASSERT_EQ(ReplicateWebhook::sign(kSecret, kId, kTimestamp, kPayload), std::string(kSignature)); + ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, std::string("v1,") + kSignature, kPayload, 1614265330 + 10)); +} + +TEST(media_signing, webhook_rejects_tampering_staleness_and_malformed_input) { + const std::string good = std::string("v1,") + kSignature; + const long now = 1614265330 + 10; + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, "{\"test\": 2432232315}", now)); // body changed + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, "msg_other", kTimestamp, good, kPayload, now)); // id changed + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, "1614265331", good, kPayload, now)); // timestamp changed + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 + 301)); // too old + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 - 301)); // too new + ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 + 300)); // boundary + ASSERT_FALSE(ReplicateWebhook::verify("whsec_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", kId, kTimestamp, good, kPayload, now)); // wrong secret + ASSERT_FALSE(ReplicateWebhook::verify("whsec_!!notbase64!!", kId, kTimestamp, good, kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify("", kId, kTimestamp, good, kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, "", kTimestamp, good, kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, "12ab", good, kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "", kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, kSignature, kPayload, now)); // no v1, prefix + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "v2," + std::string(kSignature), kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, std::string(2000, 'x'), kPayload, now)); // header cap +} + +TEST(media_signing, webhook_accepts_any_of_several_signatures) { + const long now = 1614265330; + std::string rotated = "v1,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= v1," + std::string(kSignature); + ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, rotated, kPayload, now)); + ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "v1,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", kPayload, now)); +} + +TEST(media_signing, kling_jwt_matches_an_independent_hs256_implementation) { + // Computed with python3 hmac / base64 (see SPEC "Verified against spec"): header {"alg":"HS256","typ":"JWT"}, + // payload {"exp":1700001800,"iss":"AK-test","nbf":1699999995}, secret "SK-test". + const char* expected = + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MDAwMDE4MDAsImlzcyI6IkFLLXRlc3QiLCJuYmYiOjE2OTk5OTk5OTV9." + "f3MPnq8kVzVYtGGChN2U_Rd1oYs3-a_FFYC4PCVGdSc"; + ASSERT_EQ(KlingJwt::make("AK-test", "SK-test", 1700000000), std::string(expected)); + ASSERT_TRUE(KlingJwt::make("AK-test", "SK-other", 1700000000) != std::string(expected)); + ASSERT_EQ(KlingJwt::kTtlSeconds, 1800L); + ASSERT_TRUE(KlingJwt::kRenewAfterSeconds < KlingJwt::kTtlSeconds); +}