feat(clients): media: image and video generation across 12 providers behind one job seam
Stability, BFL, Runway, Luma, Replicate, fal, Ideogram, Recraft, Leonardo, MiniMax, Kling, Higgsfield: call tables checked against vendor specs/SDKs, ImageGenerator/VideoGenerator seam with capped polling, /ai/media demo. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,589 @@
|
||||
#include "../AiDemos.hpp"
|
||||
|
||||
#include <ctime>
|
||||
#include <mutex>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "security/crypto/Sha256.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
#include "vendors/media/common/downloader.h"
|
||||
#include "vendors/media/common/job_runner.h"
|
||||
#include "vendors/media/common/registry.h"
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// media. Stub: the package agent fills this in.
|
||||
// Package media over the real src/vendors/media: image and video generation
|
||||
// across twelve providers behind one seam (submit, poll, fetch), plus a
|
||||
// generic call route over every provider's call table. GET routes need no
|
||||
// credential. Every route that calls a provider is POST or DELETE: 422 on bad
|
||||
// input, 503 media_not_configured without the provider's credential. Anything
|
||||
// that costs money or changes a resource is a dry run showing the exact
|
||||
// request unless the body has "confirm":true.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace md = media;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
constexpr std::size_t kMaxBody = 24 * 1024 * 1024; // uploads travel as base64
|
||||
constexpr std::size_t kInlineAssetCap = 4 * 1024 * 1024; // base64 echoed back for inline results
|
||||
constexpr std::size_t kTextPreviewCap = 64 * 1024;
|
||||
constexpr int kDemoMaxWaitSeconds = 60; // below the package cap: a request thread is held
|
||||
constexpr int kDemoMaxAttempts = 30;
|
||||
|
||||
const char kPage[] = R"MEDIA({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Media generation</h1>
|
||||
<p class="hint">Images and video from Stability, Black Forest Labs, Runway, Luma, Replicate, fal, Ideogram,
|
||||
Recraft, Leonardo, MiniMax, Kling and Higgsfield behind one submit, poll, fetch seam. Every call that costs
|
||||
money or changes a resource is a dry run until the body says <code>"confirm":true</code>.</p>
|
||||
<p id="md-config" class="hint">Checking credentials...</p>
|
||||
<section class="md">
|
||||
<label>Action
|
||||
<select id="md-route"></select></label>
|
||||
<label>Provider
|
||||
<select id="md-provider"></select></label>
|
||||
<label>Body (JSON object) <textarea id="md-body" rows="9"></textarea></label>
|
||||
<div class="md-actions">
|
||||
<button id="md-send" type="button" class="primary">Send (dry run if paid)</button>
|
||||
<button id="md-confirm" type="button">Send with confirm</button>
|
||||
</div>
|
||||
<p id="md-error" class="error" role="alert"></p>
|
||||
<pre id="md-out" class="md-out"></pre>
|
||||
<details><summary>Not available</summary><ul id="md-unsupported"></ul></details>
|
||||
</section>
|
||||
<style>
|
||||
.md label{display:block;margin:.5rem 0}
|
||||
.md select,.md textarea{width:100%;box-sizing:border-box}
|
||||
.md-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
||||
.md-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
|
||||
</style>
|
||||
<script src="/embedded/ai_media.js"></script>
|
||||
{% endblock %}
|
||||
)MEDIA";
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
||||
if (req.body.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(req.body);
|
||||
if (!j.is_object()) bad("body must be a JSON object");
|
||||
return j;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
bad("body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
std::string text(const Json& b, const char* key, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return "";
|
||||
}
|
||||
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
||||
return v->as_string();
|
||||
}
|
||||
|
||||
int number(const Json& b, const char* key, int fallback = 0) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return fallback;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
return static_cast<int>(v->as_int());
|
||||
}
|
||||
|
||||
long seed_of(const Json& b) {
|
||||
const Json* v = b.find("seed");
|
||||
if (!v || v->is_null()) return -1;
|
||||
if (!v->is_number()) bad("seed must be a number");
|
||||
return v->as_int();
|
||||
}
|
||||
|
||||
bool flag(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return false;
|
||||
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
||||
return v->as_bool();
|
||||
}
|
||||
|
||||
Json object_at(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return Json::object();
|
||||
if (!v->is_object()) bad(std::string(key) + " must be a JSON object");
|
||||
return *v;
|
||||
}
|
||||
|
||||
std::map<std::string, std::string> strings_at(const Json& b, const char* key) {
|
||||
std::map<std::string, std::string> out;
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return out;
|
||||
if (!v->is_object()) bad(std::string(key) + " must be a JSON object of strings");
|
||||
if (v->as_object().size() > md::MediaCaps::kMaxQueryArgs) bad(std::string(key) + " has too many entries");
|
||||
for (const auto& [k, x] : v->as_object()) {
|
||||
if (x.is_string()) out[k] = x.as_string();
|
||||
else if (x.is_number() || x.is_bool()) out[k] = x.dump();
|
||||
else bad(std::string(key) + "." + k + " must be a string");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
Json dry_run(const Json& plan, const char* what) {
|
||||
Json o = Json::object();
|
||||
o.set("dry_run", Json(true));
|
||||
o.set("what", Json(what));
|
||||
o.set("request", plan);
|
||||
o.set("hint", Json("send confirm:true to run it; this request costs money or changes a resource"));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json asset_json(const md::MediaAsset& a) {
|
||||
Json o = a.to_json();
|
||||
if (!a.data_base64.empty()) {
|
||||
if (a.data_base64.size() <= kInlineAssetCap) o.set("data_base64", Json(a.data_base64));
|
||||
else o.set("note", Json("inline data over the echo cap; fetch it with the provider client"));
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
Json status_json(const md::JobStatus& s) {
|
||||
Json o = s.to_json();
|
||||
Json arr = Json::array();
|
||||
for (const md::MediaAsset& a : s.assets) arr.push(asset_json(a));
|
||||
o.set("assets", arr);
|
||||
return o;
|
||||
}
|
||||
|
||||
Json ticket_reply(const md::JobTicket& t) {
|
||||
Json o = Json::object();
|
||||
Json tj = t.to_json();
|
||||
Json arr = Json::array();
|
||||
for (const md::MediaAsset& a : t.assets) arr.push(asset_json(a));
|
||||
tj.set("assets", arr);
|
||||
o.set("ticket", tj);
|
||||
o.set("next", Json(t.immediate ? "the result is in ticket.assets" : "POST /ai/media/poll or /ai/media/wait with the ticket"));
|
||||
return o;
|
||||
}
|
||||
|
||||
class MediaRoutes {
|
||||
public:
|
||||
// ---- read-only GET data ---------------------------------------------------
|
||||
Json status() {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
Json o = Json::object();
|
||||
Json arr = Json::array();
|
||||
for (const md::ProviderInfo& p : md::MediaRegistry::providers()) arr.push(provider_json(p));
|
||||
o.set("providers", arr);
|
||||
Json un = Json::array();
|
||||
for (const md::UnsupportedProduct& u : md::MediaRegistry::unsupported()) {
|
||||
Json x = Json::object();
|
||||
x.set("name", Json(u.name));
|
||||
x.set("reason", Json(u.reason));
|
||||
un.push(x);
|
||||
}
|
||||
o.set("unsupported", un);
|
||||
Json caps = Json::object();
|
||||
caps.set("max_poll_attempts", Json(md::MediaCaps::kMaxPollAttempts));
|
||||
caps.set("min_poll_interval_ms", Json(md::MediaCaps::kMinPollIntervalMs));
|
||||
caps.set("max_wait_seconds", Json(md::MediaCaps::kMaxWaitSeconds));
|
||||
caps.set("demo_max_wait_seconds", Json(kDemoMaxWaitSeconds));
|
||||
caps.set("max_images_per_request", Json(md::MediaCaps::kMaxImagesPerRequest));
|
||||
caps.set("max_video_seconds", Json(md::MediaCaps::kMaxVideoSeconds));
|
||||
caps.set("max_file_bytes", Json(static_cast<double>(md::MediaCaps::kMaxFileBytes)));
|
||||
caps.set("max_download_bytes", Json(static_cast<double>(md::MediaCaps::kMaxDownloadBytes)));
|
||||
o.set("caps", caps);
|
||||
return o;
|
||||
}
|
||||
|
||||
Json calls(const std::string& provider) {
|
||||
if (provider.empty()) {
|
||||
Json o = Json::object();
|
||||
Json arr = Json::array();
|
||||
for (const md::ProviderInfo& p : md::MediaRegistry::providers()) {
|
||||
Json x = Json::object();
|
||||
x.set("provider", Json(p.id));
|
||||
x.set("calls", Json(static_cast<double>(md::MediaRegistry::table(p.id).size())));
|
||||
arr.push(x);
|
||||
}
|
||||
o.set("providers", arr);
|
||||
return o;
|
||||
}
|
||||
if (!md::MediaRegistry::find(provider)) bad("unknown provider '" + provider + "'");
|
||||
const md::CallTable& t = md::MediaRegistry::table(provider);
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < t.size(); ++i) {
|
||||
const md::CallRow& r = t.at(i);
|
||||
Json x = Json::object();
|
||||
x.set("key", Json(r.key));
|
||||
x.set("http", Json(r.http_method));
|
||||
x.set("path", Json(r.path));
|
||||
x.set("effect", Json(r.effect));
|
||||
x.set("body", Json(r.body_kind));
|
||||
arr.push(x);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("provider", Json(provider));
|
||||
o.set("calls", arr);
|
||||
return o;
|
||||
}
|
||||
|
||||
// ---- generation seam ------------------------------------------------------
|
||||
fw::Response image(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
std::string provider = provider_of(b, true);
|
||||
md::ImageRequest r;
|
||||
r.prompt = text(b, "prompt", true);
|
||||
r.negative_prompt = text(b, "negative_prompt");
|
||||
r.model = text(b, "model");
|
||||
r.width = number(b, "width");
|
||||
r.height = number(b, "height");
|
||||
r.aspect_ratio = text(b, "aspect_ratio");
|
||||
r.count = number(b, "count", 1);
|
||||
r.seed = seed_of(b);
|
||||
r.source_url = text(b, "source_url");
|
||||
r.extra = object_at(b, "extra");
|
||||
std::shared_ptr<md::ImageGenerator> gen;
|
||||
Json plan;
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
gen = reg_.image(provider);
|
||||
r.validate();
|
||||
plan = gen->plan(r);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
if (!flag(b, "confirm")) return json_out(dry_run(plan, "image generation"));
|
||||
try {
|
||||
return json_out(ticket_reply(gen->submit(r, true)));
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response video(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
std::string provider = provider_of(b, false);
|
||||
md::VideoRequest r;
|
||||
r.prompt = text(b, "prompt");
|
||||
r.negative_prompt = text(b, "negative_prompt");
|
||||
r.model = text(b, "model");
|
||||
r.duration_seconds = number(b, "duration_seconds");
|
||||
r.aspect_ratio = text(b, "aspect_ratio");
|
||||
r.resolution = text(b, "resolution");
|
||||
r.seed = seed_of(b);
|
||||
r.image_url = text(b, "image_url");
|
||||
r.end_image_url = text(b, "end_image_url");
|
||||
r.extra = object_at(b, "extra");
|
||||
std::shared_ptr<md::VideoGenerator> gen;
|
||||
Json plan;
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
gen = reg_.video(provider);
|
||||
r.validate();
|
||||
plan = gen->plan(r);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
if (!flag(b, "confirm")) return json_out(dry_run(plan, "video generation"));
|
||||
try {
|
||||
return json_out(ticket_reply(gen->submit(r, true)));
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response poll(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
md::JobTicket t = ticket_of(b);
|
||||
try {
|
||||
require_configured(t.provider);
|
||||
return json_out(status_json(generator_poll(t)));
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response wait(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
md::JobTicket t = ticket_of(b);
|
||||
md::WaitPolicy policy;
|
||||
policy.max_attempts = number(b, "max_attempts", 15);
|
||||
policy.max_wait_seconds = number(b, "max_wait_seconds", 45);
|
||||
policy.first_interval_ms = number(b, "first_interval_ms", 0);
|
||||
if (policy.max_attempts > kDemoMaxAttempts) bad("max_attempts is limited to " + std::to_string(kDemoMaxAttempts) + " in the demo");
|
||||
if (policy.max_wait_seconds > kDemoMaxWaitSeconds)
|
||||
bad("max_wait_seconds is limited to " + std::to_string(kDemoMaxWaitSeconds) + " in the demo");
|
||||
try {
|
||||
md::JobRunner runner(policy); // validates against the package caps
|
||||
require_configured(t.provider);
|
||||
std::shared_ptr<md::MediaGenerator> gen = generator_for(t);
|
||||
return json_out(status_json(runner.wait(*gen, t)));
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response cancel(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
md::JobTicket t = ticket_of(b);
|
||||
Json plan = Json::object();
|
||||
plan.set("provider", Json(t.provider));
|
||||
plan.set("job", Json(t.id));
|
||||
plan.set("action", Json("cancel (some providers delete the generation)"));
|
||||
if (!flag(b, "confirm")) return json_out(dry_run(plan, "cancel a job"));
|
||||
try {
|
||||
require_configured(t.provider);
|
||||
bool done = generator_for(t)->cancel(t, true);
|
||||
Json o = Json::object();
|
||||
o.set("cancelled", Json(done));
|
||||
if (!done) o.set("note", Json(t.provider + " has no cancel endpoint; nothing was sent"));
|
||||
return json_out(o);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- generic table call ---------------------------------------------------
|
||||
fw::Response call(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
std::string provider = provider_of(b, false, false);
|
||||
std::string key = text(b, "key", true);
|
||||
md::CallArgs args;
|
||||
args.path = strings_at(b, "path");
|
||||
args.query = strings_at(b, "query");
|
||||
args.headers = strings_at(b, "headers");
|
||||
args.body = object_at(b, "body");
|
||||
if (const Json* files = b.find("files"); files && !files->is_null()) {
|
||||
if (!files->is_array() || files->as_array().size() > md::MediaCaps::kMaxFiles) bad("files must be an array of at most 16 objects");
|
||||
for (const Json& f : files->as_array()) {
|
||||
if (!f.is_object()) bad("each file must be an object");
|
||||
md::FilePart part;
|
||||
part.field = text(f, "field", true);
|
||||
part.filename = text(f, "filename");
|
||||
part.content_type = text(f, "content_type");
|
||||
if (!security::crypto::Base64::decode(text(f, "data_base64", true), part.data)) bad("data_base64 is not valid base64");
|
||||
args.files.push_back(part);
|
||||
}
|
||||
}
|
||||
md::PreparedCall prepared;
|
||||
std::shared_ptr<md::MediaService> service;
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
service = reg_.service(provider, key);
|
||||
prepared = service->prepare(key, args);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
if (prepared.effect != md::Effect::kRead && !flag(b, "confirm")) return json_out(dry_run(prepared.summary(), "provider call"));
|
||||
try {
|
||||
llm::RawResponse raw = service->execute(prepared, true);
|
||||
Json o = Json::object();
|
||||
o.set("request", prepared.summary());
|
||||
o.set("status", Json(raw.status));
|
||||
o.set("content_type", Json(raw.content_type));
|
||||
o.set("bytes", Json(static_cast<double>(raw.body.size())));
|
||||
if (raw.content_type.find("json") != std::string::npos) {
|
||||
try {
|
||||
o.set("json", Json::parse(raw.body));
|
||||
} catch (const std::exception&) {
|
||||
o.set("text", Json(raw.body.substr(0, kTextPreviewCap)));
|
||||
}
|
||||
} else if (raw.content_type.rfind("text/", 0) == 0) {
|
||||
o.set("text", Json(raw.body.substr(0, kTextPreviewCap)));
|
||||
} else {
|
||||
o.set("note", Json("binary reply; use the generation seam or the provider client to fetch it"));
|
||||
}
|
||||
return json_out(o);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response account(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
std::string provider = provider_of(b, false, false);
|
||||
if (md::MediaRegistry::find(provider)->account_key.empty()) bad(provider + " has no free account read");
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
require_configured_locked(provider);
|
||||
Json o = Json::object();
|
||||
o.set("provider", Json(provider));
|
||||
o.set("read", Json(md::MediaRegistry::find(provider)->account_key));
|
||||
o.set("result", reg_.account(provider));
|
||||
return json_out(o);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
// Downloads a result URL (public or pre-signed) and reports its size and
|
||||
// digest; no provider credential is involved.
|
||||
fw::Response fetch(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
std::string url = text(b, "url", true);
|
||||
try {
|
||||
md::Downloader::validate_url(url);
|
||||
md::Downloader dl;
|
||||
std::string type;
|
||||
std::string bytes = dl.fetch_url(url, &type);
|
||||
Json o = Json::object();
|
||||
o.set("bytes", Json(static_cast<double>(bytes.size())));
|
||||
o.set("content_type", Json(type));
|
||||
o.set("sha256", Json(security::crypto::Sha256::hex(security::crypto::Sha256::hash(bytes))));
|
||||
if (bytes.size() <= 128 * 1024) o.set("data_base64", Json(security::crypto::Base64::encode(bytes)));
|
||||
return json_out(o);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
fw::Response webhook(const fw::Request& req) {
|
||||
Json b = parse_body(req);
|
||||
std::string secret = text(b, "secret", true);
|
||||
std::string id = text(b, "webhook_id", true);
|
||||
std::string ts = text(b, "timestamp", true);
|
||||
std::string sig = text(b, "signature", true);
|
||||
std::string body = text(b, "payload", true);
|
||||
if (body.size() > md::MediaCaps::kMaxWebhookBodyBytes) bad("payload is over the webhook body cap");
|
||||
bool ok = md::ReplicateWebhook::verify(secret, id, ts, sig, body, static_cast<long>(std::time(nullptr)));
|
||||
Json o = Json::object();
|
||||
o.set("valid", Json(ok));
|
||||
o.set("scheme", Json("Standard Webhooks (HMAC-SHA256 over id.timestamp.payload)"));
|
||||
o.set("tolerance_seconds", Json(static_cast<double>(md::MediaCaps::kWebhookToleranceSeconds)));
|
||||
return json_out(o);
|
||||
}
|
||||
|
||||
private:
|
||||
Json provider_json(const md::ProviderInfo& p) {
|
||||
Json x = Json::object();
|
||||
x.set("id", Json(p.id));
|
||||
x.set("name", Json(p.name));
|
||||
x.set("configured", Json(md::MediaRegistry::configured(p.id)));
|
||||
Json env = Json::array();
|
||||
for (const std::string& e : p.env) env.push(Json(e));
|
||||
x.set("env", env);
|
||||
x.set("image", Json(p.image));
|
||||
x.set("video", Json(p.video));
|
||||
x.set("evidence", Json(md::MediaRegistry::evidence_name(p.evidence)));
|
||||
x.set("console", Json(p.console_url));
|
||||
x.set("account_read", Json(p.account_key));
|
||||
x.set("note", Json(p.note));
|
||||
x.set("calls", Json(static_cast<double>(md::MediaRegistry::table(p.id).size())));
|
||||
Json models = Json::array();
|
||||
if (p.image)
|
||||
for (const std::string& m : reg_.image(p.id)->models()) models.push(Json(m));
|
||||
x.set("image_models", models);
|
||||
Json vmodels = Json::array();
|
||||
if (p.video)
|
||||
for (const std::string& m : reg_.video(p.id)->models()) vmodels.push(Json(m));
|
||||
x.set("video_models", vmodels);
|
||||
return x;
|
||||
}
|
||||
|
||||
std::string provider_of(const Json& b, bool image, bool generating = true) {
|
||||
std::string id = text(b, "provider", true);
|
||||
const md::ProviderInfo* p = md::MediaRegistry::find(id);
|
||||
if (!p) bad("unknown provider '" + id + "'");
|
||||
if (generating && image && !p->image) bad(id + " has no image generation");
|
||||
if (generating && !image && !p->video) bad(id + " has no video generation");
|
||||
return id;
|
||||
}
|
||||
|
||||
md::JobTicket ticket_of(const Json& b) {
|
||||
const Json* t = b.find("ticket");
|
||||
if (!t || !t->is_object()) bad("ticket is required (the object returned by image or video)");
|
||||
try {
|
||||
md::JobTicket ticket = md::JobTicket::from_json(*t);
|
||||
if (!md::MediaRegistry::find(ticket.provider)) bad("unknown provider '" + ticket.provider + "'");
|
||||
return ticket;
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<md::MediaGenerator> generator_for(const md::JobTicket& t) {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
if (t.kind == md::MediaKind::kVideo) return reg_.video(t.provider);
|
||||
return reg_.image(t.provider);
|
||||
}
|
||||
|
||||
md::JobStatus generator_poll(const md::JobTicket& t) {
|
||||
if (t.immediate) {
|
||||
md::JobStatus s;
|
||||
s.state = md::JobState::kSucceeded;
|
||||
s.raw_state = "immediate";
|
||||
s.assets = t.assets;
|
||||
return s;
|
||||
}
|
||||
return generator_for(t)->poll(t);
|
||||
}
|
||||
|
||||
void require_configured(const std::string& provider) {
|
||||
std::lock_guard<std::mutex> lock(mu_);
|
||||
require_configured_locked(provider);
|
||||
}
|
||||
void require_configured_locked(const std::string& provider) {
|
||||
if (!md::MediaRegistry::configured(provider)) {
|
||||
const md::ProviderInfo* p = md::MediaRegistry::find(provider);
|
||||
throw llm::NotConfiguredError(provider, p->env[0]);
|
||||
}
|
||||
}
|
||||
|
||||
[[noreturn]] static void map(const llm::ApiError& e) {
|
||||
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "media_not_configured", e.message());
|
||||
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
if (e.type() == "WAIT_EXPIRED") throw fw::HttpError(504, "media_wait_expired", e.message());
|
||||
throw fw::HttpError(502, "media_error", e.vendor() + " " + std::to_string(e.http_status()) + " " + e.type() + ": " + e.message());
|
||||
}
|
||||
|
||||
std::mutex mu_;
|
||||
md::MediaRegistry reg_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_ai_media_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
auto routes = std::make_shared<MediaRoutes>();
|
||||
fw::Environment* env = services.templates.get();
|
||||
services.templates->add_template("ai_media.html", kPage);
|
||||
services.add_feature("ai-media", "/ai/media");
|
||||
|
||||
router.get("/ai/media", "ai-media-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Media generation");
|
||||
return fw::Response(200, env->render("ai_media.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
router.get("/ai/media/status", "ai-media-status", [routes](fw::Request req) {
|
||||
(void)req;
|
||||
return json_out(routes->status());
|
||||
});
|
||||
router.get("/ai/media/calls", "ai-media-calls", [routes](fw::Request req) {
|
||||
return json_out(routes->calls(req.query_or("provider", "")));
|
||||
});
|
||||
|
||||
using Handler = fw::Response (MediaRoutes::*)(const fw::Request&);
|
||||
auto bind = [routes](Handler h) { return [routes, h](fw::Request req) { return ((*routes).*h)(req); }; };
|
||||
router.post("/ai/media/image", "ai-media-image", bind(&MediaRoutes::image));
|
||||
router.post("/ai/media/video", "ai-media-video", bind(&MediaRoutes::video));
|
||||
router.post("/ai/media/poll", "ai-media-poll", bind(&MediaRoutes::poll));
|
||||
router.post("/ai/media/wait", "ai-media-wait", bind(&MediaRoutes::wait));
|
||||
router.post("/ai/media/call", "ai-media-call", bind(&MediaRoutes::call));
|
||||
router.post("/ai/media/account", "ai-media-account", bind(&MediaRoutes::account));
|
||||
router.post("/ai/media/fetch", "ai-media-fetch", bind(&MediaRoutes::fetch));
|
||||
router.post("/ai/media/webhook", "ai-media-webhook", bind(&MediaRoutes::webhook));
|
||||
router.del("/ai/media/job", "ai-media-cancel", bind(&MediaRoutes::cancel));
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Media page: thin client over /ai/media/* (all limits are enforced server side).
|
||||
(function () {
|
||||
"use strict";
|
||||
var $ = function (id) { return document.getElementById(id); };
|
||||
var out = $("md-out"), err = $("md-error");
|
||||
var providers = [];
|
||||
var last = null; // last ticket returned by image / video
|
||||
|
||||
var routes = {
|
||||
"image": function (p) { return { provider: p, prompt: "A lighthouse at dusk, oil painting", model: "", aspect_ratio: "", count: 1 }; },
|
||||
"video": function (p) { return { provider: p, prompt: "Waves rolling into a harbour", model: "", duration_seconds: 5 }; },
|
||||
"poll": function () { return { ticket: last || {} }; },
|
||||
"wait": function () { return { ticket: last || {}, max_attempts: 10, max_wait_seconds: 40 }; },
|
||||
"job": function () { return { ticket: last || {} }; },
|
||||
"account": function (p) { return { provider: p }; },
|
||||
"call": function (p) { return { provider: p, key: "", path: {}, query: {}, body: {} }; },
|
||||
"fetch": function () { return { url: "https://example.com/result.png" }; },
|
||||
"webhook": function () { return { secret: "whsec_...", webhook_id: "msg_...", timestamp: "0", signature: "v1,...", payload: "{}" }; }
|
||||
};
|
||||
var labels = {
|
||||
"image": "image: submit a generation",
|
||||
"video": "video: submit a generation",
|
||||
"poll": "poll: one status check",
|
||||
"wait": "wait: poll under caps",
|
||||
"job": "job: cancel (DELETE)",
|
||||
"account": "account: free credential check",
|
||||
"call": "call: any table row",
|
||||
"fetch": "fetch: download a result URL",
|
||||
"webhook": "webhook: verify a Replicate signature"
|
||||
};
|
||||
|
||||
var route = $("md-route"), prov = $("md-provider");
|
||||
Object.keys(routes).forEach(function (k) {
|
||||
var o = document.createElement("option");
|
||||
o.value = k; o.textContent = labels[k];
|
||||
route.appendChild(o);
|
||||
});
|
||||
|
||||
function fill() {
|
||||
var p = prov.value || "fal";
|
||||
$("md-body").value = JSON.stringify(routes[route.value](p), null, 1);
|
||||
}
|
||||
route.onchange = fill;
|
||||
prov.onchange = fill;
|
||||
|
||||
function send(confirm) {
|
||||
var body;
|
||||
try { body = JSON.parse($("md-body").value || "{}"); }
|
||||
catch (e) { err.textContent = "Body is not valid JSON"; return; }
|
||||
if (confirm) body.confirm = true;
|
||||
err.textContent = "";
|
||||
out.textContent = "...";
|
||||
var key = route.value;
|
||||
var url = "/ai/media/" + (key === "job" ? "job" : key);
|
||||
fetch(url, {
|
||||
method: key === "job" ? "DELETE" : "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body)
|
||||
}).then(function (res) {
|
||||
return res.text().then(function (t) {
|
||||
var j;
|
||||
try { j = JSON.parse(t); } catch (e) { j = { message: t }; }
|
||||
if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; return; }
|
||||
if (j.ticket) last = j.ticket;
|
||||
out.textContent = JSON.stringify(j, null, 2);
|
||||
var imgs = (j.ticket && j.ticket.assets) || j.assets || [];
|
||||
imgs.forEach(function (a) {
|
||||
if (a.data_base64 && /^image\//.test(a.mime || "")) {
|
||||
var im = document.createElement("img");
|
||||
im.src = "data:" + a.mime + ";base64," + a.data_base64;
|
||||
im.style.maxWidth = "100%";
|
||||
out.appendChild(document.createElement("br"));
|
||||
out.appendChild(im);
|
||||
}
|
||||
});
|
||||
});
|
||||
}).catch(function (e) { err.textContent = String(e); out.textContent = ""; });
|
||||
}
|
||||
$("md-send").onclick = function () { send(false); };
|
||||
$("md-confirm").onclick = function () { send(true); };
|
||||
|
||||
fetch("/ai/media/status").then(function (r) { return r.json(); }).then(function (s) {
|
||||
providers = s.providers || [];
|
||||
providers.forEach(function (p) {
|
||||
var o = document.createElement("option");
|
||||
o.value = p.id;
|
||||
o.textContent = p.name + (p.configured ? "" : " (no key)");
|
||||
prov.appendChild(o);
|
||||
});
|
||||
var ready = providers.filter(function (p) { return p.configured; }).length;
|
||||
$("md-config").textContent = ready
|
||||
? ready + " of " + providers.length + " providers have a credential. Paid calls need confirm."
|
||||
: "No provider credential found: calls answer 503 until one is configured; dry runs still work.";
|
||||
var ul = $("md-unsupported");
|
||||
(s.unsupported || []).forEach(function (u) {
|
||||
var li = document.createElement("li");
|
||||
li.textContent = u.name + ": " + u.reason;
|
||||
ul.appendChild(li);
|
||||
});
|
||||
fill();
|
||||
});
|
||||
})();
|
||||
Vendored
+254
@@ -0,0 +1,254 @@
|
||||
# Package media - Specification
|
||||
|
||||
Owned directory: `src/vendors/media/`. Namespace `molo::media`. One subdirectory
|
||||
per provider, one shared `common/`. Depends on `molo::llm` (`ApiClient`,
|
||||
`Service`, `ApiError`, `Config`), `molo::clients` and `security/crypto`
|
||||
(Base64, Sha256, Hmac). Plan: `plans/18-breadth.md` section 2, method of
|
||||
`plans/17-llm-vendor-platforms.md`. Demo: `example/app/ai/MediaDemo.cpp`,
|
||||
routes `/ai/media`. Nothing in `src/llm` was edited.
|
||||
|
||||
## 1. Scope and module map
|
||||
|
||||
Twelve providers, one call table each (`<p>/calls.cpp`, checked by
|
||||
`tools/check_spec.py`), one client class per host, generator adapters on a
|
||||
neutral seam.
|
||||
|
||||
| Directory | Classes | Covers |
|
||||
|---|---|---|
|
||||
| `common/` | `CallTable`, `MediaService`, `PreparedCall`, `ImageGenerator`, `VideoGenerator`, `JobTicket`, `JobStatus`, `JobRunner`, `Downloader`, `OutputScanner`, `MediaRegistry`, `ClientFactory`, `AdapterSupport`, `MediaCaps` | the seam, request building, polling, downloads, provider list |
|
||||
| `stability/` | `StabilityClient`, `StabilityImageGenerator` | v2beta Stable Image (generate ultra / core / sd3, edit x7, upscale x3, control x4), 3D (2), Stable Audio (6 + results), v1 engines list, account, balance, v1 text-to-image / image-to-image / masking |
|
||||
| `bfl/` | `BflClient`, `BflImageGenerator`, `BflVideoGenerator` | all 25 FLUX model and tool endpoints, get_result polling, credits, fine-tune list / details / delete |
|
||||
| `runway/` | `RunwayClient`, `RunwayImageGenerator`, `RunwayVideoGenerator` | every SDK route: generation (18), recipes (7), tasks, uploads, organization, avatars, voices, documents, routers, workflows, realtime sessions |
|
||||
| `luma/` | `LumaClient`, `LumaVideoGenerator`, `LumaImageGenerator` | Ray video, Photon image, reframe, modify, upscale, audio, list, delete, concepts, credits |
|
||||
| `replicate/` | `ReplicateClient`, `ReplicateWebhook`, `ReplicateJobs`, `ReplicateImageGenerator`, `ReplicateVideoGenerator` | all 36 operations: predictions (model, version, deployment), models, versions, deployments, files, trainings, hardware, collections, search, account, webhook secret; Standard Webhooks verification |
|
||||
| `fal/` | `FalQueue`, `FalPlatform`, `FalImageGenerator`, `FalVideoGenerator` | queue API (submit, submit with path, status, result, cancel) and the model-related Platform API (search, pricing, estimate, usage, analytics, billing events, request listing, payload deletion, meta) |
|
||||
| `ideogram/` | `IdeogramClient`, `IdeogramImageGenerator`, `IdeogramVideoGenerator` | all 70 v2 operations (image, SVG, video, ad tools, generations, account); free price quote through `dry_run=true` |
|
||||
| `recraft/` | `RecraftClient`, `RecraftImageGenerator` | all 22 operations: generation (3 routes), image tools, styles, prompt enhancement, user |
|
||||
| `leonardo/` | `LeonardoClient`, `LeonardoImageGenerator`, `LeonardoVideoGenerator` | v2 async and sync generations, models; v1 generation get and `me` (prose only) |
|
||||
| `minimax/` | `MinimaxClient`, `MinimaxVideoGenerator` | Hailuo video v1 (create, query, file retrieve) and v2 (create, query, list, delete) |
|
||||
| `kling/` | `KlingJwt`, `KlingClient`, `KlingVideoGenerator` | text2video, image2video, image generation, task queries; JWT auth |
|
||||
| `higgsfield/` | `HiggsfieldClient`, `HiggsfieldJobs`, `HiggsfieldImageGenerator`, `HiggsfieldVideoGenerator` | the shared lifecycle (submit by model path, status, cancel, delete) for every model |
|
||||
|
||||
Products without a public API (not implemented, shown in
|
||||
`MediaRegistry::unsupported()` and on the demo page):
|
||||
|
||||
- **Midjourney**: no public API. Its docs describe the web and Discord products
|
||||
only; third-party relays are unofficial and break its terms.
|
||||
- **Pika**: `pika.art/api` advertises a membership API that resells other
|
||||
vendors' models, but no endpoint, schema or auth document exists
|
||||
(2026-10-25). Pika models are reachable through fal.
|
||||
|
||||
## 2. The seam (`common/job.h`)
|
||||
|
||||
```
|
||||
ImageRequest / VideoRequest neutral request, validate() with caps
|
||||
ImageGenerator::plan(req) exact request as JSON, no network, no credential
|
||||
ImageGenerator::submit(req, confirm) JobTicket (confirm has no default)
|
||||
MediaGenerator::poll(ticket) JobStatus {queued|running|succeeded|failed|cancelled}
|
||||
MediaGenerator::cancel(ticket, confirm) false when the API has no cancel
|
||||
JobRunner::wait(generator, ticket) polls under hard caps
|
||||
Downloader::fetch(asset) bytes of a URL or an inline base64 output
|
||||
```
|
||||
|
||||
Synchronous providers (Stability, Recraft, Ideogram by default) return an
|
||||
`immediate` ticket that already carries its assets; `JobRunner` answers those
|
||||
without a poll. A failed job is a `JobStatus`, a transport or auth failure is an
|
||||
`ApiError`.
|
||||
|
||||
## 3. Static caps (see `common/caps.h`)
|
||||
|
||||
| Knob | Value |
|
||||
|---|---|
|
||||
| Prompt | 32000 chars |
|
||||
| Body fields / one text field | 128 / 1 MiB |
|
||||
| Uploaded file / files per call | 16 MiB / 16 |
|
||||
| Query args / value | 16 / 1024 chars |
|
||||
| Path argument / segments of a multi-segment id | 256 chars / 6 |
|
||||
| Extra headers | 8, 512 chars each |
|
||||
| Images per request / video seconds / dimension | 8 / 60 / 8192 |
|
||||
| Poll interval floor / ceiling | 500 ms / 30000 ms |
|
||||
| Poll attempts / total wait | 240 / 1800 s (demo: 30 / 60 s) |
|
||||
| Assets per job / download / URL | 16 / 128 MiB / 4096 chars |
|
||||
| Scan depth / nodes | 8 / 4096 |
|
||||
| Page walk / items returned | 20 / 1000 |
|
||||
| Webhook tolerance / body | 300 s / 1 MiB |
|
||||
| Call table | 256 rows |
|
||||
|
||||
`src/llm` caps still apply (32 MiB request, 64 MiB response).
|
||||
|
||||
## 4. Contracts
|
||||
|
||||
- Every call lives in the provider's table: key, method, path, spec operation
|
||||
id, body kind (`none|json|multipart|either`), accepted body fields, accepted
|
||||
query parameters, effect (`read|paid|write|delete`).
|
||||
- `MediaService::prepare` is pure. It rejects unknown keys, missing or extra
|
||||
path arguments, ids outside `[A-Za-z0-9._:=~-]` (and `.` / `..`), unknown query
|
||||
parameters, unknown body fields, files on a JSON row, empty or oversized
|
||||
files, bad header names. A rejected request never reaches the network.
|
||||
- `MediaService::execute` refuses any non-read effect unless `confirmed` is
|
||||
true. Typed methods that delete or cancel take `confirm` with no default.
|
||||
The only two lowered effects are named and documented: Ideogram `quote`
|
||||
(`dry_run=true`, defined by the spec as not billed) and Runway
|
||||
`organization_usage` (a POST that only queries).
|
||||
- No key: `configured()` is false and every call throws `NotConfiguredError`
|
||||
(503) before any traffic. Keys travel in headers only.
|
||||
- Lists report truncation (`"truncated": true`), never silently.
|
||||
- Poll caps: the runner never goes below the interval floor, throws
|
||||
`WAIT_EXPIRED` (504) naming the job when attempts or total wait run out, and
|
||||
never cancels on its own.
|
||||
- Errors stay `ApiError` with the vendor's status. Envelopes that hide an error
|
||||
in a 200 are converted: MiniMax `base_resp.status_code`, Kling `code`.
|
||||
|
||||
## 5. Credentials
|
||||
|
||||
| Provider | Variables | Page |
|
||||
|---|---|---|
|
||||
| Stability AI | `STABILITY_API_KEY` | https://platform.stability.ai/account/keys (read: redirects to sign-in) |
|
||||
| Black Forest Labs | `BFL_API_KEY` | https://dashboard.bfl.ai/ (read: redirects to sign-in) |
|
||||
| Runway | `RUNWAY_API_KEY` (also `RUNWAYML_API_SECRET`, the SDK name) | https://dev.runwayml.com/ (read: 200) |
|
||||
| Luma | `LUMA_API_KEY` | https://lumalabs.ai/api/keys (read: 200) |
|
||||
| Replicate | `REPLICATE_API_TOKEN` | https://replicate.com/account/api-tokens (read: redirects to sign-in) |
|
||||
| fal | `FAL_KEY` | https://fal.ai/dashboard/keys (read: redirects to sign-in) |
|
||||
| Ideogram | `IDEOGRAM_API_KEY` | https://ideogram.ai/manage-api (unverified: Cloudflare challenge blocked the fetch) |
|
||||
| Recraft | `RECRAFT_API_TOKEN` | https://www.recraft.ai/profile/api (read: redirects to sign-in with that callback) |
|
||||
| Leonardo | `LEONARDO_API_KEY` | https://app.leonardo.ai/api-access (docs; fetch answered 429) |
|
||||
| MiniMax | `MINIMAX_API_KEY` | https://platform.minimax.io/user-center/basic-information/interface-key (read: redirects to /console/access) |
|
||||
| Kling | `KLING_ACCESS_KEY` and `KLING_SECRET_KEY` | https://app.klingai.com/global/dev/api-key (read: redirect) |
|
||||
| Higgsfield | `HF_API_KEY_ID` and `HF_API_KEY_SECRET` | https://cloud.higgsfield.ai/ (docs; read: redirect) |
|
||||
|
||||
Add these rows to `src/llm/SPEC.md` section 5 at integration.
|
||||
|
||||
## 6. Sources
|
||||
|
||||
Per provider, the machine-readable source `tools/check_spec.py` reads.
|
||||
|
||||
| Provider | Source |
|
||||
|---|---|
|
||||
| Stability | https://docs.stability.ai/openapi.json (v2beta) and https://docs.stability.ai/openapi-v1.json; prose index https://docs.stability.ai/llms.txt |
|
||||
| BFL | https://api.bfl.ai/openapi.json (version 0.0.1) |
|
||||
| Replicate | https://api.replicate.com/openapi.json (server `https://api.replicate.com/v1`) |
|
||||
| fal | queue: https://fal.ai/docs/api-reference/inference-api/openapi/queue.json; platform: https://api.fal.ai/v1/openapi.json; index https://docs.fal.ai/llms.txt |
|
||||
| Ideogram | https://developer.ideogram.ai/v2/openapi.json |
|
||||
| Recraft | https://www.recraft.ai/docs/api-reference/openapi.json |
|
||||
| Higgsfield | https://docs.higgsfield.ai/docs/openapi.json (declared supplementary by the vendor) plus https://docs.higgsfield.ai/docs/llms.txt |
|
||||
| Leonardo | OpenAPI embedded in https://docs.leonardo.ai/reference/creategeneration.md, `createsyncgeneration.md`, `getmodels.md`; quick start `docs/getting-started.md`, FAQ `docs/api-faq.md` |
|
||||
| MiniMax | OpenAPI embedded in the `platform.minimax.io/docs/api-reference/video-generation-*.md` pages and `file-management-retrieve.md`; index `llms.txt` |
|
||||
| Runway | npm `@runwayml/sdk` 4.24.0: `resources/**/*.js` (paths), `*.d.ts` (fields), `client.js` (base URL, `X-Runway-Version` default 2024-11-06) |
|
||||
| Luma | npm `lumaai` 1.19.1: `resources/**/*.js`, `*.d.ts`, `index.js` (base URL) |
|
||||
| Kling | none machine-readable; community sources (a third-party OpenAPI listing, a skill pack) plus live probes |
|
||||
|
||||
Cross-check: Stability, BFL, Replicate, Ideogram and Recraft are also covered by
|
||||
live unauthenticated probes of the same paths (section 8). Standard Webhooks
|
||||
vector for the Replicate verifier comes from the Standard Webhooks project.
|
||||
|
||||
## 7. Verified against spec
|
||||
|
||||
`python3 -I src/vendors/media/tools/check_spec.py`, run 2026-10-25 (twelve
|
||||
providers, all green; a deliberately corrupted BFL row failed it with both a row
|
||||
error and a coverage error, then was restored):
|
||||
|
||||
| Provider | Rows | Source operations | Result |
|
||||
|---|---|---|---|
|
||||
| stability | 34 | 37 (3 alpha operations excluded) | ok |
|
||||
| bfl | 30 | 30 | ok |
|
||||
| replicate | 36 | 36 | ok |
|
||||
| fal | 16 | every queue and model-related platform operation (1 SSE stream route excluded; 78 platform operations outside model inference out of scope) | ok |
|
||||
| ideogram | 70 | 70 | ok |
|
||||
| recraft | 22 | 22 | ok |
|
||||
| leonardo | 6 | 3 (3 v1 rows prose only, listed by the checker) | ok |
|
||||
| runway | 66 | 66 SDK calls | ok |
|
||||
| luma | 13 | 13 SDK calls | ok |
|
||||
| minimax | 7 | 7 (5 reference pages listed as not implemented) | ok |
|
||||
| higgsfield | 10 | 9 (1 generic row by design) | ok |
|
||||
| kling | 6 | probe only | UNVERIFIED |
|
||||
|
||||
The checker verifies: each row's method and path exist, operation ids match,
|
||||
body kind matches the spec content types, every body field and query parameter
|
||||
we send exists, each response schema field we read is reachable, and every
|
||||
source operation is in the table or in the explicit exclusion list.
|
||||
|
||||
## 8. Verified live
|
||||
|
||||
Unauthenticated probes, 2026-10-25, recorded by `probe` runs and by
|
||||
`tests/src/vendors/media/test_live.cpp` (`MOLO_LIVE_NET=1`, 13 providers, green):
|
||||
|
||||
| Host and path | Invalid key answered |
|
||||
|---|---|
|
||||
| api.stability.ai `/v1/user/balance` | 401 `{"id","message","name"}` |
|
||||
| api.stability.ai `/v2beta/results/00000000` | 400 `{"errors":["id: string must contain exactly 64 character(s)"],...}` (the result id is 64 characters) |
|
||||
| api.bfl.ai `/v1/credits` | 422 `{"detail":"Invalid API key format"}` |
|
||||
| api.replicate.com `/v1/account` | 401 `{"title","detail","status"}` |
|
||||
| api.fal.ai `/v1/models/pricing` | 401 `{"error":{"type":"authorization_error",...}}` |
|
||||
| queue.fal.run `/fal-ai/flux/requests/<id>/status` | 401 `{"detail":"Authentication is required"}` |
|
||||
| api.fal.ai `/v1/models`, `/v1/meta` | 200 without any key (public) |
|
||||
| api.ideogram.ai `/v2/account/api-keys` | 401 `{"message":"Access denied..."}`; `/v2/account/usage` answers 400 for the missing `start_time` before auth |
|
||||
| external.api.recraft.ai `/v1/users/me` | 401 `request unauthorized` (plain text) |
|
||||
| cloud.leonardo.ai `/api/rest/v2/models`, `/v1/me` | 401 `{"error","path","code":"access-denied"}` |
|
||||
| api.dev.runwayml.com `/v1/organization` | 401 `{"error":"...did not start with key_","docUrl"}` |
|
||||
| api.lumalabs.ai `/dream-machine/v1/credits` | 403 `{"detail":"Not authenticated"}` |
|
||||
| api.minimax.io `/v1/query/video_generation` | 200 with `base_resp.status_code 1004` (converted to an ApiError) |
|
||||
| api-singapore.klingai.com `/v1/videos/text2video/...` | 401 `{"code":1002,...}`; api.klingai.com answers `{"code":1000,"message":"Auth failed"}` |
|
||||
| api.higgsfield.ai `/requests/<id>/status` | 401 `{"detail":"Invalid credentials"}` |
|
||||
|
||||
No success body of any provider has been seen: no key exists here. Live tests
|
||||
with keys (`MOLO_LIVE_NET=1` plus the provider variables) run one free read per
|
||||
provider (`MediaRegistry::account`). MiniMax, Kling and Higgsfield have no free
|
||||
account read, so a key is never proven for them without a paid or resource call.
|
||||
|
||||
## 9. Unverified
|
||||
|
||||
Every shape inferred rather than read from a spec or seen:
|
||||
|
||||
- All success and result bodies, for every provider (no key). Status words and
|
||||
result locations come from the specs or SDK types where they exist (BFL
|
||||
`StatusResponse` enum and `result.sample`, fal `QueueStatus`, Ideogram
|
||||
`GenerationResponse`, Higgsfield `RequestStatus`, Luma and Runway SDK
|
||||
types, MiniMax pages) and are exercised by local mapper tests only.
|
||||
- **Kling**: all six paths, the request fields, the `{code,data:{task_id,
|
||||
task_status,task_result}}` envelope, the statuses and the JWT claims
|
||||
(`iss`, `exp`, `nbf`, 1800 s) come from community sources, not from Kling.
|
||||
Only the paths and the auth refusal were probed. The JWT builder matches an
|
||||
independent HS256 implementation but Kling's acceptance of it was never seen.
|
||||
- **Leonardo**: the polling route `GET /api/rest/v1/generations/{id}`, its
|
||||
`generations_by_pk.{status,generated_images[].url}` shape, `GET /v1/me`, and
|
||||
the v1 create body are described by quick start and FAQ prose only (the v1
|
||||
reference pages were removed). Whether a v2 `generationId` is pollable on the
|
||||
v1 route was not confirmed.
|
||||
- **Higgsfield**: only five model paths are in the OpenAPI document; any other
|
||||
path goes through the generic row (`model.submit`) with an open body.
|
||||
- **fal**: result URL locations differ per model; `OutputScanner` finds URLs
|
||||
generically. A failed request is assumed to answer the result fetch with an
|
||||
HTTP 4xx (the queue spec says `COMPLETED` also covers failure).
|
||||
- **BFL**: `polling_url` may point at a regional host; polling uses
|
||||
`get_result?id=` on the configured host instead.
|
||||
- **Recraft** default raster output is lossless WebP (docs); inline results are
|
||||
labelled `image/webp` on that basis.
|
||||
- **Replicate** model names listed by `models()` and the other adapters'
|
||||
`models()` lists are names seen in the docs and specs, not a live catalogue.
|
||||
- Runway `X-Runway-Version` 2024-11-06 is the SDK default; newer versions may
|
||||
change bodies.
|
||||
- Console URLs in section 5 marked as such.
|
||||
|
||||
## 10. Gaps
|
||||
|
||||
- Foundation (`src/llm`, report to its owner): no unauthenticated requests (so
|
||||
fal's public model search needs a key through our client); no GET stream (fal
|
||||
status stream, excluded); per-call timeouts missing; `error_from_response`
|
||||
leaves `type` empty for `{"message"}`, `{"detail"}` and `{"errors"}`
|
||||
envelopes (Stability, BFL, Replicate, Higgsfield, Luma, Runway answer
|
||||
`UNKNOWN`); the 32 MiB request ceiling blocks large video uploads (package
|
||||
files are capped at 16 MiB each); `post_multipart` is bypassed through
|
||||
`request_raw`.
|
||||
- Another suite in the shared test binary unsets `MOLO_LIVE_NET`
|
||||
(`tests/src/google/optimize/test_gaps.cpp`); the media live tests read it once
|
||||
at startup to be immune.
|
||||
- Not implemented: Stability gRPC and v2beta image-to-video (absent from the
|
||||
published spec); MiniMax Video Agent, H3-Context-IR and regeneration tasks;
|
||||
fal Platform assets / serverless / compute / keys / workflows; Replicate list
|
||||
cursors (first page only, `truncated` is set); Ideogram and Replicate webhook
|
||||
receivers (Replicate verification exists); WebSocket / realtime routes of
|
||||
Runway beyond session creation.
|
||||
- Recipes and avatar conversations of Runway are table rows only, with no typed
|
||||
wrapper: use `MediaService::call_json`.
|
||||
- Fetching a result needs a second step: `Downloader` is separate from
|
||||
`poll()` so that callers choose when to spend bandwidth.
|
||||
Vendored
+146
@@ -0,0 +1,146 @@
|
||||
#include "vendors/media/bfl/bfl.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs with_body(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
JobTicket ticket_from(const Json& out, const std::string& model, MediaKind kind) {
|
||||
JobTicket t;
|
||||
t.provider = "bfl";
|
||||
t.model = model;
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(out, "id");
|
||||
if (t.id.empty()) throw llm::ApiError("bfl", 502, "BAD_RESPONSE", "no id in the submit reply");
|
||||
t.meta = Json::object();
|
||||
AdapterSupport::put(t.meta, "cost", AdapterSupport::str(out, "cost"));
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus status_of(BflClient& client, const JobTicket& ticket, MediaKind kind) {
|
||||
CallArgs a;
|
||||
a.query["id"] = ticket.id;
|
||||
Json out = client.call_json("get_result", a);
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(out, "status");
|
||||
s.state = BflImageGenerator::state_of(s.raw_state);
|
||||
if (const Json* p = out.find("progress"); p && p->is_number()) s.progress = p->as_number();
|
||||
if (s.state == JobState::kSucceeded) {
|
||||
if (const Json* r = out.find("result")) s.assets = OutputScanner::scan(*r, kind);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "Ready but no result URL in the reply";
|
||||
}
|
||||
} else if (s.state == JobState::kFailed) {
|
||||
s.error = s.raw_state;
|
||||
if (const Json* d = out.find("details"); d && d->is_object()) s.error += ": " + d->dump();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall BflClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
|
||||
|
||||
Json BflClient::submit(const std::string& key, const Json& body, bool confirm) {
|
||||
return call_json(key, with_body(body), confirm);
|
||||
}
|
||||
|
||||
Json BflClient::get_result(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.query["id"] = id;
|
||||
return call_json("get_result", a);
|
||||
}
|
||||
|
||||
Json BflClient::credits() { return call_json("credits"); }
|
||||
Json BflClient::my_finetunes() { return call_json("my_finetunes"); }
|
||||
|
||||
Json BflClient::finetune_details(const std::string& finetune_id) {
|
||||
CallArgs a;
|
||||
a.query["finetune_id"] = finetune_id;
|
||||
return call_json("finetune_details", a);
|
||||
}
|
||||
|
||||
Json BflClient::delete_finetune(const std::string& finetune_id, bool confirm) {
|
||||
Json b = Json::object();
|
||||
b.set("finetune_id", Json(finetune_id));
|
||||
return call_json("delete_finetune", with_body(b), confirm);
|
||||
}
|
||||
|
||||
JobState BflImageGenerator::state_of(const std::string& status) {
|
||||
if (status == "Ready") return JobState::kSucceeded;
|
||||
if (status == "Pending" || status == "Reasoning") return JobState::kQueued;
|
||||
if (status == "Generating") return JobState::kRunning;
|
||||
if (status == "Task not found" || status == "Error" || status == "Request Moderated" ||
|
||||
status == "Content Moderated" || status == "No Card Generated")
|
||||
return JobState::kFailed;
|
||||
return JobState::kRunning;
|
||||
}
|
||||
|
||||
std::vector<std::string> BflImageGenerator::models() const {
|
||||
return {"flux-2-pro", "flux-2-max", "flux-2-flex", "flux-2-klein-9b", "flux-2-klein-4b", "flux-kontext-pro",
|
||||
"flux-kontext-max", "flux-pro-1.1", "flux-pro-1.1-ultra", "flux-dev", "flux-3-image"};
|
||||
}
|
||||
|
||||
Json BflImageGenerator::body(const ImageRequest& r, std::string& key) const {
|
||||
r.validate();
|
||||
key = r.model.empty() ? "flux-2-pro" : r.model;
|
||||
if (r.count != 1) throw llm::ApiError::invalid("bfl", "one image per call (count must be 1)");
|
||||
Json b = Json::object();
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put_int(b, "width", r.width);
|
||||
AdapterSupport::put_int(b, "height", r.height);
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put_seed(b, "seed", r.seed);
|
||||
AdapterSupport::put(b, "input_image", r.source_url);
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json BflImageGenerator::plan(const ImageRequest& request) const {
|
||||
std::string key;
|
||||
Json b = body(request, key);
|
||||
return client_->plan(key, b).summary();
|
||||
}
|
||||
|
||||
JobTicket BflImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
std::string key;
|
||||
Json b = body(request, key);
|
||||
AdapterSupport::need_confirm(confirm, "bfl", "generate");
|
||||
return ticket_from(client_->submit(key, b, true), key, MediaKind::kImage);
|
||||
}
|
||||
|
||||
JobStatus BflImageGenerator::poll(const JobTicket& ticket) { return status_of(*client_, ticket, MediaKind::kImage); }
|
||||
|
||||
Json BflVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put_int(b, "duration", r.duration_seconds);
|
||||
AdapterSupport::put(b, "resolution", r.resolution);
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json BflVideoGenerator::plan(const VideoRequest& request) const {
|
||||
return client_->plan("flux-3-video", body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket BflVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "bfl", "generate");
|
||||
return ticket_from(client_->submit("flux-3-video", b, true), "flux-3-video", MediaKind::kVideo);
|
||||
}
|
||||
|
||||
JobStatus BflVideoGenerator::poll(const JobTicket& ticket) { return status_of(*client_, ticket, MediaKind::kVideo); }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+76
@@ -0,0 +1,76 @@
|
||||
#pragma once
|
||||
|
||||
// Black Forest Labs (FLUX): submit to a model endpoint, poll /v1/get_result.
|
||||
// Auth is the x-key header. Models are endpoints (flux-2-pro, flux-kontext-pro,
|
||||
// flux-pro-1.1-ultra, ...), tools are endpoints under /v1/flux-tools/.
|
||||
|
||||
#include "vendors/media/bfl/calls.h"
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class BflClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.bfl.ai";
|
||||
static constexpr const char* kKeyEnv = "BFL_API_KEY";
|
||||
|
||||
explicit BflClient(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), BflCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::header("bfl", kHost, kKeyEnv, "x-key"); }
|
||||
static std::shared_ptr<BflClient> from_env() {
|
||||
return std::make_shared<BflClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<BflClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<BflClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// key = a POST model / tool row ("flux-2-pro", "tools.erase-v1"). Returns {id, polling_url, cost, ...}.
|
||||
Json submit(const std::string& key, const Json& body, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& body) const;
|
||||
Json get_result(const std::string& id);
|
||||
Json credits();
|
||||
Json my_finetunes();
|
||||
Json finetune_details(const std::string& finetune_id);
|
||||
// Destructive: no default for confirm.
|
||||
Json delete_finetune(const std::string& finetune_id, bool confirm);
|
||||
};
|
||||
|
||||
class BflImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit BflImageGenerator(std::shared_ptr<BflClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "bfl"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override;
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // no cancel endpoint
|
||||
int suggested_poll_ms() const override { return 1500; }
|
||||
|
||||
// Maps the BFL status word onto JobState (shared with the video adapter).
|
||||
static JobState state_of(const std::string& status);
|
||||
|
||||
private:
|
||||
Json body(const ImageRequest& request, std::string& key) const;
|
||||
std::shared_ptr<BflClient> client_;
|
||||
};
|
||||
|
||||
class BflVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit BflVideoGenerator(std::shared_ptr<BflClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "bfl"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override { return {"flux-3-video"}; }
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; }
|
||||
int suggested_poll_ms() const override { return 4000; }
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<BflClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+57
@@ -0,0 +1,57 @@
|
||||
#include "vendors/media/bfl/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"finetune_details", "GET", "/v1/finetune_details", "finetune_details_v1_finetune_details_get", "none", "", "finetune_id", "read"},
|
||||
{"my_finetunes", "GET", "/v1/my_finetunes", "my_finetunes_v1_my_finetunes_get", "none", "", "", "read"},
|
||||
{"delete_finetune", "POST", "/v1/delete_finetune", "delete_finetune_v1_delete_finetune_post", "json", "finetune_id", "", "delete"},
|
||||
{"flux-pro-1.1", "POST", "/v1/flux-pro-1.1", "flux_pro_1_1_v1_flux_pro_1_1_post", "json", "user,prompt,image_prompt,width,height,prompt_upsampling,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-dev", "POST", "/v1/flux-dev", "flux_dev_v1_flux_dev_post", "json", "user,prompt,image_prompt,width,height,steps,prompt_upsampling,seed,guidance,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-pro-1.1-ultra", "POST", "/v1/flux-pro-1.1-ultra", "generate_flux_1_1_ultra_v1_flux_pro_1_1_ultra_post", "json", "user,prompt,prompt_upsampling,seed,aspect_ratio,safety_tolerance,output_format,raw,image_prompt,image_prompt_strength,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-pro-1.0-fill", "POST", "/v1/flux-pro-1.0-fill", "fill_v1_flux_pro_1_0_fill_post", "json", "user,image,mask,prompt,steps,prompt_upsampling,seed,guidance,output_format,safety_tolerance,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-pro-1.0-expand", "POST", "/v1/flux-pro-1.0-expand", "expand_v1_flux_pro_1_0_expand_post", "json", "user,image,top,bottom,left,right,prompt,steps,prompt_upsampling,seed,guidance,output_format,safety_tolerance,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-pro-1.1-ultra-finetuned", "POST", "/v1/flux-pro-1.1-ultra-finetuned", "generate_bigblue_finetuned_v1_flux_pro_1_1_ultra_finetuned_post", "json", "user,finetune_id,finetune_strength,prompt,seed,aspect_ratio,safety_tolerance,output_format,image_prompt,image_prompt_strength,prompt_upsampling,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-flex", "POST", "/v1/flux-2-flex", "generate_flux_2_flex_v1_flux_2_flex_post", "json", "user,prompt,prompt_upsampling,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,input_image_blob_path,seed,width,height,guidance,steps,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-klein-9b", "POST", "/v1/flux-2-klein-9b", "generate_flux_2_klein_9b_v1_flux_2_klein_9b_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-klein-9b-preview", "POST", "/v1/flux-2-klein-9b-preview", "generate_flux_2_klein_9b_kv_v1_flux_2_klein_9b_preview_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-klein-4b", "POST", "/v1/flux-2-klein-4b", "generate_flux_2_klein_4b_v1_flux_2_klein_4b_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-max", "POST", "/v1/flux-2-max", "generate_flux_2_max_v1_flux_2_max_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-pro-preview", "POST", "/v1/flux-2-pro-preview", "generate_flux_2_pro_preview_v1_flux_2_pro_preview_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-2-pro", "POST", "/v1/flux-2-pro", "generate_flux_2_pro_v1_flux_2_pro_post", "json", "user,prompt,disable_pup,input_image,input_image_2,input_image_3,input_image_4,input_image_5,input_image_6,input_image_7,input_image_8,seed,width,height,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-3-video", "POST", "/v1/flux-3-video", "flux_3_video_v1_flux_3_video_post", "json", "user,prompt,aspect_ratio,duration,resolution,version,generate_audio,safety_tolerance,draft,mode,keyframes,start_video,draft_cache", "", "paid"},
|
||||
{"flux-3-image", "POST", "/v1/flux-3-image", "flux_3_image_v1_flux_3_image_post", "json", "user,prompt,images,aspect_ratio,resolution,width,height,safety_tolerance,grounding,version", "", "paid"},
|
||||
{"flux-3-image-klein", "POST", "/v1/flux-3-image-klein", "flux_3_image_klein_v1_flux_3_image_klein_post", "json", "user,prompt,images,aspect_ratio,resolution,width,height,seed,safety_tolerance,output_format,steps", "", "paid"},
|
||||
{"tools.video-upscale-v1", "POST", "/v1/flux-tools/video-upscale-v1", "generate_flux_tools_video_upscale_v1_v1_flux_tools_video_upscale_v1_post", "json", "user,input_video,prompt,creativity,upscale_factor,safety_tolerance,webhook_url,webhook_secret", "", "paid"},
|
||||
{"tools.video-edit-v1", "POST", "/v1/flux-tools/video-edit-v1", "generate_flux_tools_video_edit_v1_v1_flux_tools_video_edit_v1_post", "json", "user,video,prompt,safety_tolerance", "", "paid"},
|
||||
{"tools.outpainting-v1", "POST", "/v1/flux-tools/outpainting-v1", "generate_flux_tools_outpainting_v1_v1_flux_tools_outpainting_v1_post", "json", "user,input_image,width,height,auto_crop,safety_tolerance,output_format,prompt,reference_offset_x,reference_offset_y,mode,disable_pup", "", "paid"},
|
||||
{"tools.vto-v1", "POST", "/v1/flux-tools/vto-v1", "generate_flux_tools_vto_v1_v1_flux_tools_vto_v1_post", "json", "user,prompt,person,garment,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"tools.vto-v2", "POST", "/v1/flux-tools/vto-v2", "generate_flux_tools_vto_v2_v1_flux_tools_vto_v2_post", "json", "user,prompt,person,garment,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"tools.erase-v1", "POST", "/v1/flux-tools/erase-v1", "generate_flux_tools_erase_v1_v1_flux_tools_erase_v1_post", "json", "user,image,mask,dilate_pixels,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"tools.deblur-v1", "POST", "/v1/flux-tools/deblur-v1", "generate_flux_tools_deblur_v1_v1_flux_tools_deblur_v1_post", "json", "user,image,seed,safety_tolerance,output_format,webhook_url,webhook_secret", "", "paid"},
|
||||
{"flux-kontext-max", "POST", "/v1/flux-kontext-max", "generate_flux_kontext_max_v1_flux_kontext_max_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,aspect_ratio,output_format,webhook_url,webhook_secret,prompt_upsampling,safety_tolerance", "", "paid"},
|
||||
{"flux-kontext-pro", "POST", "/v1/flux-kontext-pro", "generate_flux_kontext_pro_v1_flux_kontext_pro_post", "json", "user,prompt,input_image,input_image_2,input_image_3,input_image_4,seed,aspect_ratio,output_format,webhook_url,webhook_secret,prompt_upsampling,safety_tolerance", "", "paid"},
|
||||
{"get_result", "GET", "/v1/get_result", "get_result_v1_get_result_get", "none", "", "id", "read"},
|
||||
{"credits", "GET", "/v1/credits", "get_credits_v1_credits_get", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"AsyncResponse", "id,polling_url,cost,input_mp,output_mp"},
|
||||
{"ResultResponse", "id,status,result,progress,details,preview"},
|
||||
{"CreditsResponse", "credits"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& BflCalls::table() {
|
||||
static const CallTable t("bfl", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t BflCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* BflCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct BflCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
#pragma once
|
||||
|
||||
// Small helpers every generator adapter shares: the confirmation gate, body
|
||||
// assembly that skips unset values, and extra-field merging.
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct AdapterSupport {
|
||||
// Every submit / cancel passes through here first: no confirm, no call.
|
||||
static void need_confirm(bool confirm, const std::string& provider, const std::string& what) {
|
||||
if (!confirm)
|
||||
throw llm::ApiError::invalid(provider, what + " needs explicit confirmation (it costs money or changes a resource)");
|
||||
}
|
||||
|
||||
static void put(clients::body::Json& obj, const std::string& key, const std::string& value) {
|
||||
if (!value.empty()) obj.set(key, clients::body::Json(value));
|
||||
}
|
||||
static void put_int(clients::body::Json& obj, const std::string& key, int value) {
|
||||
if (value > 0) obj.set(key, clients::body::Json(value));
|
||||
}
|
||||
static void put_seed(clients::body::Json& obj, const std::string& key, long seed) {
|
||||
if (seed >= 0) obj.set(key, clients::body::Json(static_cast<double>(seed)));
|
||||
}
|
||||
|
||||
// Provider fields from `extra` win over derived ones (the caller asked).
|
||||
static void merge_extra(clients::body::Json& obj, const clients::body::Json& extra) {
|
||||
if (!extra.is_object()) return;
|
||||
for (const auto& [k, v] : extra.as_object()) obj.set(k, v);
|
||||
}
|
||||
|
||||
static std::string str(const clients::body::Json& doc, const std::string& key) {
|
||||
const clients::body::Json* v = doc.is_object() ? doc.find(key) : nullptr;
|
||||
if (v && v->is_string()) return v->as_string();
|
||||
if (v && v->is_number()) return v->dump();
|
||||
return "";
|
||||
}
|
||||
|
||||
static const clients::body::Json* obj_at(const clients::body::Json& doc, const std::string& key) {
|
||||
const clients::body::Json* v = doc.is_object() ? doc.find(key) : nullptr;
|
||||
return v && v->is_object() ? v : nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
bool one_of(const char* value, std::initializer_list<const char*> options) {
|
||||
for (const char* o : options)
|
||||
if (std::strcmp(value, o) == 0) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
CallTable::CallTable(const char* vendor, const CallRow* rows, std::size_t count)
|
||||
: vendor_(vendor), rows_(rows), count_(count) {
|
||||
if (count > kMaxRows) throw std::logic_error(vendor_ + ": call table over " + std::to_string(kMaxRows) + " rows");
|
||||
for (std::size_t i = 0; i < count; ++i) {
|
||||
const CallRow& r = rows[i];
|
||||
if (!index_.emplace(r.key, i).second) throw std::logic_error(vendor_ + ": duplicate call key " + r.key);
|
||||
if (!one_of(r.http_method, {"GET", "POST", "PUT", "PATCH", "DELETE"}))
|
||||
throw std::logic_error(vendor_ + ": bad method on " + r.key);
|
||||
if (!one_of(r.body_kind, {"none", "json", "multipart", "either", "form"}))
|
||||
throw std::logic_error(vendor_ + ": bad body kind on " + r.key);
|
||||
if (!one_of(r.effect, {"read", "paid", "write", "delete"}))
|
||||
throw std::logic_error(vendor_ + ": bad effect on " + r.key);
|
||||
if (r.path[0] != '/') throw std::logic_error(vendor_ + ": path must start with / on " + r.key);
|
||||
}
|
||||
}
|
||||
|
||||
const CallRow& CallTable::at(std::size_t index) const {
|
||||
if (index >= count_) throw llm::ApiError::invalid(vendor_, "call index out of range");
|
||||
return rows_[index];
|
||||
}
|
||||
|
||||
const CallRow* CallTable::find(const std::string& key) const {
|
||||
auto it = index_.find(key);
|
||||
return it == index_.end() ? nullptr : &rows_[it->second];
|
||||
}
|
||||
|
||||
const CallRow& CallTable::get(const std::string& key) const {
|
||||
const CallRow* row = find(key);
|
||||
if (!row) throw llm::ApiError::invalid(vendor_, "unknown call '" + key + "'");
|
||||
return *row;
|
||||
}
|
||||
|
||||
Effect CallTable::effect_of(const CallRow& row) {
|
||||
if (std::strcmp(row.effect, "paid") == 0) return Effect::kPaid;
|
||||
if (std::strcmp(row.effect, "write") == 0) return Effect::kWrite;
|
||||
if (std::strcmp(row.effect, "delete") == 0) return Effect::kDelete;
|
||||
return Effect::kRead;
|
||||
}
|
||||
|
||||
const char* CallTable::effect_name(Effect effect) {
|
||||
switch (effect) {
|
||||
case Effect::kPaid: return "paid";
|
||||
case Effect::kWrite: return "write";
|
||||
case Effect::kDelete: return "delete";
|
||||
default: return "read";
|
||||
}
|
||||
}
|
||||
|
||||
clients::Method CallTable::method_of(const CallRow& row) {
|
||||
const char* m = row.http_method;
|
||||
if (std::strcmp(m, "POST") == 0) return clients::Method::kPost;
|
||||
if (std::strcmp(m, "PUT") == 0) return clients::Method::kPut;
|
||||
if (std::strcmp(m, "PATCH") == 0) return clients::Method::kPatch;
|
||||
if (std::strcmp(m, "DELETE") == 0) return clients::Method::kDelete;
|
||||
return clients::Method::kGet;
|
||||
}
|
||||
|
||||
std::vector<std::string> CallTable::split(const char* csv) {
|
||||
std::vector<std::string> out;
|
||||
std::string cur;
|
||||
for (const char* p = csv; *p; ++p) {
|
||||
if (*p == ',') {
|
||||
if (!cur.empty()) out.push_back(cur);
|
||||
cur.clear();
|
||||
} else {
|
||||
cur.push_back(*p);
|
||||
}
|
||||
}
|
||||
if (!cur.empty()) out.push_back(cur);
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<std::string> CallTable::placeholders(const char* path) {
|
||||
std::vector<std::string> out;
|
||||
std::string p(path);
|
||||
std::size_t pos = 0;
|
||||
while ((pos = p.find('{', pos)) != std::string::npos) {
|
||||
std::size_t end = p.find('}', pos);
|
||||
if (end == std::string::npos) break;
|
||||
out.push_back(p.substr(pos + 1, end - pos - 1));
|
||||
pos = end + 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool CallTable::body_allows(const CallRow& row, const std::string& field) const {
|
||||
for (const std::string& f : split(row.body_fields))
|
||||
if (f == "*" || f == field) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool CallTable::query_allows(const CallRow& row, const std::string& name) const {
|
||||
for (const std::string& f : split(row.query_params))
|
||||
if (f == name) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool CallTable::valid_segment(const std::string& id) {
|
||||
if (id.empty() || id.size() > MediaCaps::kMaxPathArgChars || id == "." || id == "..") return false;
|
||||
for (char c : id)
|
||||
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' ||
|
||||
c == '_' || c == '.' || c == ':' || c == '=' || c == '~'))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string CallTable::render_path(const CallRow& row,
|
||||
const std::map<std::string, std::string>& args) const {
|
||||
std::vector<std::string> names = placeholders(row.path);
|
||||
for (const auto& [name, value] : args) {
|
||||
bool known = false;
|
||||
for (const std::string& n : names) known = known || n == name;
|
||||
if (!known) throw llm::ApiError::invalid(vendor_, "call '" + std::string(row.key) + "' has no path argument '" + name + "'");
|
||||
(void)value;
|
||||
}
|
||||
std::string path = row.path;
|
||||
for (const std::string& name : names) {
|
||||
auto it = args.find(name);
|
||||
if (it == args.end()) throw llm::ApiError::invalid(vendor_, "call '" + std::string(row.key) + "' needs path argument '" + name + "'");
|
||||
const std::string& value = it->second;
|
||||
bool multi = name == "endpoint_id" || name == "model_path" || name == "path";
|
||||
if (multi) {
|
||||
std::size_t segments = 0, start = 0;
|
||||
while (true) {
|
||||
std::size_t slash = value.find('/', start);
|
||||
std::string seg = value.substr(start, slash == std::string::npos ? std::string::npos : slash - start);
|
||||
if (!valid_segment(seg)) throw llm::ApiError::invalid(vendor_, "bad " + name);
|
||||
if (++segments > MediaCaps::kMaxPathSegments) throw llm::ApiError::invalid(vendor_, name + " has too many segments");
|
||||
if (slash == std::string::npos) break;
|
||||
start = slash + 1;
|
||||
}
|
||||
} else if (!valid_segment(value)) {
|
||||
throw llm::ApiError::invalid(vendor_, "bad " + name);
|
||||
}
|
||||
std::string token = "{" + name + "}";
|
||||
std::size_t at = path.find(token);
|
||||
path.replace(at, token.size(), value);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
#pragma once
|
||||
|
||||
// One table row per REST call. Every provider has exactly one table (calls.cpp
|
||||
// in its directory); tools/check_spec.py checks each row against the
|
||||
// provider's published spec. Code never builds a path from a literal.
|
||||
|
||||
#include <cstddef>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/request.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// What a call does to the account. Anything but kRead needs explicit
|
||||
// confirmation at execution time (MediaService::execute).
|
||||
enum class Effect { kRead, kPaid, kWrite, kDelete };
|
||||
|
||||
struct CallRow {
|
||||
const char* key; // short stable name, unique within the provider
|
||||
const char* http_method; // GET POST PUT PATCH DELETE
|
||||
const char* path; // from the host, "{placeholder}" segments
|
||||
const char* op; // spec operation id when the spec has one
|
||||
const char* body_kind; // none | json | multipart | either | form
|
||||
const char* body_fields; // comma list of accepted body fields; "*" = open
|
||||
const char* query_params; // comma list of accepted query parameters
|
||||
const char* effect; // read | paid | write | delete
|
||||
};
|
||||
|
||||
class CallTable {
|
||||
public:
|
||||
static constexpr std::size_t kMaxRows = 256;
|
||||
|
||||
// Throws std::logic_error on a duplicate key, a bad method / kind / effect,
|
||||
// or more than kMaxRows rows (a table bug, caught by the unit tests).
|
||||
CallTable(const char* vendor, const CallRow* rows, std::size_t count);
|
||||
|
||||
const std::string& vendor() const { return vendor_; }
|
||||
std::size_t size() const { return count_; }
|
||||
const CallRow& at(std::size_t index) const;
|
||||
const CallRow* find(const std::string& key) const;
|
||||
// Throws llm::ApiError::invalid for an unknown key.
|
||||
const CallRow& get(const std::string& key) const;
|
||||
|
||||
static Effect effect_of(const CallRow& row);
|
||||
static const char* effect_name(Effect effect);
|
||||
static clients::Method method_of(const CallRow& row);
|
||||
static std::vector<std::string> split(const char* csv);
|
||||
static std::vector<std::string> placeholders(const char* path);
|
||||
|
||||
bool body_allows(const CallRow& row, const std::string& field) const;
|
||||
bool query_allows(const CallRow& row, const std::string& name) const;
|
||||
|
||||
// One URL path segment: 1..256 of [A-Za-z0-9._:=~-], not "." or "..".
|
||||
static bool valid_segment(const std::string& id);
|
||||
// Replaces every {placeholder}; each must be supplied, none may be extra.
|
||||
// Placeholders named endpoint_id, model_path or path accept 1..6 "/"-joined
|
||||
// segments, every other placeholder exactly one. Throws ApiError::invalid.
|
||||
std::string render_path(const CallRow& row,
|
||||
const std::map<std::string, std::string>& args) const;
|
||||
|
||||
private:
|
||||
std::string vendor_;
|
||||
const CallRow* rows_;
|
||||
std::size_t count_;
|
||||
std::map<std::string, std::size_t> index_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+47
@@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
// Static ceilings for package media. Never autotuned. Request and response
|
||||
// byte ceilings that come from src/llm (32 MiB / 64 MiB) are not repeated here;
|
||||
// ours are the ones the media workflows add: files, polls, downloads, pages.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct MediaCaps {
|
||||
// Request shaping.
|
||||
static constexpr std::size_t kMaxPromptChars = 32000;
|
||||
static constexpr std::size_t kMaxBodyFields = 128;
|
||||
static constexpr std::size_t kMaxFieldChars = 1u << 20; // one text field, 1 MiB
|
||||
static constexpr std::size_t kMaxFileBytes = 16u * 1024u * 1024u; // one uploaded file
|
||||
static constexpr std::size_t kMaxFiles = 16;
|
||||
static constexpr std::size_t kMaxQueryArgs = 16;
|
||||
static constexpr std::size_t kMaxQueryValueChars = 1024;
|
||||
static constexpr std::size_t kMaxPathArgChars = 256;
|
||||
static constexpr std::size_t kMaxPathSegments = 6; // multi-segment ids (fal endpoints)
|
||||
static constexpr std::size_t kMaxHeaders = 8;
|
||||
static constexpr std::size_t kMaxHeaderValueChars = 512;
|
||||
// Generation limits we enforce before spending money.
|
||||
static constexpr int kMaxImagesPerRequest = 8;
|
||||
static constexpr int kMaxVideoSeconds = 60;
|
||||
static constexpr int kMaxDimension = 8192;
|
||||
// Job polling: floors and ceilings (the machine never goes below / above).
|
||||
static constexpr int kMinPollIntervalMs = 500;
|
||||
static constexpr int kMaxPollIntervalMs = 30000;
|
||||
static constexpr int kMaxPollAttempts = 240;
|
||||
static constexpr int kMaxWaitSeconds = 1800;
|
||||
// Outputs.
|
||||
static constexpr std::size_t kMaxAssetsPerJob = 16;
|
||||
static constexpr std::size_t kMaxDownloadBytes = 128u * 1024u * 1024u;
|
||||
static constexpr std::size_t kMaxUrlChars = 4096;
|
||||
static constexpr std::size_t kMaxScanDepth = 8;
|
||||
static constexpr std::size_t kMaxScanNodes = 4096;
|
||||
// Listings.
|
||||
static constexpr std::size_t kMaxPages = 20;
|
||||
static constexpr std::size_t kMaxItemsReturned = 1000;
|
||||
// Webhooks.
|
||||
static constexpr long kWebhookToleranceSeconds = 300;
|
||||
static constexpr std::size_t kMaxWebhookBodyBytes = 1u * 1024u * 1024u;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
#pragma once
|
||||
|
||||
// Builds the llm::ApiClient every provider client wraps. One place, so the
|
||||
// credential lookup (environment, then .env.json.local, then .env.json via
|
||||
// llm::Config) and the transport choice are identical for all providers.
|
||||
|
||||
#include <initializer_list>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "llm/api_client.h"
|
||||
#include "llm/config.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct ClientFactory {
|
||||
// First non-empty value among `names` (llm::Config::get), else "".
|
||||
static std::string env(std::initializer_list<const char*> names) {
|
||||
for (const char* n : names) {
|
||||
std::string v = llm::Config::get(n);
|
||||
if (!v.empty()) return v;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
// Real HTTPS. An empty key leaves the client "not configured".
|
||||
static std::shared_ptr<llm::ApiClient> real(llm::ApiClientConfig config, const std::string& key) {
|
||||
return std::make_shared<llm::ApiClient>(std::make_unique<clients::transports::HttpTransport>(),
|
||||
std::move(config), key);
|
||||
}
|
||||
|
||||
// Any transport (tests record the requests the real client builds).
|
||||
static std::shared_ptr<llm::ApiClient> with(std::unique_ptr<clients::Transport> transport,
|
||||
llm::ApiClientConfig config, const std::string& key) {
|
||||
return std::make_shared<llm::ApiClient>(std::move(transport), std::move(config), key);
|
||||
}
|
||||
|
||||
static llm::ApiClientConfig bearer(const std::string& vendor, const std::string& base_url,
|
||||
const std::string& key_env) {
|
||||
llm::ApiClientConfig c;
|
||||
c.vendor = vendor;
|
||||
c.base_url = base_url;
|
||||
c.key_env = key_env;
|
||||
c.auth = llm::AuthStyle::kBearer;
|
||||
return c;
|
||||
}
|
||||
|
||||
static llm::ApiClientConfig header(const std::string& vendor, const std::string& base_url,
|
||||
const std::string& key_env, const std::string& header_name) {
|
||||
llm::ApiClientConfig c = bearer(vendor, base_url, key_env);
|
||||
c.auth = llm::AuthStyle::kHeader;
|
||||
c.auth_header = header_name;
|
||||
return c;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
#include "vendors/media/common/downloader.h"
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "llm/error.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
clients::ClientConfig plain_config() {
|
||||
clients::ClientConfig c;
|
||||
c.persist_cookies = false;
|
||||
return c;
|
||||
}
|
||||
|
||||
std::unique_ptr<clients::Transport> real_or(std::unique_ptr<clients::Transport> t) {
|
||||
if (t) return t;
|
||||
return std::make_unique<clients::transports::HttpTransport>();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Downloader::Downloader(std::unique_ptr<clients::Transport> transport)
|
||||
: client_(real_or(std::move(transport)), plain_config()) {}
|
||||
|
||||
void Downloader::validate_url(const std::string& url) {
|
||||
auto bad = [](const std::string& why) { throw llm::ApiError::invalid("media", why); };
|
||||
if (url.empty() || url.size() > MediaCaps::kMaxUrlChars) bad("download url length");
|
||||
if (url.rfind("https://", 0) != 0) bad("download url must be https");
|
||||
for (unsigned char c : url)
|
||||
if (c <= 0x20 || c == 0x7f) bad("download url has whitespace or control characters");
|
||||
std::size_t host_end = url.find_first_of("/?#", 8);
|
||||
std::string authority = url.substr(8, host_end == std::string::npos ? std::string::npos : host_end - 8);
|
||||
if (authority.empty()) bad("download url has no host");
|
||||
if (authority.find('@') != std::string::npos) bad("download url must not carry credentials");
|
||||
}
|
||||
|
||||
std::string Downloader::fetch_url(const std::string& url, std::string* content_type) {
|
||||
validate_url(url);
|
||||
try {
|
||||
clients::ClientRequest req(clients::Method::kGet, clients::Url::parse(url));
|
||||
clients::ClientResponse res = client_.send_blocking(std::move(req));
|
||||
if (!res.is_success())
|
||||
throw llm::ApiError("media", res.status(), "DOWNLOAD_FAILED", "download answered HTTP " + std::to_string(res.status()));
|
||||
if (res.body().size() > MediaCaps::kMaxDownloadBytes)
|
||||
throw llm::ApiError("media", 502, "BAD_RESPONSE", "download over the byte cap");
|
||||
if (content_type) *content_type = res.content_type();
|
||||
return res.body();
|
||||
} catch (const llm::ApiError&) {
|
||||
throw;
|
||||
} catch (const std::exception& e) {
|
||||
throw llm::ApiError("media", 502, "UNAVAILABLE", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
FetchedAsset Downloader::fetch(const MediaAsset& asset) {
|
||||
FetchedAsset out;
|
||||
out.asset = asset;
|
||||
if (!asset.data_base64.empty()) {
|
||||
if (!security::crypto::Base64::decode(asset.data_base64, out.bytes))
|
||||
throw llm::ApiError::invalid("media", "inline asset is not valid base64");
|
||||
if (out.bytes.size() > MediaCaps::kMaxDownloadBytes)
|
||||
throw llm::ApiError("media", 502, "BAD_RESPONSE", "inline asset over the byte cap");
|
||||
out.content_type = asset.mime;
|
||||
return out;
|
||||
}
|
||||
if (asset.url.empty()) throw llm::ApiError::invalid("media", "asset has neither url nor data");
|
||||
out.bytes = fetch_url(asset.url, &out.content_type);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
// Downloader: fetch an output (https URL or inline base64) as bytes under the
|
||||
// download cap. Sends no credential: result URLs are public or pre-signed.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "clients/client.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct FetchedAsset {
|
||||
MediaAsset asset;
|
||||
std::string bytes;
|
||||
std::string content_type;
|
||||
};
|
||||
|
||||
class Downloader {
|
||||
public:
|
||||
// Null transport = real HTTPS.
|
||||
explicit Downloader(std::unique_ptr<clients::Transport> transport = nullptr);
|
||||
|
||||
// https only, 1..4096 chars, no userinfo, no whitespace or control chars.
|
||||
// Throws ApiError::invalid("media", ...).
|
||||
static void validate_url(const std::string& url);
|
||||
|
||||
// Decodes inline data or downloads the URL. Throws ApiError on a bad status
|
||||
// (the body is never returned for a failure) or when the cap is exceeded.
|
||||
FetchedAsset fetch(const MediaAsset& asset);
|
||||
std::string fetch_url(const std::string& url, std::string* content_type = nullptr);
|
||||
|
||||
private:
|
||||
clients::Client client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+145
@@ -0,0 +1,145 @@
|
||||
#include "vendors/media/common/job.h"
|
||||
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
using clients::body::Json;
|
||||
|
||||
const char* media_kind_name(MediaKind kind) {
|
||||
switch (kind) {
|
||||
case MediaKind::kImage: return "image";
|
||||
case MediaKind::kVideo: return "video";
|
||||
case MediaKind::kAudio: return "audio";
|
||||
case MediaKind::kModel3d: return "model3d";
|
||||
default: return "other";
|
||||
}
|
||||
}
|
||||
|
||||
const char* job_state_name(JobState state) {
|
||||
switch (state) {
|
||||
case JobState::kQueued: return "queued";
|
||||
case JobState::kRunning: return "running";
|
||||
case JobState::kSucceeded: return "succeeded";
|
||||
case JobState::kFailed: return "failed";
|
||||
default: return "cancelled";
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
MediaKind kind_from(const std::string& s) {
|
||||
if (s == "image") return MediaKind::kImage;
|
||||
if (s == "video") return MediaKind::kVideo;
|
||||
if (s == "audio") return MediaKind::kAudio;
|
||||
if (s == "model3d") return MediaKind::kModel3d;
|
||||
return MediaKind::kOther;
|
||||
}
|
||||
|
||||
std::string str(const Json& doc, const char* key) {
|
||||
const Json* v = doc.is_object() ? doc.find(key) : nullptr;
|
||||
return v && v->is_string() ? v->as_string() : "";
|
||||
}
|
||||
|
||||
Json assets_json(const std::vector<MediaAsset>& assets) {
|
||||
Json a = Json::array();
|
||||
for (const MediaAsset& x : assets) a.push(x.to_json());
|
||||
return a;
|
||||
}
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("media", why); }
|
||||
|
||||
void check_text(const std::string& what, const std::string& v, std::size_t cap) {
|
||||
if (v.size() > cap) bad(what + " is over " + std::to_string(cap) + " characters");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Json MediaAsset::to_json() const {
|
||||
Json o = Json::object();
|
||||
o.set("kind", Json(media_kind_name(kind)));
|
||||
if (!url.empty()) o.set("url", Json(url));
|
||||
if (!data_base64.empty()) o.set("data_base64_bytes", Json(static_cast<double>(data_base64.size())));
|
||||
if (!mime.empty()) o.set("mime", Json(mime));
|
||||
if (!label.empty()) o.set("label", Json(label));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json JobTicket::to_json() const {
|
||||
Json o = Json::object();
|
||||
o.set("provider", Json(provider));
|
||||
o.set("id", Json(id));
|
||||
o.set("model", Json(model));
|
||||
o.set("kind", Json(media_kind_name(kind)));
|
||||
o.set("immediate", Json(immediate));
|
||||
o.set("assets", assets_json(assets));
|
||||
if (!meta.is_null()) o.set("meta", meta);
|
||||
return o;
|
||||
}
|
||||
|
||||
JobTicket JobTicket::from_json(const Json& doc) {
|
||||
if (!doc.is_object()) bad("ticket must be a JSON object");
|
||||
JobTicket t;
|
||||
t.provider = str(doc, "provider");
|
||||
t.id = str(doc, "id");
|
||||
t.model = str(doc, "model");
|
||||
t.kind = kind_from(str(doc, "kind"));
|
||||
if (t.provider.empty() || t.id.empty()) bad("ticket needs provider and id");
|
||||
if (const Json* m = doc.find("meta"); m && m->is_object()) t.meta = *m;
|
||||
if (const Json* im = doc.find("immediate"); im && im->is_bool()) t.immediate = im->as_bool();
|
||||
// Only URL assets travel back: inline (base64) outputs are not echoed by
|
||||
// to_json(), so an immediate ticket restored from JSON carries its URLs only.
|
||||
if (const Json* a = doc.find("assets"); a && a->is_array())
|
||||
for (const Json& x : a->as_array()) {
|
||||
std::string url = str(x, "url");
|
||||
if (url.empty() || t.assets.size() >= MediaCaps::kMaxAssetsPerJob) continue;
|
||||
MediaAsset asset;
|
||||
asset.url = url;
|
||||
asset.kind = kind_from(str(x, "kind"));
|
||||
asset.mime = str(x, "mime");
|
||||
asset.label = str(x, "label");
|
||||
t.assets.push_back(asset);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
Json JobStatus::to_json() const {
|
||||
Json o = Json::object();
|
||||
o.set("state", Json(job_state_name(state)));
|
||||
o.set("raw_state", Json(raw_state));
|
||||
if (progress >= 0) o.set("progress", Json(progress));
|
||||
if (!error.empty()) o.set("error", Json(error));
|
||||
o.set("assets", assets_json(assets));
|
||||
return o;
|
||||
}
|
||||
|
||||
void ImageRequest::validate() const {
|
||||
if (prompt.empty()) bad("prompt is required");
|
||||
check_text("prompt", prompt, MediaCaps::kMaxPromptChars);
|
||||
check_text("negative_prompt", negative_prompt, MediaCaps::kMaxPromptChars);
|
||||
check_text("model", model, 256);
|
||||
check_text("source_url", source_url, MediaCaps::kMaxUrlChars);
|
||||
if (count < 1 || count > MediaCaps::kMaxImagesPerRequest)
|
||||
bad("count must be 1.." + std::to_string(MediaCaps::kMaxImagesPerRequest));
|
||||
if (width < 0 || width > MediaCaps::kMaxDimension || height < 0 || height > MediaCaps::kMaxDimension)
|
||||
bad("width and height must be 0.." + std::to_string(MediaCaps::kMaxDimension));
|
||||
if ((width == 0) != (height == 0)) bad("width and height go together");
|
||||
if (seed < -1 || seed > 4294967295L) bad("seed must be -1 or 0..4294967295");
|
||||
if (!extra.is_null() && !extra.is_object()) bad("extra must be a JSON object");
|
||||
}
|
||||
|
||||
void VideoRequest::validate() const {
|
||||
if (prompt.empty() && image_url.empty()) bad("prompt or image_url is required");
|
||||
check_text("prompt", prompt, MediaCaps::kMaxPromptChars);
|
||||
check_text("negative_prompt", negative_prompt, MediaCaps::kMaxPromptChars);
|
||||
check_text("model", model, 256);
|
||||
check_text("image_url", image_url, MediaCaps::kMaxUrlChars);
|
||||
check_text("end_image_url", end_image_url, MediaCaps::kMaxUrlChars);
|
||||
if (duration_seconds < 0 || duration_seconds > MediaCaps::kMaxVideoSeconds)
|
||||
bad("duration_seconds must be 0.." + std::to_string(MediaCaps::kMaxVideoSeconds));
|
||||
if (seed < -1 || seed > 4294967295L) bad("seed must be -1 or 0..4294967295");
|
||||
if (!extra.is_null() && !extra.is_object()) bad("extra must be a JSON object");
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+124
@@ -0,0 +1,124 @@
|
||||
#pragma once
|
||||
|
||||
// The neutral generation seam of package media: submit a request, poll the
|
||||
// job, fetch the outputs. Every provider adapter implements ImageGenerator
|
||||
// and / or VideoGenerator; callers (the demo, tests, other packages) code
|
||||
// against these types only. Nothing here knows a provider's wire format.
|
||||
//
|
||||
// Money rule: submit() and cancel() take an explicit confirm flag with no
|
||||
// default. Without it they throw before any network traffic.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
enum class MediaKind { kImage, kVideo, kAudio, kModel3d, kOther };
|
||||
enum class JobState { kQueued, kRunning, kSucceeded, kFailed, kCancelled };
|
||||
|
||||
const char* media_kind_name(MediaKind kind);
|
||||
const char* job_state_name(JobState state);
|
||||
|
||||
// One output. Exactly one of url / data_base64 is set.
|
||||
struct MediaAsset {
|
||||
MediaKind kind = MediaKind::kOther;
|
||||
std::string url;
|
||||
std::string data_base64;
|
||||
std::string mime;
|
||||
std::string label; // provider note: seed, finish reason, model
|
||||
|
||||
clients::body::Json to_json() const;
|
||||
};
|
||||
|
||||
// Handle of a submitted job. `immediate` jobs (synchronous providers) carry
|
||||
// their assets already; poll() answers from the ticket without a network call.
|
||||
struct JobTicket {
|
||||
std::string provider;
|
||||
std::string id;
|
||||
std::string model;
|
||||
MediaKind kind = MediaKind::kImage;
|
||||
bool immediate = false;
|
||||
std::vector<MediaAsset> assets;
|
||||
clients::body::Json meta; // provider extras the poll needs (endpoint, urls, cost)
|
||||
|
||||
clients::body::Json to_json() const;
|
||||
// Rebuilds a ticket from to_json() output (the demo round-trips tickets).
|
||||
static JobTicket from_json(const clients::body::Json& doc);
|
||||
};
|
||||
|
||||
struct JobStatus {
|
||||
JobState state = JobState::kQueued;
|
||||
std::string raw_state; // the provider's own word
|
||||
double progress = -1; // 0..1, -1 = unknown
|
||||
std::string error;
|
||||
std::vector<MediaAsset> assets;
|
||||
clients::body::Json meta;
|
||||
|
||||
bool terminal() const { return state == JobState::kSucceeded || state == JobState::kFailed || state == JobState::kCancelled; }
|
||||
clients::body::Json to_json() const;
|
||||
};
|
||||
|
||||
struct ImageRequest {
|
||||
std::string prompt;
|
||||
std::string negative_prompt;
|
||||
std::string model; // provider model / endpoint id; empty = adapter default
|
||||
int width = 0; // 0 = provider default
|
||||
int height = 0;
|
||||
std::string aspect_ratio; // "16:9"; empty = default
|
||||
int count = 1;
|
||||
long seed = -1; // -1 = random
|
||||
std::string source_url; // reference / source image for edit styles
|
||||
clients::body::Json extra; // provider fields merged last, checked against the call table
|
||||
|
||||
// Throws llm::ApiError::invalid(vendor "media") on a bad value or cap breach.
|
||||
void validate() const;
|
||||
};
|
||||
|
||||
struct VideoRequest {
|
||||
std::string prompt;
|
||||
std::string negative_prompt;
|
||||
std::string model;
|
||||
int duration_seconds = 0; // 0 = provider default
|
||||
std::string aspect_ratio;
|
||||
std::string resolution; // "720p", "1080p"; empty = default
|
||||
long seed = -1;
|
||||
std::string image_url; // start frame for image-to-video
|
||||
std::string end_image_url; // end frame
|
||||
clients::body::Json extra;
|
||||
|
||||
void validate() const;
|
||||
};
|
||||
|
||||
class MediaGenerator {
|
||||
public:
|
||||
virtual ~MediaGenerator() = default;
|
||||
virtual std::string provider() const = 0;
|
||||
virtual bool configured() const = 0;
|
||||
// Model / endpoint ids the adapter knows by name (not a live catalogue).
|
||||
virtual std::vector<std::string> models() const = 0;
|
||||
// One status check. A failed job is a JobStatus with state kFailed, a
|
||||
// transport or auth failure is an ApiError.
|
||||
virtual JobStatus poll(const JobTicket& ticket) = 0;
|
||||
// Cancels where the provider supports it; returns false (without any call)
|
||||
// where it does not. Explicit confirm, no default.
|
||||
virtual bool cancel(const JobTicket& ticket, bool confirm) = 0;
|
||||
// Suggested first poll interval; JobRunner clamps it to the caps.
|
||||
virtual int suggested_poll_ms() const { return 2000; }
|
||||
};
|
||||
|
||||
class ImageGenerator : public MediaGenerator {
|
||||
public:
|
||||
// Exact request the submit would send, as JSON; no network, no credential.
|
||||
virtual clients::body::Json plan(const ImageRequest& request) const = 0;
|
||||
virtual JobTicket submit(const ImageRequest& request, bool confirm) = 0;
|
||||
};
|
||||
|
||||
class VideoGenerator : public MediaGenerator {
|
||||
public:
|
||||
virtual clients::body::Json plan(const VideoRequest& request) const = 0;
|
||||
virtual JobTicket submit(const VideoRequest& request, bool confirm) = 0;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
#include "vendors/media/common/job_runner.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
void WaitPolicy::validate() const {
|
||||
auto bad = [](const std::string& why) { throw llm::ApiError::invalid("media", why); };
|
||||
if (first_interval_ms < 0 || first_interval_ms > MediaCaps::kMaxPollIntervalMs) bad("first_interval_ms out of range");
|
||||
if (backoff < 1.0 || backoff > 3.0) bad("backoff must be 1.0..3.0");
|
||||
if (max_interval_ms < MediaCaps::kMinPollIntervalMs || max_interval_ms > MediaCaps::kMaxPollIntervalMs)
|
||||
bad("max_interval_ms must be " + std::to_string(MediaCaps::kMinPollIntervalMs) + ".." +
|
||||
std::to_string(MediaCaps::kMaxPollIntervalMs));
|
||||
if (max_attempts < 1 || max_attempts > MediaCaps::kMaxPollAttempts)
|
||||
bad("max_attempts must be 1.." + std::to_string(MediaCaps::kMaxPollAttempts));
|
||||
if (max_wait_seconds < 1 || max_wait_seconds > MediaCaps::kMaxWaitSeconds)
|
||||
bad("max_wait_seconds must be 1.." + std::to_string(MediaCaps::kMaxWaitSeconds));
|
||||
}
|
||||
|
||||
JobRunner::JobRunner(WaitPolicy policy, Sleeper sleeper, Clock clock)
|
||||
: policy_(policy), sleeper_(std::move(sleeper)), clock_(std::move(clock)) {
|
||||
policy_.validate();
|
||||
if (!sleeper_)
|
||||
sleeper_ = [](int ms) { std::this_thread::sleep_for(std::chrono::milliseconds(ms)); };
|
||||
if (!clock_)
|
||||
clock_ = [] {
|
||||
return static_cast<long>(std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count());
|
||||
};
|
||||
}
|
||||
|
||||
JobStatus JobRunner::wait(MediaGenerator& generator, const JobTicket& ticket) const {
|
||||
if (ticket.immediate) {
|
||||
JobStatus s;
|
||||
s.state = JobState::kSucceeded;
|
||||
s.raw_state = "immediate";
|
||||
s.progress = 1;
|
||||
s.assets = ticket.assets;
|
||||
return s;
|
||||
}
|
||||
const long start = clock_();
|
||||
const long deadline_ms = static_cast<long>(policy_.max_wait_seconds) * 1000L;
|
||||
int interval = policy_.first_interval_ms > 0 ? policy_.first_interval_ms : generator.suggested_poll_ms();
|
||||
interval = std::clamp(interval, MediaCaps::kMinPollIntervalMs, policy_.max_interval_ms);
|
||||
for (int attempt = 1; attempt <= policy_.max_attempts; ++attempt) {
|
||||
JobStatus s = generator.poll(ticket);
|
||||
if (s.terminal()) return s;
|
||||
if (clock_() - start + interval > deadline_ms) break;
|
||||
sleeper_(interval);
|
||||
interval = std::clamp(static_cast<int>(interval * policy_.backoff), MediaCaps::kMinPollIntervalMs,
|
||||
policy_.max_interval_ms);
|
||||
}
|
||||
throw llm::ApiError("media", 504, "WAIT_EXPIRED",
|
||||
generator.provider() + " job " + ticket.id + " is not finished after the poll caps (" +
|
||||
std::to_string(policy_.max_attempts) + " polls, " + std::to_string(policy_.max_wait_seconds) +
|
||||
" s); it was NOT cancelled, check it with poll");
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
|
||||
// JobRunner: poll a job to a terminal state under hard caps. The caps are a
|
||||
// floor on the interval (never hammer a provider) and ceilings on attempts
|
||||
// and total wait (never hang). When a cap is hit the runner throws; it never
|
||||
// cancels the job on its own (a human decides, the money may already be spent).
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "vendors/media/common/job.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct WaitPolicy {
|
||||
int first_interval_ms = 0; // 0 = the generator's suggestion
|
||||
double backoff = 1.5; // interval multiplier per attempt, 1.0 .. 3.0
|
||||
int max_interval_ms = 15000; // <= MediaCaps::kMaxPollIntervalMs
|
||||
int max_attempts = 120; // <= MediaCaps::kMaxPollAttempts
|
||||
int max_wait_seconds = 900; // <= MediaCaps::kMaxWaitSeconds
|
||||
|
||||
// Throws ApiError::invalid when a value is outside the caps.
|
||||
void validate() const;
|
||||
};
|
||||
|
||||
class JobRunner {
|
||||
public:
|
||||
using Sleeper = std::function<void(int milliseconds)>;
|
||||
using Clock = std::function<long()>; // monotonic milliseconds
|
||||
|
||||
// Null sleeper / clock use the real ones. Tests inject fakes.
|
||||
explicit JobRunner(WaitPolicy policy = {}, Sleeper sleeper = nullptr, Clock clock = nullptr);
|
||||
|
||||
const WaitPolicy& policy() const { return policy_; }
|
||||
|
||||
// Immediate tickets answer without a poll. Otherwise polls until terminal.
|
||||
// Throws ApiError (504, WAIT_EXPIRED) when attempts or wait are exhausted.
|
||||
JobStatus wait(MediaGenerator& generator, const JobTicket& ticket) const;
|
||||
|
||||
private:
|
||||
WaitPolicy policy_;
|
||||
Sleeper sleeper_;
|
||||
Clock clock_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
#include "vendors/media/common/media_service.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "clients/body/form.h"
|
||||
#include "clients/url.h"
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
bool valid_header_name(const std::string& n) {
|
||||
if (n.empty() || n.size() > 64) return false;
|
||||
for (char c : n)
|
||||
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-')) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string field_text(const Json& v) {
|
||||
if (v.is_string()) return v.as_string();
|
||||
if (v.is_bool()) return v.as_bool() ? "true" : "false";
|
||||
return v.dump(); // numbers, arrays, objects travel as JSON text
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Json PreparedCall::summary() const {
|
||||
Json o = Json::object();
|
||||
o.set("vendor", Json(vendor));
|
||||
o.set("key", Json(key));
|
||||
o.set("method", Json(method));
|
||||
o.set("target", Json(target));
|
||||
o.set("effect", Json(CallTable::effect_name(effect)));
|
||||
if (!content_type.empty()) o.set("content_type", Json(content_type.substr(0, content_type.find(';'))));
|
||||
if (!preview.is_null()) o.set("body", preview);
|
||||
return o;
|
||||
}
|
||||
|
||||
MediaService::MediaService(std::shared_ptr<llm::ApiClient> client, const CallTable& table)
|
||||
: llm::Service(std::move(client)), table_(table) {}
|
||||
|
||||
PreparedCall MediaService::prepare(const std::string& key, const CallArgs& args) const {
|
||||
const std::string v = table_.vendor();
|
||||
const CallRow& row = table_.get(key);
|
||||
PreparedCall out;
|
||||
out.vendor = v;
|
||||
out.key = key;
|
||||
out.method = row.http_method;
|
||||
out.effect = CallTable::effect_of(row);
|
||||
|
||||
std::string target = table_.render_path(row, args.path);
|
||||
|
||||
if (args.query.size() > MediaCaps::kMaxQueryArgs) throw llm::ApiError::invalid(v, "too many query parameters");
|
||||
std::string qs;
|
||||
for (const auto& [name, value] : args.query) {
|
||||
if (!table_.query_allows(row, name))
|
||||
throw llm::ApiError::invalid(v, "call '" + key + "' does not accept query parameter '" + name + "'");
|
||||
if (value.size() > MediaCaps::kMaxQueryValueChars) throw llm::ApiError::invalid(v, "query value too long: " + name);
|
||||
qs += qs.empty() ? '?' : '&';
|
||||
qs += clients::Url::encode_component(name) + "=" + clients::Url::encode_component(value);
|
||||
}
|
||||
out.target = target + qs;
|
||||
|
||||
if (args.headers.size() > MediaCaps::kMaxHeaders) throw llm::ApiError::invalid(v, "too many headers");
|
||||
for (const auto& [k, val] : args.headers) {
|
||||
if (!valid_header_name(k) || val.size() > MediaCaps::kMaxHeaderValueChars ||
|
||||
val.find_first_of("\r\n") != std::string::npos)
|
||||
throw llm::ApiError::invalid(v, "bad header '" + k + "'");
|
||||
out.headers[k] = val;
|
||||
}
|
||||
|
||||
const std::string kind = row.body_kind;
|
||||
const bool has_body = !args.body.is_null() && !(args.body.is_object() && args.body.as_object().empty());
|
||||
if (kind == "none") {
|
||||
if (has_body || !args.files.empty()) throw llm::ApiError::invalid(v, "call '" + key + "' takes no body");
|
||||
return out;
|
||||
}
|
||||
if (kind == "form") throw llm::ApiError::invalid(v, "form bodies are not supported by this package");
|
||||
|
||||
if (!args.body.is_null() && !args.body.is_object()) throw llm::ApiError::invalid(v, "body must be a JSON object");
|
||||
Json fields = args.body.is_null() ? Json::object() : args.body;
|
||||
if (fields.as_object().size() > MediaCaps::kMaxBodyFields) throw llm::ApiError::invalid(v, "too many body fields");
|
||||
for (const auto& [name, value] : fields.as_object()) {
|
||||
if (!table_.body_allows(row, name))
|
||||
throw llm::ApiError::invalid(v, "call '" + key + "' does not accept body field '" + name + "'");
|
||||
if (value.is_string() && value.as_string().size() > MediaCaps::kMaxFieldChars)
|
||||
throw llm::ApiError::invalid(v, "field '" + name + "' is over the size cap");
|
||||
}
|
||||
|
||||
const bool multipart = kind == "multipart" || (kind == "either" && !args.files.empty());
|
||||
if (!args.files.empty() && !multipart) throw llm::ApiError::invalid(v, "call '" + key + "' does not take files");
|
||||
if (args.files.size() > MediaCaps::kMaxFiles) throw llm::ApiError::invalid(v, "too many files");
|
||||
|
||||
if (!multipart) {
|
||||
out.body = fields.dump();
|
||||
out.content_type = "application/json";
|
||||
out.preview = fields;
|
||||
return out;
|
||||
}
|
||||
|
||||
clients::body::MultipartBody form;
|
||||
Json preview = Json::object();
|
||||
for (const auto& [name, value] : fields.as_object()) {
|
||||
form.add_text(name, field_text(value));
|
||||
preview.set(name, value);
|
||||
}
|
||||
Json files = Json::array();
|
||||
for (const FilePart& f : args.files) {
|
||||
if (!table_.body_allows(row, f.field))
|
||||
throw llm::ApiError::invalid(v, "call '" + key + "' does not accept file field '" + f.field + "'");
|
||||
if (f.data.empty()) throw llm::ApiError::invalid(v, "file '" + f.field + "' is empty");
|
||||
if (f.data.size() > MediaCaps::kMaxFileBytes) throw llm::ApiError::invalid(v, "file '" + f.field + "' is over the size cap");
|
||||
if (f.filename.find_first_of("\"\r\n") != std::string::npos || f.content_type.find_first_of("\r\n") != std::string::npos)
|
||||
throw llm::ApiError::invalid(v, "bad file name or content type");
|
||||
form.add_file(f.field, f.filename.empty() ? f.field : f.filename,
|
||||
f.content_type.empty() ? "application/octet-stream" : f.content_type, f.data);
|
||||
Json info = Json::object();
|
||||
info.set("field", Json(f.field));
|
||||
info.set("filename", Json(f.filename));
|
||||
info.set("bytes", Json(static_cast<double>(f.data.size())));
|
||||
files.push(info);
|
||||
}
|
||||
if (!args.files.empty()) preview.set("_files", files);
|
||||
out.body = form.encode();
|
||||
out.content_type = form.content_type();
|
||||
out.preview = preview;
|
||||
return out;
|
||||
}
|
||||
|
||||
llm::RawResponse MediaService::execute(const PreparedCall& call, bool confirmed) {
|
||||
if (call.effect != Effect::kRead && !confirmed)
|
||||
throw llm::ApiError::invalid(table_.vendor(), "call '" + call.key + "' has effect " +
|
||||
CallTable::effect_name(call.effect) +
|
||||
" and needs explicit confirmation");
|
||||
const CallRow& row = table_.get(call.key);
|
||||
const bool has_body = !call.body.empty();
|
||||
return client_->request_raw(CallTable::method_of(row), call.target, has_body ? &call.body : nullptr,
|
||||
call.content_type, call.headers);
|
||||
}
|
||||
|
||||
llm::RawResponse MediaService::call_raw(const std::string& key, const CallArgs& args, bool confirmed) {
|
||||
return execute(prepare(key, args), confirmed);
|
||||
}
|
||||
|
||||
Json MediaService::parse_json(const std::string& vendor, const llm::RawResponse& raw) {
|
||||
if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object();
|
||||
try {
|
||||
return Json::parse(raw.body);
|
||||
} catch (const std::exception& e) {
|
||||
throw llm::ApiError(vendor, 502, "BAD_RESPONSE", std::string("not JSON: ") + e.what());
|
||||
}
|
||||
}
|
||||
|
||||
Json MediaService::call_json(const std::string& key, const CallArgs& args, bool confirmed) {
|
||||
return parse_json(table_.vendor(), call_raw(key, args, confirmed));
|
||||
}
|
||||
|
||||
const Json* MediaService::member(const Json& doc, const std::string& key) {
|
||||
return doc.is_object() ? doc.find(key) : nullptr;
|
||||
}
|
||||
|
||||
std::string MediaService::text(const Json& doc, const std::string& key, const std::string& fallback) {
|
||||
const Json* v = member(doc, key);
|
||||
if (v && v->is_string()) return v->as_string();
|
||||
if (v && v->is_number()) return v->dump();
|
||||
return fallback;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
#pragma once
|
||||
|
||||
// MediaService: the validated path from "I want call X with these arguments"
|
||||
// to bytes on the wire. prepare() is pure (no network): it checks the key,
|
||||
// path arguments, query parameters, body fields and files against the call
|
||||
// table row and builds the exact request. execute() sends it and refuses any
|
||||
// call whose effect is not "read" unless the caller confirms explicitly.
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
#include "llm/api_client.h"
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
using Json = clients::body::Json;
|
||||
|
||||
struct FilePart {
|
||||
std::string field;
|
||||
std::string filename;
|
||||
std::string content_type;
|
||||
std::string data;
|
||||
};
|
||||
|
||||
struct CallArgs {
|
||||
std::map<std::string, std::string> path;
|
||||
std::map<std::string, std::string> query;
|
||||
Json body; // null = no body; otherwise a JSON object
|
||||
std::vector<FilePart> files; // multipart rows only
|
||||
llm::Headers headers; // extra request headers (Accept, Prefer, ...)
|
||||
};
|
||||
|
||||
struct PreparedCall {
|
||||
std::string vendor;
|
||||
std::string key;
|
||||
std::string method; // GET POST ...
|
||||
std::string target; // path plus query string
|
||||
std::string body; // exact bytes ("" = none)
|
||||
std::string content_type; // "" when no body
|
||||
llm::Headers headers;
|
||||
Effect effect = Effect::kRead;
|
||||
Json preview; // body as JSON, or the multipart part list
|
||||
|
||||
// What a dry run shows: method, target, effect, content type, body preview.
|
||||
Json summary() const;
|
||||
};
|
||||
|
||||
class MediaService : public llm::Service {
|
||||
public:
|
||||
MediaService(std::shared_ptr<llm::ApiClient> client, const CallTable& table);
|
||||
|
||||
const CallTable& table() const { return table_; }
|
||||
const std::string& vendor() const { return table_.vendor(); }
|
||||
|
||||
PreparedCall prepare(const std::string& key, const CallArgs& args = {}) const;
|
||||
|
||||
// Non-read effects throw ApiError::invalid unless confirmed is true. The
|
||||
// missing-credential error surfaces here, before any network traffic.
|
||||
llm::RawResponse execute(const PreparedCall& call, bool confirmed);
|
||||
|
||||
// prepare + execute + parse. Empty body -> {}. Non-JSON body -> ApiError.
|
||||
Json call_json(const std::string& key, const CallArgs& args = {}, bool confirmed = false);
|
||||
llm::RawResponse call_raw(const std::string& key, const CallArgs& args = {}, bool confirmed = false);
|
||||
|
||||
// Helpers for typed wrappers.
|
||||
static Json parse_json(const std::string& vendor, const llm::RawResponse& raw);
|
||||
static const Json* member(const Json& doc, const std::string& key);
|
||||
static std::string text(const Json& doc, const std::string& key, const std::string& fallback = "");
|
||||
|
||||
private:
|
||||
const CallTable& table_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
|
||||
#include "vendors/media/common/caps.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
using clients::body::Json;
|
||||
|
||||
namespace {
|
||||
|
||||
std::string lower_ext(const std::string& name) {
|
||||
std::string n = name.substr(0, name.find_first_of("?#"));
|
||||
std::size_t dot = n.rfind('.');
|
||||
if (dot == std::string::npos || n.size() - dot > 8) return "";
|
||||
std::string e = n.substr(dot + 1);
|
||||
std::transform(e.begin(), e.end(), e.begin(), [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
|
||||
return e;
|
||||
}
|
||||
|
||||
struct Walk {
|
||||
std::vector<MediaAsset> out;
|
||||
std::set<std::string> seen;
|
||||
std::size_t nodes = 0;
|
||||
MediaKind fallback;
|
||||
|
||||
void add(const std::string& s, const std::string& key) {
|
||||
if (out.size() >= MediaCaps::kMaxAssetsPerJob) return;
|
||||
bool http = s.rfind("https://", 0) == 0 || s.rfind("http://", 0) == 0;
|
||||
bool data = s.rfind("data:", 0) == 0;
|
||||
if (!http && !data) return;
|
||||
if (s.size() > MediaCaps::kMaxUrlChars && !data) return;
|
||||
if (!seen.insert(s).second) return;
|
||||
MediaAsset a;
|
||||
if (data) {
|
||||
std::size_t comma = s.find(',');
|
||||
std::size_t semi = s.find(';');
|
||||
if (comma == std::string::npos) return;
|
||||
a.mime = s.substr(5, std::min(semi, comma) - 5);
|
||||
a.data_base64 = s.substr(comma + 1);
|
||||
a.kind = fallback;
|
||||
if (a.mime.rfind("video/", 0) == 0) a.kind = MediaKind::kVideo;
|
||||
else if (a.mime.rfind("image/", 0) == 0) a.kind = MediaKind::kImage;
|
||||
else if (a.mime.rfind("audio/", 0) == 0) a.kind = MediaKind::kAudio;
|
||||
} else {
|
||||
a.url = s;
|
||||
a.kind = OutputScanner::kind_from_name(s, fallback);
|
||||
if (a.kind == fallback) {
|
||||
if (key.find("video") != std::string::npos) a.kind = MediaKind::kVideo;
|
||||
else if (key.find("image") != std::string::npos) a.kind = MediaKind::kImage;
|
||||
else if (key.find("audio") != std::string::npos) a.kind = MediaKind::kAudio;
|
||||
}
|
||||
a.mime = OutputScanner::mime_from_name(s);
|
||||
}
|
||||
out.push_back(a);
|
||||
}
|
||||
|
||||
void go(const Json& v, const std::string& key, std::size_t depth) {
|
||||
if (depth > MediaCaps::kMaxScanDepth || ++nodes > MediaCaps::kMaxScanNodes) return;
|
||||
if (v.is_string()) {
|
||||
add(v.as_string(), key);
|
||||
} else if (v.is_array()) {
|
||||
for (const Json& x : v.as_array()) go(x, key, depth + 1);
|
||||
} else if (v.is_object()) {
|
||||
for (const auto& [k, x] : v.as_object()) {
|
||||
if (OutputScanner::skipped_key(k)) continue;
|
||||
go(x, k, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
bool OutputScanner::skipped_key(const std::string& key) {
|
||||
static const std::set<std::string> skip = {
|
||||
"input", "request", "urls", "logs", "metrics", "webhook", "webhook_url", "callback_url",
|
||||
"status_url", "cancel_url", "response_url", "polling_url", "get", "cancel", "stream", "web",
|
||||
"params", "parameters", "meta", "links", "self", "thumbnail_url", "model_url"};
|
||||
return skip.count(key) > 0;
|
||||
}
|
||||
|
||||
MediaKind OutputScanner::kind_from_name(const std::string& name, MediaKind fallback) {
|
||||
std::string e = lower_ext(name);
|
||||
if (e == "mp4" || e == "webm" || e == "mov" || e == "mkv" || e == "m4v") return MediaKind::kVideo;
|
||||
if (e == "png" || e == "jpg" || e == "jpeg" || e == "webp" || e == "gif" || e == "svg" || e == "avif") return MediaKind::kImage;
|
||||
if (e == "mp3" || e == "wav" || e == "flac" || e == "ogg" || e == "m4a" || e == "aac") return MediaKind::kAudio;
|
||||
if (e == "glb" || e == "gltf" || e == "obj" || e == "ply" || e == "usdz" || e == "fbx" || e == "stl") return MediaKind::kModel3d;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
std::string OutputScanner::mime_from_name(const std::string& name) {
|
||||
std::string e = lower_ext(name);
|
||||
if (e == "mp4" || e == "m4v") return "video/mp4";
|
||||
if (e == "webm") return "video/webm";
|
||||
if (e == "mov") return "video/quicktime";
|
||||
if (e == "png") return "image/png";
|
||||
if (e == "jpg" || e == "jpeg") return "image/jpeg";
|
||||
if (e == "webp") return "image/webp";
|
||||
if (e == "gif") return "image/gif";
|
||||
if (e == "svg") return "image/svg+xml";
|
||||
if (e == "mp3") return "audio/mpeg";
|
||||
if (e == "wav") return "audio/wav";
|
||||
if (e == "flac") return "audio/flac";
|
||||
if (e == "ogg") return "audio/ogg";
|
||||
if (e == "glb") return "model/gltf-binary";
|
||||
return "";
|
||||
}
|
||||
|
||||
std::vector<MediaAsset> OutputScanner::scan(const Json& doc, MediaKind fallback) {
|
||||
Walk w;
|
||||
w.fallback = fallback;
|
||||
w.go(doc, "", 0);
|
||||
return std::move(w.out);
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
// OutputScanner: finds result URLs / inline data in a provider's result JSON.
|
||||
// Providers shape results differently (fal: images[].url, Replicate: output as
|
||||
// string or array, Runway: output[]), so the adapters share one bounded walk
|
||||
// instead of seven hand-rolled ones. Keys that carry request echoes or
|
||||
// control links (input, request, urls, webhook, status_url, ...) are skipped.
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/media/common/job.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class OutputScanner {
|
||||
public:
|
||||
// Walks `doc` (depth and node caps in MediaCaps) and returns de-duplicated
|
||||
// assets in document order, at most MediaCaps::kMaxAssetsPerJob.
|
||||
static std::vector<MediaAsset> scan(const clients::body::Json& doc, MediaKind fallback);
|
||||
|
||||
// Kind from a URL / file name extension, else `fallback`.
|
||||
static MediaKind kind_from_name(const std::string& name, MediaKind fallback);
|
||||
// MIME for a kind + extension, "" when unknown.
|
||||
static std::string mime_from_name(const std::string& name);
|
||||
// True for keys whose subtree is request echo or control metadata.
|
||||
static bool skipped_key(const std::string& key);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
#include "vendors/media/common/registry.h"
|
||||
|
||||
#include <ctime>
|
||||
|
||||
#include "llm/config.h"
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/bfl/bfl.h"
|
||||
#include "vendors/media/fal/fal.h"
|
||||
#include "vendors/media/higgsfield/higgsfield.h"
|
||||
#include "vendors/media/ideogram/ideogram.h"
|
||||
#include "vendors/media/kling/kling.h"
|
||||
#include "vendors/media/leonardo/leonardo.h"
|
||||
#include "vendors/media/luma/luma.h"
|
||||
#include "vendors/media/minimax/minimax.h"
|
||||
#include "vendors/media/recraft/recraft.h"
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
#include "vendors/media/runway/runway.h"
|
||||
#include "vendors/media/stability/stability.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
const std::vector<ProviderInfo>& MediaRegistry::providers() {
|
||||
static const std::vector<ProviderInfo> list = {
|
||||
{"stability", "Stability AI", {"STABILITY_API_KEY"}, true, false, Evidence::kOpenApi,
|
||||
"https://platform.stability.ai/account/keys", "v1.user.balance",
|
||||
"v2beta image (generate, edit, upscale, control), 3D, audio; v1 engines. No video route in the published spec."},
|
||||
{"bfl", "Black Forest Labs", {"BFL_API_KEY"}, true, true, Evidence::kOpenApi,
|
||||
"https://dashboard.bfl.ai/", "credits", "FLUX models and tools, polling via get_result, fine-tunes."},
|
||||
{"replicate", "Replicate", {"REPLICATE_API_TOKEN"}, true, true, Evidence::kOpenApi,
|
||||
"https://replicate.com/account/api-tokens", "account.get",
|
||||
"Predictions, models, deployments, files, trainings, webhook verification. Any model by owner/name."},
|
||||
{"fal", "fal", {"FAL_KEY"}, true, true, Evidence::kOpenApi, "https://fal.ai/dashboard/keys",
|
||||
"platform.models.pricing", "Queue API (any endpoint id) and the model-related Platform API."},
|
||||
{"ideogram", "Ideogram", {"IDEOGRAM_API_KEY"}, true, true, Evidence::kOpenApi,
|
||||
"https://ideogram.ai/manage-api", "account.usage",
|
||||
"v2 image, SVG, video and ad tools; dry_run=true gives a free price quote."},
|
||||
{"recraft", "Recraft", {"RECRAFT_API_TOKEN"}, true, false, Evidence::kOpenApi,
|
||||
"https://www.recraft.ai/profile/api", "users.me", "Generation, vector, background, upscale, styles."},
|
||||
{"leonardo", "Leonardo.Ai", {"LEONARDO_API_KEY"}, true, true, Evidence::kDocs,
|
||||
"https://app.leonardo.ai/api-access", "v2.models",
|
||||
"v2 generations for image, video, audio and 3D models. v1 reference pages are gone from the docs."},
|
||||
{"runway", "Runway", {"RUNWAY_API_KEY", "RUNWAYML_API_SECRET"}, true, true, Evidence::kSdk,
|
||||
"https://dev.runwayml.com/", "organization.get",
|
||||
"Text / image / video generation, tasks, uploads, avatars, voices, documents, routers, workflows."},
|
||||
{"luma", "Luma Dream Machine", {"LUMA_API_KEY"}, true, true, Evidence::kSdk,
|
||||
"https://lumalabs.ai/api/keys", "credits.get", "Ray video, Photon image, reframe, modify, upscale, audio."},
|
||||
{"minimax", "MiniMax Hailuo", {"MINIMAX_API_KEY"}, false, true, Evidence::kDocs,
|
||||
"https://platform.minimax.io/user-center/basic-information/interface-key", "",
|
||||
"Hailuo video v1 and v2, file retrieval. No free account read exists."},
|
||||
{"kling", "Kling AI", {"KLING_ACCESS_KEY", "KLING_SECRET_KEY"}, false, true, Evidence::kProbeOnly,
|
||||
"https://app.klingai.com/global/dev/api-key", "",
|
||||
"Official API with JWT auth; docs are a JS app with no machine-readable spec: paths are unverified."},
|
||||
{"higgsfield", "Higgsfield", {"HF_API_KEY_ID", "HF_API_KEY_SECRET"}, true, true, Evidence::kOpenApi,
|
||||
"https://cloud.higgsfield.ai/", "",
|
||||
"One lifecycle for every model path; the OpenAPI document is explicitly supplementary."},
|
||||
};
|
||||
return list;
|
||||
}
|
||||
|
||||
const std::vector<UnsupportedProduct>& MediaRegistry::unsupported() {
|
||||
static const std::vector<UnsupportedProduct> list = {
|
||||
{"Midjourney",
|
||||
"No public API. Midjourney documents only its web and Discord products; third-party relays are unofficial "
|
||||
"and violate its terms, so none is implemented."},
|
||||
{"Pika",
|
||||
"pika.art/api advertises a membership API that resells other vendors' models, but no endpoint, schema or "
|
||||
"authentication document was found (2026-10-25), so nothing is implemented. Pika models are reachable "
|
||||
"through fal."},
|
||||
};
|
||||
return list;
|
||||
}
|
||||
|
||||
const ProviderInfo* MediaRegistry::find(const std::string& id) {
|
||||
for (const ProviderInfo& p : providers())
|
||||
if (p.id == id) return &p;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
const char* MediaRegistry::evidence_name(Evidence e) {
|
||||
switch (e) {
|
||||
case Evidence::kOpenApi: return "openapi";
|
||||
case Evidence::kSdk: return "sdk";
|
||||
case Evidence::kDocs: return "docs";
|
||||
default: return "probe-only";
|
||||
}
|
||||
}
|
||||
|
||||
const CallTable& MediaRegistry::table(const std::string& id) {
|
||||
if (id == "stability") return StabilityCalls::table();
|
||||
if (id == "bfl") return BflCalls::table();
|
||||
if (id == "replicate") return ReplicateCalls::table();
|
||||
if (id == "fal") return FalCalls::table();
|
||||
if (id == "ideogram") return IdeogramCalls::table();
|
||||
if (id == "recraft") return RecraftCalls::table();
|
||||
if (id == "leonardo") return LeonardoCalls::table();
|
||||
if (id == "runway") return RunwayCalls::table();
|
||||
if (id == "luma") return LumaCalls::table();
|
||||
if (id == "minimax") return MinimaxCalls::table();
|
||||
if (id == "kling") return KlingCalls::table();
|
||||
if (id == "higgsfield") return HiggsfieldCalls::table();
|
||||
throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
|
||||
}
|
||||
|
||||
bool MediaRegistry::configured(const std::string& id) {
|
||||
const ProviderInfo* p = find(id);
|
||||
if (!p) throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
|
||||
if (id == "runway") return !llm::Config::get("RUNWAY_API_KEY").empty() || !llm::Config::get("RUNWAYML_API_SECRET").empty();
|
||||
for (const std::string& e : p->env)
|
||||
if (llm::Config::get(e.c_str()).empty()) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::shared_ptr<MediaService> MediaRegistry::service(const std::string& id, const std::string& key) {
|
||||
std::string slot = id;
|
||||
if (id == "fal") slot = key.rfind("platform.", 0) == 0 ? "fal.platform" : "fal.queue";
|
||||
auto it = services_.find(slot);
|
||||
if (it != services_.end()) return it->second;
|
||||
std::shared_ptr<MediaService> s;
|
||||
if (id == "stability") s = StabilityClient::from_env();
|
||||
else if (id == "bfl") s = BflClient::from_env();
|
||||
else if (id == "replicate") s = ReplicateClient::from_env();
|
||||
else if (id == "fal") s = slot == "fal.platform" ? std::shared_ptr<MediaService>(FalPlatform::from_env())
|
||||
: std::shared_ptr<MediaService>(FalQueue::from_env());
|
||||
else if (id == "ideogram") s = IdeogramClient::from_env();
|
||||
else if (id == "recraft") s = RecraftClient::from_env();
|
||||
else if (id == "leonardo") s = LeonardoClient::from_env();
|
||||
else if (id == "runway") s = RunwayClient::from_env();
|
||||
else if (id == "luma") s = LumaClient::from_env();
|
||||
else if (id == "minimax") s = MinimaxClient::from_env();
|
||||
else if (id == "kling") s = KlingClient::from_env();
|
||||
else if (id == "higgsfield") s = HiggsfieldClient::from_env();
|
||||
else throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
|
||||
services_[slot] = s;
|
||||
return s;
|
||||
}
|
||||
|
||||
std::shared_ptr<ImageGenerator> MediaRegistry::image(const std::string& id) {
|
||||
auto it = images_.find(id);
|
||||
if (it != images_.end()) return it->second;
|
||||
std::shared_ptr<ImageGenerator> g;
|
||||
if (id == "stability") g = std::make_shared<StabilityImageGenerator>(std::static_pointer_cast<StabilityClient>(service(id)));
|
||||
else if (id == "bfl") g = std::make_shared<BflImageGenerator>(std::static_pointer_cast<BflClient>(service(id)));
|
||||
else if (id == "replicate") g = std::make_shared<ReplicateImageGenerator>(std::static_pointer_cast<ReplicateClient>(service(id)));
|
||||
else if (id == "fal") g = std::make_shared<FalImageGenerator>(std::static_pointer_cast<FalQueue>(service(id)));
|
||||
else if (id == "ideogram") g = std::make_shared<IdeogramImageGenerator>(std::static_pointer_cast<IdeogramClient>(service(id)));
|
||||
else if (id == "recraft") g = std::make_shared<RecraftImageGenerator>(std::static_pointer_cast<RecraftClient>(service(id)));
|
||||
else if (id == "leonardo") g = std::make_shared<LeonardoImageGenerator>(std::static_pointer_cast<LeonardoClient>(service(id)));
|
||||
else if (id == "runway") g = std::make_shared<RunwayImageGenerator>(std::static_pointer_cast<RunwayClient>(service(id)));
|
||||
else if (id == "luma") g = std::make_shared<LumaImageGenerator>(std::static_pointer_cast<LumaClient>(service(id)));
|
||||
else if (id == "higgsfield") g = std::make_shared<HiggsfieldImageGenerator>(std::static_pointer_cast<HiggsfieldClient>(service(id)));
|
||||
else throw llm::ApiError::invalid("media", "provider '" + id + "' has no image generator");
|
||||
images_[id] = g;
|
||||
return g;
|
||||
}
|
||||
|
||||
std::shared_ptr<VideoGenerator> MediaRegistry::video(const std::string& id) {
|
||||
auto it = videos_.find(id);
|
||||
if (it != videos_.end()) return it->second;
|
||||
std::shared_ptr<VideoGenerator> g;
|
||||
if (id == "bfl") g = std::make_shared<BflVideoGenerator>(std::static_pointer_cast<BflClient>(service(id)));
|
||||
else if (id == "replicate") g = std::make_shared<ReplicateVideoGenerator>(std::static_pointer_cast<ReplicateClient>(service(id)));
|
||||
else if (id == "fal") g = std::make_shared<FalVideoGenerator>(std::static_pointer_cast<FalQueue>(service(id)));
|
||||
else if (id == "ideogram") g = std::make_shared<IdeogramVideoGenerator>(std::static_pointer_cast<IdeogramClient>(service(id)));
|
||||
else if (id == "leonardo") g = std::make_shared<LeonardoVideoGenerator>(std::static_pointer_cast<LeonardoClient>(service(id)));
|
||||
else if (id == "runway") g = std::make_shared<RunwayVideoGenerator>(std::static_pointer_cast<RunwayClient>(service(id)));
|
||||
else if (id == "luma") g = std::make_shared<LumaVideoGenerator>(std::static_pointer_cast<LumaClient>(service(id)));
|
||||
else if (id == "minimax") g = std::make_shared<MinimaxVideoGenerator>(std::static_pointer_cast<MinimaxClient>(service(id)));
|
||||
else if (id == "kling") g = std::make_shared<KlingVideoGenerator>(std::static_pointer_cast<KlingClient>(service(id)));
|
||||
else if (id == "higgsfield") g = std::make_shared<HiggsfieldVideoGenerator>(std::static_pointer_cast<HiggsfieldClient>(service(id)));
|
||||
else throw llm::ApiError::invalid("media", "provider '" + id + "' has no video generator");
|
||||
videos_[id] = g;
|
||||
return g;
|
||||
}
|
||||
|
||||
Json MediaRegistry::account(const std::string& id) {
|
||||
const ProviderInfo* p = find(id);
|
||||
if (!p) throw llm::ApiError::invalid("media", "unknown provider '" + id + "'");
|
||||
if (p->account_key.empty())
|
||||
throw llm::ApiError::invalid("media", id + " has no free account read; use a generation poll to prove the credential");
|
||||
std::shared_ptr<MediaService> s = service(id, p->account_key);
|
||||
CallArgs args;
|
||||
if (id == "fal") args.query["endpoint_id"] = "fal-ai/flux/dev";
|
||||
if (id == "stability") args.headers["Accept"] = "application/json";
|
||||
if (id == "ideogram") { // usage requires start_time (spec: required, date-time)
|
||||
std::time_t t = std::time(nullptr) - 24 * 3600;
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
char buf[32];
|
||||
std::strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%SZ", &tm);
|
||||
args.query["start_time"] = buf;
|
||||
}
|
||||
return s->call_json(p->account_key, args);
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+72
@@ -0,0 +1,72 @@
|
||||
#pragma once
|
||||
|
||||
// MediaRegistry: the one list of providers in package media, what each one
|
||||
// can do, which environment variables it needs and how well its API surface
|
||||
// is verified. Builds generators and call services from the environment on
|
||||
// demand and caches them. Products without a public API are listed too, with
|
||||
// the reason, so nothing is silently missing.
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// How the provider's call table was checked (see each SPEC).
|
||||
enum class Evidence {
|
||||
kOpenApi, // vendor-published OpenAPI document, checked by tools/check_spec.py
|
||||
kSdk, // official SDK source, checked by tools/check_spec.py
|
||||
kDocs, // vendor doc pages, checked by tools/check_spec.py
|
||||
kProbeOnly // no machine-readable source: community sources plus live probes
|
||||
};
|
||||
|
||||
struct ProviderInfo {
|
||||
std::string id;
|
||||
std::string name;
|
||||
std::vector<std::string> env; // every variable the credential needs
|
||||
bool image = false;
|
||||
bool video = false;
|
||||
Evidence evidence = Evidence::kOpenApi;
|
||||
std::string console_url; // where the key is created
|
||||
std::string account_key; // table key of a free read that proves the credential ("" = none)
|
||||
std::string note;
|
||||
};
|
||||
|
||||
struct UnsupportedProduct {
|
||||
std::string name;
|
||||
std::string reason;
|
||||
};
|
||||
|
||||
class MediaRegistry {
|
||||
public:
|
||||
static const std::vector<ProviderInfo>& providers();
|
||||
static const std::vector<UnsupportedProduct>& unsupported();
|
||||
static const ProviderInfo* find(const std::string& id);
|
||||
static const char* evidence_name(Evidence evidence);
|
||||
// Table of one provider (no credential needed). Throws ApiError::invalid.
|
||||
static const CallTable& table(const std::string& id);
|
||||
|
||||
// True when every env variable the provider needs is set (llm::Config).
|
||||
static bool configured(const std::string& id);
|
||||
|
||||
// Built from the environment on first use, then cached for this registry.
|
||||
// `key` picks the host for providers with two (fal: queue.* / platform.*).
|
||||
std::shared_ptr<MediaService> service(const std::string& id, const std::string& key = "");
|
||||
std::shared_ptr<ImageGenerator> image(const std::string& id);
|
||||
std::shared_ptr<VideoGenerator> video(const std::string& id);
|
||||
|
||||
// Free read that proves the credential works (balance, account, models).
|
||||
// Returns the provider's JSON. Throws NotConfiguredError without a key.
|
||||
Json account(const std::string& id);
|
||||
|
||||
private:
|
||||
std::map<std::string, std::shared_ptr<MediaService>> services_;
|
||||
std::map<std::string, std::shared_ptr<ImageGenerator>> images_;
|
||||
std::map<std::string, std::shared_ptr<VideoGenerator>> videos_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+41
@@ -0,0 +1,41 @@
|
||||
#include "vendors/media/fal/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"queue.submit", "POST", "/{owner}/{app}", "submitQueueRequest", "json", "*", "fal_webhook,fal_max_queue_length", "paid"},
|
||||
{"queue.submit_path", "POST", "/{owner}/{app}/{path}", "submitQueueRequestWithPath", "json", "*", "fal_webhook,fal_max_queue_length", "paid"},
|
||||
{"queue.status", "GET", "/{owner}/{app}/requests/{request_id}/status", "getQueueRequestStatus", "none", "", "logs", "read"},
|
||||
{"queue.result", "GET", "/{owner}/{app}/requests/{request_id}", "getQueueRequestResult", "none", "", "", "read"},
|
||||
{"queue.cancel", "PUT", "/{owner}/{app}/requests/{request_id}/cancel", "cancelQueueRequest", "none", "", "", "write"},
|
||||
{"platform.models.insights", "GET", "/v1/models/insights", "getModelInsights", "none", "", "endpoint_id,q,limit", "read"},
|
||||
{"platform.models.search", "GET", "/v1/models", "getModels", "none", "", "limit,cursor,endpoint_id,q,category,status,sort,expand", "read"},
|
||||
{"platform.models.pricing", "GET", "/v1/models/pricing", "getPricing", "none", "", "endpoint_id", "read"},
|
||||
{"platform.models.estimate", "POST", "/v1/models/pricing/estimate", "estimatePricing", "json", "estimate_type,endpoints", "", "read"},
|
||||
{"platform.models.usage", "GET", "/v1/models/usage", "getUsage", "none", "", "limit,cursor,start,end,timezone,timeframe,bound_to_timeframe,source,endpoint_id,api_key_id,login_username,tag,expand", "read"},
|
||||
{"platform.models.analytics", "GET", "/v1/models/analytics", "getAnalytics", "none", "", "limit,cursor,start,end,timezone,timeframe,bound_to_timeframe,endpoint_id,expand", "read"},
|
||||
{"platform.models.billing_events", "GET", "/v1/models/billing-events", "getBillingEvents", "none", "", "limit,cursor,start,end,source,endpoint_id,request_id,tag,api_key_id,login_username,expand", "read"},
|
||||
{"platform.models.delete_payloads", "DELETE", "/v1/models/requests/{request_id}/payloads", "deleteRequestPayloads", "none", "", "", "delete"},
|
||||
{"platform.models.requests_by_endpoint", "GET", "/v1/models/requests/by-endpoint", "listRequestsByEndpoint", "none", "", "limit,cursor,endpoint_id,start,end,status,request_id,expand,sort_by", "read"},
|
||||
{"platform.models.requests_search", "GET", "/v1/models/requests/search", "searchRequests", "none", "", "limit,cursor,query,image_url,video_url,endpoint_id,endpoint,exclude_api_requests,only_api_requests,min_similarity", "read"},
|
||||
{"platform.meta", "GET", "/v1/meta", "getMeta", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"QueueStatus", "status,request_id,response_url,status_url,cancel_url,logs,metrics,queue_position"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& FalCalls::table() {
|
||||
static const CallTable t("fal", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t FalCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* FalCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct FalCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+260
@@ -0,0 +1,260 @@
|
||||
#include "vendors/media/fal/fal.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs ids(const FalEndpoint& e, const std::string& request_id = "") {
|
||||
CallArgs a;
|
||||
a.path["owner"] = e.owner;
|
||||
a.path["app"] = e.app;
|
||||
if (!request_id.empty()) a.path["request_id"] = request_id;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
FalEndpoint FalEndpoint::parse(const std::string& id) {
|
||||
std::vector<std::string> parts;
|
||||
std::size_t start = 0;
|
||||
while (true) {
|
||||
std::size_t slash = id.find('/', start);
|
||||
parts.push_back(id.substr(start, slash == std::string::npos ? std::string::npos : slash - start));
|
||||
if (slash == std::string::npos) break;
|
||||
start = slash + 1;
|
||||
}
|
||||
if (parts.size() < 2 || parts.size() > 5)
|
||||
throw llm::ApiError::invalid("fal", "endpoint must look like owner/app or owner/app/path");
|
||||
for (const std::string& p : parts)
|
||||
if (!CallTable::valid_segment(p)) throw llm::ApiError::invalid("fal", "bad endpoint id");
|
||||
FalEndpoint e;
|
||||
e.owner = parts[0];
|
||||
e.app = parts[1];
|
||||
// Everything below owner/app is the {path} of the submit_path row; the
|
||||
// call table accepts several "/"-joined segments for it.
|
||||
for (std::size_t i = 2; i < parts.size(); ++i) e.path += (i > 2 ? "/" : "") + parts[i];
|
||||
return e;
|
||||
}
|
||||
|
||||
PreparedCall FalQueue::plan(const FalEndpoint& e, const Json& body, const FalSubmitOptions& o) const {
|
||||
CallArgs a = ids(e);
|
||||
a.body = body;
|
||||
if (!e.path.empty()) {
|
||||
a.path["path"] = e.path;
|
||||
}
|
||||
if (!o.webhook.empty()) a.query["fal_webhook"] = o.webhook;
|
||||
if (o.max_queue_length > 0) a.query["fal_max_queue_length"] = std::to_string(o.max_queue_length);
|
||||
if (!o.priority.empty()) {
|
||||
if (o.priority != "low" && o.priority != "normal" && o.priority != "high")
|
||||
throw llm::ApiError::invalid("fal", "priority must be low, normal or high");
|
||||
a.headers["X-Fal-Queue-Priority"] = o.priority;
|
||||
}
|
||||
return prepare(e.path.empty() ? "queue.submit" : "queue.submit_path", a);
|
||||
}
|
||||
|
||||
Json FalQueue::submit(const FalEndpoint& e, const Json& body, const FalSubmitOptions& o, bool confirm) {
|
||||
return parse_json("fal", execute(plan(e, body, o), confirm));
|
||||
}
|
||||
|
||||
Json FalQueue::status(const FalEndpoint& e, const std::string& request_id, bool logs) {
|
||||
CallArgs a = ids(e, request_id);
|
||||
if (logs) a.query["logs"] = "1";
|
||||
return call_json("queue.status", a);
|
||||
}
|
||||
|
||||
Json FalQueue::result(const FalEndpoint& e, const std::string& request_id) {
|
||||
return call_json("queue.result", ids(e, request_id));
|
||||
}
|
||||
|
||||
Json FalQueue::cancel(const FalEndpoint& e, const std::string& request_id, bool confirm) {
|
||||
return call_json("queue.cancel", ids(e, request_id), confirm);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- platform
|
||||
|
||||
Json FalPlatform::models(const std::string& query, const std::string& category, std::size_t max_pages,
|
||||
std::size_t max_items) {
|
||||
if (max_pages < 1 || max_pages > MediaCaps::kMaxPages) throw llm::ApiError::invalid("fal", "max_pages out of range");
|
||||
if (max_items < 1 || max_items > MediaCaps::kMaxItemsReturned) throw llm::ApiError::invalid("fal", "max_items out of range");
|
||||
Json all = Json::array();
|
||||
std::string cursor;
|
||||
bool more = false;
|
||||
std::size_t pages = 0;
|
||||
while (pages < max_pages) {
|
||||
CallArgs a;
|
||||
a.query["limit"] = std::to_string(std::min<std::size_t>(max_items - all.as_array().size(), 100));
|
||||
if (!query.empty()) a.query["q"] = query;
|
||||
if (!category.empty()) a.query["category"] = category;
|
||||
if (!cursor.empty()) a.query["cursor"] = cursor;
|
||||
Json page = call_json("platform.models.search", a);
|
||||
++pages;
|
||||
if (const Json* m = page.find("models"); m && m->is_array())
|
||||
for (const Json& x : m->as_array())
|
||||
if (all.as_array().size() < max_items) all.push(x);
|
||||
const Json* hm = page.find("has_more");
|
||||
more = hm && hm->is_bool() && hm->as_bool();
|
||||
cursor = AdapterSupport::str(page, "next_cursor");
|
||||
if (!more || cursor.empty() || all.as_array().size() >= max_items) break;
|
||||
}
|
||||
Json out = Json::object();
|
||||
out.set("models", all);
|
||||
out.set("pages", Json(static_cast<double>(pages)));
|
||||
out.set("truncated", Json(more));
|
||||
return out;
|
||||
}
|
||||
|
||||
Json FalPlatform::pricing(const std::string& endpoint_id) {
|
||||
CallArgs a;
|
||||
a.query["endpoint_id"] = endpoint_id;
|
||||
return call_json("platform.models.pricing", a);
|
||||
}
|
||||
|
||||
Json FalPlatform::estimate(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return call_json("platform.models.estimate", a);
|
||||
}
|
||||
|
||||
Json FalPlatform::usage(const std::map<std::string, std::string>& query) {
|
||||
CallArgs a;
|
||||
a.query = query;
|
||||
return call_json("platform.models.usage", a);
|
||||
}
|
||||
|
||||
Json FalPlatform::analytics(const std::map<std::string, std::string>& query) {
|
||||
CallArgs a;
|
||||
a.query = query;
|
||||
return call_json("platform.models.analytics", a);
|
||||
}
|
||||
|
||||
Json FalPlatform::meta() { return call_json("platform.meta"); }
|
||||
|
||||
Json FalPlatform::delete_payloads(const std::string& request_id, bool confirm) {
|
||||
CallArgs a;
|
||||
a.path["request_id"] = request_id;
|
||||
return call_json("platform.models.delete_payloads", a, confirm);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- jobs
|
||||
|
||||
JobTicket FalJobs::ticket_from(const Json& reply, const FalEndpoint& e, MediaKind kind) {
|
||||
JobTicket t;
|
||||
t.provider = "fal";
|
||||
t.model = e.full();
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(reply, "request_id");
|
||||
if (t.id.empty()) throw llm::ApiError("fal", 502, "BAD_RESPONSE", "no request_id in the submit reply");
|
||||
t.meta = Json::object();
|
||||
t.meta.set("endpoint", Json(e.full()));
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus FalJobs::poll(FalQueue& queue, const JobTicket& ticket) {
|
||||
FalEndpoint e = FalEndpoint::parse(AdapterSupport::str(ticket.meta, "endpoint"));
|
||||
Json st = queue.status(e, ticket.id);
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(st, "status");
|
||||
if (s.raw_state == "IN_QUEUE") {
|
||||
s.state = JobState::kQueued;
|
||||
return s;
|
||||
}
|
||||
if (s.raw_state != "COMPLETED") {
|
||||
s.state = JobState::kRunning;
|
||||
return s;
|
||||
}
|
||||
// COMPLETED is final for success, failure and cancellation alike: the
|
||||
// result fetch tells which (a failed request answers with an HTTP error).
|
||||
try {
|
||||
Json res = queue.result(e, ticket.id);
|
||||
s.state = JobState::kSucceeded;
|
||||
s.assets = OutputScanner::scan(res, ticket.kind);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "completed without a result URL: " + res.dump().substr(0, 300);
|
||||
}
|
||||
} catch (const llm::ApiError& err) {
|
||||
if (err.http_status() == 401 || err.http_status() == 403 || err.http_status() >= 500 || err.http_status() == 429) throw;
|
||||
s.state = JobState::kFailed;
|
||||
s.error = err.message();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
bool FalJobs::cancel(FalQueue& queue, const JobTicket& ticket, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "fal", "cancel");
|
||||
queue.cancel(FalEndpoint::parse(AdapterSupport::str(ticket.meta, "endpoint")), ticket.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
Json FalImageGenerator::body(const ImageRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
|
||||
if (r.width && r.height) {
|
||||
Json size = Json::object();
|
||||
size.set("width", Json(r.width));
|
||||
size.set("height", Json(r.height));
|
||||
b.set("image_size", size);
|
||||
}
|
||||
AdapterSupport::put_seed(b, "seed", r.seed);
|
||||
if (r.count > 1) b.set("num_images", Json(r.count));
|
||||
AdapterSupport::put(b, "image_url", r.source_url);
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json FalImageGenerator::plan(const ImageRequest& r) const {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
|
||||
return queue_->plan(FalEndpoint::parse(r.model), body(r), {}).summary();
|
||||
}
|
||||
|
||||
JobTicket FalImageGenerator::submit(const ImageRequest& r, bool confirm) {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
|
||||
FalEndpoint e = FalEndpoint::parse(r.model);
|
||||
Json b = body(r);
|
||||
AdapterSupport::need_confirm(confirm, "fal", "queue submit");
|
||||
return FalJobs::ticket_from(queue_->submit(e, b, {}, true), e, MediaKind::kImage);
|
||||
}
|
||||
|
||||
JobStatus FalImageGenerator::poll(const JobTicket& t) { return FalJobs::poll(*queue_, t); }
|
||||
bool FalImageGenerator::cancel(const JobTicket& t, bool confirm) { return FalJobs::cancel(*queue_, t, confirm); }
|
||||
|
||||
Json FalVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
|
||||
if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds)));
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put(b, "resolution", r.resolution);
|
||||
AdapterSupport::put_seed(b, "seed", r.seed);
|
||||
AdapterSupport::put(b, "image_url", r.image_url);
|
||||
AdapterSupport::put(b, "end_image_url", r.end_image_url);
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json FalVideoGenerator::plan(const VideoRequest& r) const {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
|
||||
return queue_->plan(FalEndpoint::parse(r.model), body(r), {}).summary();
|
||||
}
|
||||
|
||||
JobTicket FalVideoGenerator::submit(const VideoRequest& r, bool confirm) {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("fal", "model (endpoint id) is required");
|
||||
FalEndpoint e = FalEndpoint::parse(r.model);
|
||||
Json b = body(r);
|
||||
AdapterSupport::need_confirm(confirm, "fal", "queue submit");
|
||||
return FalJobs::ticket_from(queue_->submit(e, b, {}, true), e, MediaKind::kVideo);
|
||||
}
|
||||
|
||||
JobStatus FalVideoGenerator::poll(const JobTicket& t) { return FalJobs::poll(*queue_, t); }
|
||||
bool FalVideoGenerator::cancel(const JobTicket& t, bool confirm) { return FalJobs::cancel(*queue_, t, confirm); }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+128
@@ -0,0 +1,128 @@
|
||||
#pragma once
|
||||
|
||||
// fal: the queue API (queue.fal.run: submit, status, result, cancel) and the
|
||||
// model-related Platform API (api.fal.ai/v1: model search, pricing, estimate,
|
||||
// usage, analytics, billing events, request listing). Auth is
|
||||
// "Authorization: Key <FAL_KEY>" on both hosts. Platform groups outside model
|
||||
// inference (assets, serverless, compute, keys, organization) are out of scope
|
||||
// and listed as exclusions in tools/check_spec.py.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/fal/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// "owner/app" or "owner/app/path" (fal-ai/flux/dev, fal-ai/flux/dev/image-to-image).
|
||||
struct FalEndpoint {
|
||||
std::string owner, app, path;
|
||||
static FalEndpoint parse(const std::string& id); // throws ApiError::invalid
|
||||
std::string app_id() const { return owner + "/" + app; }
|
||||
std::string full() const { return path.empty() ? app_id() : app_id() + "/" + path; }
|
||||
};
|
||||
|
||||
struct FalSubmitOptions {
|
||||
std::string priority; // "", "low", "normal", "high" -> X-Fal-Queue-Priority
|
||||
std::string webhook; // https URL -> fal_webhook
|
||||
int max_queue_length = 0; // 0 = unset -> fal_max_queue_length
|
||||
};
|
||||
|
||||
class FalQueue : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://queue.fal.run";
|
||||
static constexpr const char* kKeyEnv = "FAL_KEY";
|
||||
|
||||
explicit FalQueue(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), FalCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::header("fal", kHost, kKeyEnv, "Authorization"); }
|
||||
// fal sends the key as "Key <value>".
|
||||
static std::string header_value(const std::string& key) { return key.empty() ? "" : "Key " + key; }
|
||||
static std::shared_ptr<FalQueue> from_env() {
|
||||
return std::make_shared<FalQueue>(ClientFactory::real(config(), header_value(ClientFactory::env({kKeyEnv}))));
|
||||
}
|
||||
static std::shared_ptr<FalQueue> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<FalQueue>(ClientFactory::with(std::move(t), config(), header_value(key)));
|
||||
}
|
||||
|
||||
PreparedCall plan(const FalEndpoint& endpoint, const Json& body, const FalSubmitOptions& options) const;
|
||||
// Paid. Returns {request_id, response_url, status_url, cancel_url, ...}.
|
||||
Json submit(const FalEndpoint& endpoint, const Json& body, const FalSubmitOptions& options, bool confirm);
|
||||
Json status(const FalEndpoint& endpoint, const std::string& request_id, bool logs = false);
|
||||
Json result(const FalEndpoint& endpoint, const std::string& request_id);
|
||||
Json cancel(const FalEndpoint& endpoint, const std::string& request_id, bool confirm);
|
||||
};
|
||||
|
||||
class FalPlatform : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.fal.ai";
|
||||
|
||||
explicit FalPlatform(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), FalCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::header("fal", kHost, FalQueue::kKeyEnv, "Authorization"); }
|
||||
static std::shared_ptr<FalPlatform> from_env() {
|
||||
return std::make_shared<FalPlatform>(
|
||||
ClientFactory::real(config(), FalQueue::header_value(ClientFactory::env({FalQueue::kKeyEnv}))));
|
||||
}
|
||||
static std::shared_ptr<FalPlatform> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<FalPlatform>(ClientFactory::with(std::move(t), config(), FalQueue::header_value(key)));
|
||||
}
|
||||
|
||||
// Search; walks next_cursor up to max_pages / max_items and says so loudly
|
||||
// ("truncated": true) when the server had more.
|
||||
Json models(const std::string& query, const std::string& category, std::size_t max_pages = 3,
|
||||
std::size_t max_items = 200);
|
||||
Json pricing(const std::string& endpoint_id);
|
||||
Json estimate(const Json& body); // pure computation on fal's side, no charge
|
||||
Json usage(const std::map<std::string, std::string>& query);
|
||||
Json analytics(const std::map<std::string, std::string>& query);
|
||||
Json meta();
|
||||
Json delete_payloads(const std::string& request_id, bool confirm); // destructive, no default
|
||||
};
|
||||
|
||||
class FalImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit FalImageGenerator(std::shared_ptr<FalQueue> q) : queue_(std::move(q)) {}
|
||||
std::string provider() const override { return "fal"; }
|
||||
bool configured() const override { return queue_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"fal-ai/flux/dev", "fal-ai/flux/schnell", "fal-ai/flux-pro/v1.1", "fal-ai/recraft/v3/text-to-image",
|
||||
"fal-ai/ideogram/v3", "fal-ai/stable-diffusion-v35-large"};
|
||||
}
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 1500; }
|
||||
|
||||
private:
|
||||
Json body(const ImageRequest& request) const;
|
||||
std::shared_ptr<FalQueue> queue_;
|
||||
};
|
||||
|
||||
class FalVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit FalVideoGenerator(std::shared_ptr<FalQueue> q) : queue_(std::move(q)) {}
|
||||
std::string provider() const override { return "fal"; }
|
||||
bool configured() const override { return queue_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"fal-ai/kling-video/v2.1/master/text-to-video", "fal-ai/minimax/hailuo-02/standard/text-to-video",
|
||||
"fal-ai/veo3", "fal-ai/luma-dream-machine"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 4000; }
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<FalQueue> queue_;
|
||||
};
|
||||
|
||||
// Shared queue polling for both fal adapters.
|
||||
struct FalJobs {
|
||||
static JobTicket ticket_from(const Json& submit_reply, const FalEndpoint& endpoint, MediaKind kind);
|
||||
static JobStatus poll(FalQueue& queue, const JobTicket& ticket);
|
||||
static bool cancel(FalQueue& queue, const JobTicket& ticket, bool confirm);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
#include "vendors/media/higgsfield/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"higgsfield-ai.soul.standard", "POST", "/higgsfield-ai/soul/standard", "post_higgsfield_ai_soul_standard", "json", "prompt,num_images,resolution,aspect_ratio", "", "paid"},
|
||||
{"requests.id.status", "GET", "/requests/{request_id}/status", "get_request_status", "none", "", "", "read"},
|
||||
{"requests.id.cancel", "POST", "/requests/{request_id}/cancel", "cancel_request", "none", "", "", "write"},
|
||||
{"kling-video.v2.5-turbo.pro.image-to-video", "POST", "/kling-video/v2.5-turbo/pro/image-to-video", "post_kling_video_v2_5_turbo_pro_image_to_video", "json", "prompt,duration,cfg_scale,image_url,negative_prompt", "", "paid"},
|
||||
{"kling-video.v2.5-turbo.pro.text-to-video", "POST", "/kling-video/v2.5-turbo/pro/text-to-video", "post_kling_video_v2_5_turbo_pro_text_to_video", "json", "prompt,duration,cfg_scale,negative_prompt", "", "paid"},
|
||||
{"kling-video.v2.5-turbo.standard.image-to-video", "POST", "/kling-video/v2.5-turbo/standard/image-to-video", "post_kling_video_v2_5_turbo_standard_image_to_video", "json", "prompt,duration,cfg_scale,image_url,negative_prompt", "", "paid"},
|
||||
{"minimax.hailuo-2.3.standard.image-to-video", "POST", "/minimax/hailuo-2.3/standard/image-to-video", "post_minimax_hailuo_2_3_standard_image_to_video", "json", "prompt,duration,image_url,prompt_optimizer", "", "paid"},
|
||||
{"minimax.hailuo-2.3.standard.text-to-video", "POST", "/minimax/hailuo-2.3/standard/text-to-video", "post_minimax_hailuo_2_3_standard_text_to_video", "json", "prompt,duration,prompt_optimizer", "", "paid"},
|
||||
{"requests.id", "DELETE", "/requests/{request_id}", "delete_request", "none", "", "", "delete"},
|
||||
{"model.submit", "POST", "/{model_path}", "", "json", "*", "", "paid"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"RequestStatus", "status,request_id,status_url,cancel_url,error,images,video,url"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& HiggsfieldCalls::table() {
|
||||
static const CallTable t("higgsfield", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t HiggsfieldCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* HiggsfieldCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct HiggsfieldCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
#include "vendors/media/higgsfield/higgsfield.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs for_model(const std::string& model_path, const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
if (HiggsfieldClient::row_for(model_path) == "model.submit") a.path["model_path"] = model_path;
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs by_request(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["request_id"] = id;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string HiggsfieldClient::row_for(const std::string& model_path) {
|
||||
std::string key = model_path;
|
||||
for (char& c : key)
|
||||
if (c == '/') c = '.';
|
||||
return HiggsfieldCalls::table().find(key) ? key : "model.submit";
|
||||
}
|
||||
|
||||
PreparedCall HiggsfieldClient::plan(const std::string& model_path, const Json& body) const {
|
||||
return prepare(row_for(model_path), for_model(model_path, body));
|
||||
}
|
||||
|
||||
Json HiggsfieldClient::submit(const std::string& model_path, const Json& body, bool confirm) {
|
||||
return call_json(row_for(model_path), for_model(model_path, body), confirm);
|
||||
}
|
||||
|
||||
Json HiggsfieldClient::status(const std::string& id) { return call_json("requests.id.status", by_request(id)); }
|
||||
Json HiggsfieldClient::cancel(const std::string& id, bool confirm) {
|
||||
return call_json("requests.id.cancel", by_request(id), confirm);
|
||||
}
|
||||
Json HiggsfieldClient::remove(const std::string& id, bool confirm) {
|
||||
return call_json("requests.id", by_request(id), confirm);
|
||||
}
|
||||
|
||||
JobState HiggsfieldJobs::state_of(const std::string& status) {
|
||||
if (status == "completed") return JobState::kSucceeded;
|
||||
if (status == "failed" || status == "nsfw") return JobState::kFailed;
|
||||
if (status == "canceled") return JobState::kCancelled;
|
||||
if (status == "in_progress") return JobState::kRunning;
|
||||
return JobState::kQueued; // queued
|
||||
}
|
||||
|
||||
JobTicket HiggsfieldJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) {
|
||||
JobTicket t;
|
||||
t.provider = "higgsfield";
|
||||
t.model = model;
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(reply, "request_id");
|
||||
if (t.id.empty()) throw llm::ApiError("higgsfield", 502, "BAD_RESPONSE", "no request_id in the reply");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus HiggsfieldJobs::poll(HiggsfieldClient& client, const JobTicket& ticket) {
|
||||
Json st = client.status(ticket.id);
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(st, "status");
|
||||
s.state = state_of(s.raw_state);
|
||||
if (s.raw_state == "nsfw") s.error = "rejected by the content filter (nsfw)";
|
||||
if (s.state == JobState::kFailed && s.error.empty()) s.error = AdapterSupport::str(st, "error");
|
||||
if (s.state == JobState::kSucceeded) {
|
||||
for (const char* key : {"images", "video"})
|
||||
if (const Json* x = st.find(key)) {
|
||||
auto found = OutputScanner::scan(*x, ticket.kind);
|
||||
s.assets.insert(s.assets.end(), found.begin(), found.end());
|
||||
}
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "completed without images or video";
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Json HiggsfieldImageGenerator::body(const ImageRequest& r, std::string& model) const {
|
||||
r.validate();
|
||||
model = r.model.empty() ? "higgsfield-ai/soul/standard" : r.model;
|
||||
Json b = Json::object();
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
if (r.count > 1) b.set("num_images", Json(r.count));
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put_seed(b, "seed", r.seed);
|
||||
AdapterSupport::put(b, "image_url", r.source_url);
|
||||
if (r.width || r.height || !r.negative_prompt.empty())
|
||||
throw llm::ApiError::invalid("higgsfield", "width / height / negative_prompt are model-specific: pass them in extra");
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json HiggsfieldImageGenerator::plan(const ImageRequest& request) const {
|
||||
std::string model;
|
||||
Json b = body(request, model);
|
||||
return client_->plan(model, b).summary();
|
||||
}
|
||||
|
||||
JobTicket HiggsfieldImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
std::string model;
|
||||
Json b = body(request, model);
|
||||
AdapterSupport::need_confirm(confirm, "higgsfield", "submit");
|
||||
return HiggsfieldJobs::ticket_from(client_->submit(model, b, true), model, MediaKind::kImage);
|
||||
}
|
||||
|
||||
bool HiggsfieldImageGenerator::cancel(const JobTicket& t, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "higgsfield", "cancel");
|
||||
client_->cancel(t.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
Json HiggsfieldVideoGenerator::body(const VideoRequest& r, std::string& model) const {
|
||||
r.validate();
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("higgsfield", "model path is required (see models())");
|
||||
model = r.model;
|
||||
Json b = Json::object();
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put(b, "resolution", r.resolution);
|
||||
AdapterSupport::put_int(b, "duration", r.duration_seconds);
|
||||
AdapterSupport::put(b, "image_url", r.image_url);
|
||||
AdapterSupport::put(b, "end_image_url", r.end_image_url);
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json HiggsfieldVideoGenerator::plan(const VideoRequest& request) const {
|
||||
std::string model;
|
||||
Json b = body(request, model);
|
||||
return client_->plan(model, b).summary();
|
||||
}
|
||||
|
||||
JobTicket HiggsfieldVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
std::string model;
|
||||
Json b = body(request, model);
|
||||
AdapterSupport::need_confirm(confirm, "higgsfield", "submit");
|
||||
return HiggsfieldJobs::ticket_from(client_->submit(model, b, true), model, MediaKind::kVideo);
|
||||
}
|
||||
|
||||
bool HiggsfieldVideoGenerator::cancel(const JobTicket& t, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "higgsfield", "cancel");
|
||||
client_->cancel(t.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
#pragma once
|
||||
|
||||
// Higgsfield cloud API (https://api.higgsfield.ai): every model is an
|
||||
// endpoint path ("higgsfield-ai/soul/standard", "kling-video/v2.5-turbo/pro/
|
||||
// text-to-video", ...); the shared lifecycle is submit -> GET
|
||||
// /requests/{id}/status -> optional cancel / delete. Auth is
|
||||
// "Authorization: Key <HF_API_KEY_ID>:<HF_API_KEY_SECRET>". The published
|
||||
// OpenAPI document is explicitly supplementary: a model absent from it is not
|
||||
// unavailable, so submit() accepts any model path through the generic row.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/higgsfield/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class HiggsfieldClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.higgsfield.ai";
|
||||
static constexpr const char* kIdEnv = "HF_API_KEY_ID";
|
||||
static constexpr const char* kSecretEnv = "HF_API_KEY_SECRET";
|
||||
|
||||
explicit HiggsfieldClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), HiggsfieldCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::header("higgsfield", kHost, kIdEnv, "Authorization"); }
|
||||
// "Key id:secret"; empty when either half is missing.
|
||||
static std::string header_value(const std::string& id, const std::string& secret) {
|
||||
return id.empty() || secret.empty() ? "" : "Key " + id + ":" + secret;
|
||||
}
|
||||
static std::shared_ptr<HiggsfieldClient> from_env() {
|
||||
return std::make_shared<HiggsfieldClient>(ClientFactory::real(
|
||||
config(), header_value(ClientFactory::env({kIdEnv}), ClientFactory::env({kSecretEnv}))));
|
||||
}
|
||||
static std::shared_ptr<HiggsfieldClient> with(std::unique_ptr<clients::Transport> t, const std::string& id,
|
||||
const std::string& secret) {
|
||||
return std::make_shared<HiggsfieldClient>(ClientFactory::with(std::move(t), config(), header_value(id, secret)));
|
||||
}
|
||||
|
||||
// Uses the spec's typed row when the path is one of them, else model.submit.
|
||||
Json submit(const std::string& model_path, const Json& body, bool confirm);
|
||||
PreparedCall plan(const std::string& model_path, const Json& body) const;
|
||||
Json status(const std::string& request_id);
|
||||
Json cancel(const std::string& request_id, bool confirm); // queued requests only
|
||||
Json remove(const std::string& request_id, bool confirm); // deletes files and parameters
|
||||
static std::string row_for(const std::string& model_path);
|
||||
};
|
||||
|
||||
class HiggsfieldJobs {
|
||||
public:
|
||||
static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind);
|
||||
static JobStatus poll(HiggsfieldClient& client, const JobTicket& ticket);
|
||||
static JobState state_of(const std::string& status);
|
||||
};
|
||||
|
||||
class HiggsfieldImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit HiggsfieldImageGenerator(std::shared_ptr<HiggsfieldClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "higgsfield"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override { return {"higgsfield-ai/soul/standard"}; }
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override { return HiggsfieldJobs::poll(*client_, ticket); }
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 3000; }
|
||||
|
||||
private:
|
||||
Json body(const ImageRequest& request, std::string& model) const;
|
||||
std::shared_ptr<HiggsfieldClient> client_;
|
||||
};
|
||||
|
||||
class HiggsfieldVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit HiggsfieldVideoGenerator(std::shared_ptr<HiggsfieldClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "higgsfield"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"kling-video/v2.5-turbo/pro/text-to-video", "kling-video/v2.5-turbo/pro/image-to-video",
|
||||
"kling-video/v2.5-turbo/standard/image-to-video", "minimax/hailuo-2.3/standard/text-to-video",
|
||||
"minimax/hailuo-2.3/standard/image-to-video"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override { return HiggsfieldJobs::poll(*client_, ticket); }
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 5000; }
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request, std::string& model) const;
|
||||
std::shared_ptr<HiggsfieldClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+96
@@ -0,0 +1,96 @@
|
||||
#include "vendors/media/ideogram/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"image.precise-edit.ideogram-4-5", "POST", "/v2/image/precise-edit/ideogram-4-5", "imagesPreciseEdit_post_precise_edit_image_v2_ideogram45", "multipart", "prompt,image,reference_images,mask,context_window,quality,seed,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.auto", "POST", "/v2/image/generate/auto", "autoModel_post_generate_image_v2_auto", "multipart", "prompt,images,negative_prompt,aspect_ratio,resolution,resolution_tier,magic_prompt,num_images,seed,private,async", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-4-5", "POST", "/v2/image/generate/ideogram-4-5", "imagesGenerate_post_generate_image_v2_ideogram45", "multipart", "prompt,magic_prompt,images,mask,size,background,quality,seed,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-4", "POST", "/v2/image/generate/ideogram-4", "imagesGenerate_post_generate_image_v2_ideogram_v4", "json", "prompt,magic_prompt,seed,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-4-custom-model", "POST", "/v2/image/generate/ideogram-4-custom-model", "imagesGenerate_post_generate_image_v2_ideogram_v4_custom_model", "json", "prompt,custom_model_uri,transparent_background,magic_prompt,seed,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-4-transparent", "POST", "/v2/image/generate/ideogram-4-transparent", "imagesGenerate_post_generate_image_v2_ideogram_v4_transparent", "json", "prompt,magic_prompt,seed,num_images,aspect_ratio,output_resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-3", "POST", "/v2/image/generate/ideogram-3", "imagesGenerate_post_generate_image_v2_ideogram_v3", "multipart", "prompt,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_type,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-3-character", "POST", "/v2/image/generate/ideogram-3-character", "imagesGenerate_post_generate_image_v2_ideogram_v3_character", "multipart", "prompt,negative_prompt,character_reference_images,character_reference_mask,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-3-custom-model", "POST", "/v2/image/generate/ideogram-3-custom-model", "imagesGenerate_post_generate_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,transparent_background,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-3-transparent", "POST", "/v2/image/generate/ideogram-3-transparent", "imagesGenerate_post_generate_image_v2_ideogram_v3_transparent", "json", "prompt,negative_prompt,seed,upscale_factor,aspect_ratio,rendering_speed,magic_prompt,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-2a", "POST", "/v2/image/generate/ideogram-2a", "imagesGenerate_post_generate_image_v2_ideogram_v2_a", "json", "prompt,seed,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.ideogram-2", "POST", "/v2/image/generate/ideogram-2", "imagesGenerate_post_generate_image_v2_ideogram_v2", "json", "prompt,negative_prompt,seed,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.gpt-image-2-5-flare", "POST", "/v2/image/generate/gpt-image-2-5-flare", "imagesGenerate_post_generate_image_v2_gpt_image25_flare", "multipart", "prompt,images,mask,background,quality,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.gpt-image-2-5-sunburst", "POST", "/v2/image/generate/gpt-image-2-5-sunburst", "imagesGenerate_post_generate_image_v2_gpt_image25_sunburst", "multipart", "prompt,images,mask,background,quality,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.gpt-image-2", "POST", "/v2/image/generate/gpt-image-2", "imagesGenerate_post_generate_image_v2_gpt_image2", "multipart", "prompt,images,mask,num_images,seed,aspect_ratio,resolution,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.nano-banana-2", "POST", "/v2/image/generate/nano-banana-2", "imagesGenerate_post_generate_image_v2_nano_banana2", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.nano-banana-2-1", "POST", "/v2/image/generate/nano-banana-2-1", "imagesGenerate_post_generate_image_v2_nano_banana21", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.nano-banana-pro", "POST", "/v2/image/generate/nano-banana-pro", "imagesGenerate_post_generate_image_v2_nano_banana_pro", "multipart", "prompt,images,aspect_ratio,resolution_tier,num_images,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.p-image-ideogram", "POST", "/v2/image/generate/p-image-ideogram", "imagesGenerate_post_generate_image_v2_p_image_ideogram", "json", "prompt,magic_prompt,seed,num_images,quality,resolution,aspect_ratio,custom_width,custom_height,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.generate.z-image", "POST", "/v2/image/generate/z-image", "imagesGenerate_post_generate_image_v2_z_image", "json", "prompt,resolution,num_inference_steps,seed,num_images,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.remix.auto", "POST", "/v2/image/remix/auto", "autoModel_post_tool_remix", "multipart", "prompt,image,image_weight,negative_prompt,resolution,aspect_ratio,resolution_tier,magic_prompt,seed,style_preset,color_palette,style_codes,style_type,num_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.remix.ideogram-4", "POST", "/v2/image/remix/ideogram-4", "imagesRemix_post_remix_image_v2_ideogram_v4", "multipart", "prompt,image,image_weight,num_images,resolution,rendering_speed,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.remix.ideogram-3", "POST", "/v2/image/remix/ideogram-3", "imagesRemix_post_remix_image_v2_ideogram_v3", "multipart", "prompt,image,image_weight,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_type,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.remix.ideogram-3-character", "POST", "/v2/image/remix/ideogram-3-character", "imagesRemix_post_remix_image_v2_ideogram_v3_character", "multipart", "prompt,image,image_weight,negative_prompt,character_reference_images,character_reference_mask,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,style_type,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.remix.ideogram-3-custom-model", "POST", "/v2/image/remix/ideogram-3-custom-model", "imagesRemix_post_remix_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,image,image_weight,negative_prompt,seed,resolution,aspect_ratio,rendering_speed,magic_prompt,num_images,color_palette,style_codes,style_preset,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.inpaint.ideogram-3", "POST", "/v2/image/inpaint/ideogram-3", "imagesInpaint_post_inpaint_image_v2_ideogram_v3", "multipart", "prompt,image,mask,magic_prompt,num_images,seed,rendering_speed,style_type,style_preset,style_codes,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.inpaint.ideogram-3-character", "POST", "/v2/image/inpaint/ideogram-3-character", "imagesInpaint_post_inpaint_image_v2_ideogram_v3_character", "multipart", "prompt,image,mask,character_reference_images,character_reference_mask,magic_prompt,num_images,seed,rendering_speed,style_type,style_codes,style_reference_images,enable_copyright_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.inpaint.ideogram-3-custom-model", "POST", "/v2/image/inpaint/ideogram-3-custom-model", "imagesInpaint_post_inpaint_image_v2_ideogram_v3_custom_model", "multipart", "prompt,custom_model_uri,image,mask,magic_prompt,num_images,seed,rendering_speed,style_preset,style_codes,style_reference_images,enable_copyright_detection,async", "dry_run", "paid"},
|
||||
{"image.reframe.ideogram-3", "POST", "/v2/image/reframe/ideogram-3", "reframe_ideogramV3", "multipart", "image,resolution,num_images,seed,rendering_speed,style_preset,color_palette,style_codes,style_reference_images,async", "dry_run", "paid"},
|
||||
{"image.reframe.nano-banana-2", "POST", "/v2/image/reframe/nano-banana-2", "reframe_nanoBanana2", "multipart", "image,aspect_ratio,private,num_images,seed", "dry_run", "paid"},
|
||||
{"image.upscale.auto", "POST", "/v2/image/upscale/auto", "tools_post_upscale", "multipart", "image,upscale_factor,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.upscale.nano-banana-pro", "POST", "/v2/image/upscale/nano-banana-pro", "imagesUpscale_post_upscale_image_nano_banana_pro", "multipart", "image,prompt,resolution_tier,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.upscale.topaz-bloom-2", "POST", "/v2/image/upscale/topaz-bloom-2", "imagesUpscale_post_upscale_image_topaz_bloom2", "multipart", "image,upscale_factor,prompt,creativity,color_preservation,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.upscale.topaz-redefine", "POST", "/v2/image/upscale/topaz-redefine", "imagesUpscale_post_upscale_image_topaz_redefine", "multipart", "image,upscale_factor,prompt,creativity,autoprompt,texture,detail,detail_strength,sharpen,denoise,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.upscale.topaz-standard-2", "POST", "/v2/image/upscale/topaz-standard-2", "imagesUpscale_post_upscale_image_topaz_standard_v2", "multipart", "image,upscale_factor,strength,sharpen,denoise,fix_compression,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.upscale.topaz-text-refine", "POST", "/v2/image/upscale/topaz-text-refine", "imagesUpscale_post_upscale_image_topaz_text_refine", "multipart", "image,upscale_factor,strength,sharpen,denoise,fix_compression,face_enhancement,face_enhancement_strength,face_enhancement_creativity,subject_detection,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.upscale.topaz-wonder-3-5", "POST", "/v2/image/upscale/topaz-wonder-3-5", "imagesUpscale_post_upscale_image_topaz_wonder35", "multipart", "image,upscale_factor,enhancement_strength,seed,async,webhook_url", "dry_run", "paid"},
|
||||
{"image.replace-background.ideogram-3", "POST", "/v2/image/replace-background/ideogram-3", "tool_replaceBackgroundIdeogramV3", "multipart", "image,prompt,rendering_speed,num_images,private,webhook_url", "dry_run", "paid"},
|
||||
{"image.replace-background.gpt-image-2", "POST", "/v2/image/replace-background/gpt-image-2", "tool_replaceBackground", "multipart", "image,prompt,quality,num_images,private,webhook_url", "dry_run", "paid"},
|
||||
{"image.remove-background.ideogram-1", "POST", "/v2/image/remove-background/ideogram-1", "tool_removeBackground", "multipart", "image,private,async", "dry_run", "paid"},
|
||||
{"image.remove-object.ideogram-1", "POST", "/v2/image/remove-object/ideogram-1", "objectRemoval_post_remove_object_from_v2_assets", "multipart", "image,mask,seed", "dry_run", "paid"},
|
||||
{"image.describe.ideogram-4", "POST", "/v2/image/describe/ideogram-4", "imageDescribe_post_describe_image_ideogram_v4", "multipart", "image,include_bbox,include_style_descriptions,include_tags", "", "paid"},
|
||||
{"image.describe.ideogram-3", "POST", "/v2/image/describe/ideogram-3", "imageDescribe_post_describe_image_ideogram_v3", "multipart", "image", "", "paid"},
|
||||
{"design.layerize.ideogram-3", "POST", "/v2/design/layerize/ideogram-3", "design_post_layerize_design_ideogram_v3", "multipart", "image,prompt,seed,font_candidate_files,async,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.kling-3-standard-image-to-video", "POST", "/v2/video/generate/kling-3-standard-image-to-video", "videoGenerate_post_generate_video_kling_v3_standard_image_to_video", "multipart", "image,end_image,prompt,negative_prompt,duration,generate_audio,shot_type,cfg_scale,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.kling-3-standard-text-to-video", "POST", "/v2/video/generate/kling-3-standard-text-to-video", "videoGenerate_post_generate_video_kling_v3_standard_text_to_video", "json", "prompt,negative_prompt,aspect_ratio,duration,generate_audio,shot_type,cfg_scale,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.minimax-h3-image-to-video", "POST", "/v2/video/generate/minimax-h3-image-to-video", "videoGenerate_post_generate_video_minimax_h3_image_to_video", "multipart", "image,end_image,prompt,resolution,duration,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.minimax-h3-reference-to-video", "POST", "/v2/video/generate/minimax-h3-reference-to-video", "videoEdit_post_edit_video_minimax_h3_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,prompt_expansion_mode,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.minimax-h3-text-to-video", "POST", "/v2/video/generate/minimax-h3-text-to-video", "videoGenerate_post_generate_video_minimax_h3_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.seedance-2-5-image-to-video", "POST", "/v2/video/generate/seedance-2-5-image-to-video", "videoGenerate_post_generate_video_seed_dance25_image_to_video", "multipart", "image,end_image,prompt,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.seedance-2-5-reference-to-video", "POST", "/v2/video/generate/seedance-2-5-reference-to-video", "videoGenerate_post_generate_video_seed_dance25_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.seedance-2-5-text-to-video", "POST", "/v2/video/generate/seedance-2-5-text-to-video", "videoGenerate_post_generate_video_seed_dance25_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.seedance-2-image-to-video", "POST", "/v2/video/generate/seedance-2-image-to-video", "videoGenerate_post_generate_video_seed_dance2_image_to_video", "multipart", "image,end_image,prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.seedance-2-reference-to-video", "POST", "/v2/video/generate/seedance-2-reference-to-video", "videoGenerate_post_generate_video_seed_dance2_reference_to_video", "multipart", "prompt,reference_images,reference_audios,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
|
||||
{"video.generate.seedance-2-text-to-video", "POST", "/v2/video/generate/seedance-2-text-to-video", "videoGenerate_post_generate_video_seed_dance2_text_to_video", "json", "prompt,aspect_ratio,resolution,duration,generate_audio,webhook_url", "dry_run", "paid"},
|
||||
{"svg.generate.ideogram-vector-1", "POST", "/v2/svg/generate/ideogram-vector-1", "svgGenerate_post_generate_svg", "multipart", "prompt,image_asset_identifiers,images,aspect_ratio,quality,num_images,seed,private,target_collection_id,webhook_url", "dry_run", "paid"},
|
||||
{"svg.vectorize.ideogram-vector-1", "POST", "/v2/svg/vectorize/ideogram-vector-1", "svgGenerate_post_vectorizer", "multipart", "image,image_asset_identifier,mode,private,target_collection_id,webhook_url", "dry_run", "paid"},
|
||||
{"tool.ad-localizer", "POST", "/v2/tool/ad-localizer", "editWorkflow_post_ad_localizer", "multipart", "image,language,exact_copy,prompt,quality,seed,num_images,webhook_url", "dry_run", "paid"},
|
||||
{"tool.ad-resizer", "POST", "/v2/tool/ad-resizer", "editWorkflow_post_ad_resizer", "multipart", "image,resolution,platform,prompt,quality,num_images,webhook_url", "dry_run", "paid"},
|
||||
{"tool.ad-variations", "POST", "/v2/tool/ad-variations", "editWorkflow_post_ad_variations", "multipart", "image,variation_type,prompt,quality,num_images,webhook_url", "", "paid"},
|
||||
{"tool.colorways", "POST", "/v2/tool/colorways", "editWorkflow_post_colorways", "multipart", "image,masks,colors,aspect_ratio,quality,webhook_url", "dry_run", "paid"},
|
||||
{"tool.ghost-mannequin", "POST", "/v2/tool/ghost-mannequin", "editWorkflow_post_ghost_mannequin", "multipart", "garment_images,front_image,back_image,left_image,right_image,top_image,bottom_image,output_reference_asset_identifier,output_reference_image,view,instruction,metadata,aspect_ratio,quality,seed,webhook_url", "dry_run", "paid"},
|
||||
{"tool.material-swap", "POST", "/v2/tool/material-swap", "editWorkflow_post_material_swap", "multipart", "image,masks,materials,aspect_ratio,quality,webhook_url", "dry_run", "paid"},
|
||||
{"tool.model-pose-variants", "POST", "/v2/tool/model-pose-variants", "editWorkflow_post_model_pose_variants", "multipart", "source_image,instruction,aspect_ratio,quality,seed,num_images,webhook_url", "", "paid"},
|
||||
{"tool.sketch-to-render", "POST", "/v2/tool/sketch-to-render", "editWorkflow_post_sketch_to_render", "multipart", "sketch_image,instruction,aspect_ratio,quality,seed,num_images,webhook_url", "", "paid"},
|
||||
{"generations.id", "GET", "/v2/generations/{generation_id}", "generations_get_generation_v2", "none", "", "", "read"},
|
||||
{"account.api-keys", "GET", "/v2/account/api-keys", "account_get_account_api_keys", "none", "", "", "read"},
|
||||
{"account.invoices", "GET", "/v2/account/invoices", "account_get_account_invoices", "none", "", "", "read"},
|
||||
{"account.usage", "GET", "/v2/account/usage", "account_get_account_usage", "none", "", "start_time,end_time,bucket_width,sources", "read"},
|
||||
{"image.replace-background.ideogram-4-5", "POST", "/v2/image/replace-background/ideogram-4-5", "tool_replaceBackgroundIdeogram45", "multipart", "image,background_reference,num_images,private,webhook_url", "dry_run", "paid"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"GenerateImageIdeogramV3Response", "generation_id,data,seed"},
|
||||
{"GenerationResponse", "generation_id,status,created,response_type,usage_cost_usd_micros,failure_reason,data"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& IdeogramCalls::table() {
|
||||
static const CallTable t("ideogram", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t IdeogramCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* IdeogramCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct IdeogramCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
#include "vendors/media/ideogram/ideogram.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs args(const Json& fields, const std::vector<FilePart>& files, bool dry_run) {
|
||||
CallArgs a;
|
||||
a.body = fields;
|
||||
a.files = files;
|
||||
if (dry_run) a.query["dry_run"] = "true";
|
||||
return a;
|
||||
}
|
||||
|
||||
// Image routes take `async` in the body; video routes are always asynchronous.
|
||||
bool is_async(const Json& fields) {
|
||||
const Json* a = fields.find("async");
|
||||
return a && a->is_bool() && a->as_bool();
|
||||
}
|
||||
|
||||
JobStatus status_from(const Json& g, MediaKind kind) {
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(g, "status");
|
||||
if (s.raw_state == "completed") {
|
||||
s.state = JobState::kSucceeded;
|
||||
if (const Json* d = g.find("data")) s.assets = OutputScanner::scan(*d, kind);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "completed without a URL (images that fail the safety check have an empty url)";
|
||||
}
|
||||
} else if (s.raw_state == "failed") {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = AdapterSupport::str(g, "failure_reason");
|
||||
} else {
|
||||
s.state = s.raw_state == "pending" ? JobState::kRunning : JobState::kQueued;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall IdeogramClient::plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
|
||||
bool dry_run) const {
|
||||
return prepare(key, args(fields, files, dry_run));
|
||||
}
|
||||
|
||||
Json IdeogramClient::generate(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
|
||||
bool confirm) {
|
||||
return call_json(key, args(fields, files, false), confirm);
|
||||
}
|
||||
|
||||
Json IdeogramClient::quote(const std::string& key, const Json& fields, const std::vector<FilePart>& files) {
|
||||
PreparedCall c = plan(key, fields, files, true);
|
||||
c.effect = Effect::kRead; // dry_run=true is defined by the spec as not billed
|
||||
return parse_json("ideogram", execute(c, false));
|
||||
}
|
||||
|
||||
Json IdeogramClient::generation(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["generation_id"] = id;
|
||||
return call_json("generations.id", a);
|
||||
}
|
||||
|
||||
Json IdeogramClient::usage(const std::map<std::string, std::string>& query) {
|
||||
CallArgs a;
|
||||
a.query = query;
|
||||
return call_json("account.usage", a);
|
||||
}
|
||||
Json IdeogramClient::api_keys() { return call_json("account.api-keys"); }
|
||||
Json IdeogramClient::invoices() { return call_json("account.invoices"); }
|
||||
|
||||
std::string IdeogramImageGenerator::key_for(const std::string& model) {
|
||||
std::string m = model.empty() ? "ideogram-3" : model;
|
||||
return "image.generate." + m;
|
||||
}
|
||||
|
||||
Json IdeogramImageGenerator::fields(const ImageRequest& r, std::string& key) const {
|
||||
r.validate();
|
||||
key = key_for(r.model);
|
||||
Json f = Json::object();
|
||||
AdapterSupport::put(f, "prompt", r.prompt);
|
||||
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put_seed(f, "seed", r.seed > 2147483647L ? 2147483647L : r.seed);
|
||||
if (r.count > 1) f.set("num_images", Json(r.count));
|
||||
AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio);
|
||||
if (r.width && r.height) f.set("resolution", Json(std::to_string(r.width) + "x" + std::to_string(r.height)));
|
||||
if (!r.source_url.empty())
|
||||
throw llm::ApiError::invalid("ideogram", "source images are files: use IdeogramClient::generate with a FilePart and an edit row");
|
||||
AdapterSupport::merge_extra(f, r.extra);
|
||||
return f;
|
||||
}
|
||||
|
||||
Json IdeogramImageGenerator::plan(const ImageRequest& request) const {
|
||||
std::string key;
|
||||
Json f = fields(request, key);
|
||||
return client_->plan(key, f, {}).summary();
|
||||
}
|
||||
|
||||
JobTicket IdeogramImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
std::string key;
|
||||
Json f = fields(request, key);
|
||||
AdapterSupport::need_confirm(confirm, "ideogram", "generate");
|
||||
Json out = client_->generate(key, f, {}, true);
|
||||
JobTicket t;
|
||||
t.provider = "ideogram";
|
||||
t.model = request.model.empty() ? "ideogram-3" : request.model;
|
||||
t.kind = MediaKind::kImage;
|
||||
t.id = AdapterSupport::str(out, "generation_id");
|
||||
if (is_async(f)) {
|
||||
if (t.id.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no generation_id in the async reply");
|
||||
return t;
|
||||
}
|
||||
if (t.id.empty()) t.id = "immediate";
|
||||
t.immediate = true;
|
||||
if (const Json* d = out.find("data")) t.assets = OutputScanner::scan(*d, MediaKind::kImage);
|
||||
if (t.assets.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no image URL in the reply (safety check?)");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus IdeogramImageGenerator::poll(const JobTicket& ticket) {
|
||||
if (ticket.immediate) {
|
||||
JobStatus s;
|
||||
s.state = JobState::kSucceeded;
|
||||
s.raw_state = "immediate";
|
||||
s.assets = ticket.assets;
|
||||
return s;
|
||||
}
|
||||
return status_from(client_->generation(ticket.id), MediaKind::kImage);
|
||||
}
|
||||
|
||||
Json IdeogramVideoGenerator::fields(const VideoRequest& r, std::string& key) const {
|
||||
r.validate();
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("ideogram", "model is required (for example kling-3-standard-text-to-video)");
|
||||
key = "video.generate." + r.model;
|
||||
Json f = Json::object();
|
||||
AdapterSupport::put(f, "prompt", r.prompt);
|
||||
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put(f, "resolution", r.resolution);
|
||||
AdapterSupport::put_int(f, "duration", r.duration_seconds);
|
||||
if (!r.image_url.empty())
|
||||
throw llm::ApiError::invalid("ideogram", "start frames are files: use IdeogramClient::generate with an image-to-video row");
|
||||
AdapterSupport::merge_extra(f, r.extra);
|
||||
return f;
|
||||
}
|
||||
|
||||
Json IdeogramVideoGenerator::plan(const VideoRequest& request) const {
|
||||
std::string key;
|
||||
Json f = fields(request, key);
|
||||
return client_->plan(key, f, {}).summary();
|
||||
}
|
||||
|
||||
JobTicket IdeogramVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
std::string key;
|
||||
Json f = fields(request, key);
|
||||
AdapterSupport::need_confirm(confirm, "ideogram", "generate");
|
||||
Json out = client_->generate(key, f, {}, true);
|
||||
JobTicket t;
|
||||
t.provider = "ideogram";
|
||||
t.model = request.model;
|
||||
t.kind = MediaKind::kVideo;
|
||||
t.id = AdapterSupport::str(out, "generation_id");
|
||||
if (t.id.empty()) throw llm::ApiError("ideogram", 502, "BAD_RESPONSE", "no generation_id in the reply");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus IdeogramVideoGenerator::poll(const JobTicket& ticket) {
|
||||
return status_from(client_->generation(ticket.id), MediaKind::kVideo);
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
#pragma once
|
||||
|
||||
// Ideogram API v2: image generate / remix / inpaint / reframe / upscale /
|
||||
// background tools, SVG, video models, ad tools, generation polling, account.
|
||||
// Auth is the Api-Key header. Every POST accepts dry_run=true, which the spec
|
||||
// defines as "validated and priced but not run: nothing generated, stored or
|
||||
// billed": quote() uses it, so a price check is free.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/ideogram/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class IdeogramClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.ideogram.ai";
|
||||
static constexpr const char* kKeyEnv = "IDEOGRAM_API_KEY";
|
||||
|
||||
explicit IdeogramClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), IdeogramCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::header("ideogram", kHost, kKeyEnv, "Api-Key"); }
|
||||
static std::shared_ptr<IdeogramClient> from_env() {
|
||||
return std::make_shared<IdeogramClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<IdeogramClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<IdeogramClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// Paid: any POST row of the table (image.*, video.*, svg.*, tool.*).
|
||||
Json generate(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
|
||||
bool dry_run = false) const;
|
||||
// Free price quote (dry_run=true): the only non-read call that needs no confirm.
|
||||
Json quote(const std::string& key, const Json& fields, const std::vector<FilePart>& files);
|
||||
Json generation(const std::string& generation_id);
|
||||
Json usage(const std::map<std::string, std::string>& query);
|
||||
Json api_keys();
|
||||
Json invoices();
|
||||
};
|
||||
|
||||
class IdeogramImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit IdeogramImageGenerator(std::shared_ptr<IdeogramClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "ideogram"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"ideogram-4-5", "ideogram-4", "ideogram-3", "ideogram-3-transparent", "ideogram-2a", "ideogram-2", "auto"};
|
||||
}
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the API
|
||||
int suggested_poll_ms() const override { return 2000; }
|
||||
|
||||
static std::string key_for(const std::string& model);
|
||||
|
||||
private:
|
||||
Json fields(const ImageRequest& request, std::string& key) const;
|
||||
std::shared_ptr<IdeogramClient> client_;
|
||||
};
|
||||
|
||||
class IdeogramVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit IdeogramVideoGenerator(std::shared_ptr<IdeogramClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "ideogram"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"kling-3-standard-text-to-video", "minimax-h3-text-to-video", "seedance-2-text-to-video",
|
||||
"seedance-2-5-text-to-video"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; }
|
||||
int suggested_poll_ms() const override { return 4000; }
|
||||
|
||||
private:
|
||||
Json fields(const VideoRequest& request, std::string& key) const;
|
||||
std::shared_ptr<IdeogramClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
#include "vendors/media/kling/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"text2video.create", "POST", "/v1/videos/text2video", "", "json", "model_name,prompt,negative_prompt,cfg_scale,mode,aspect_ratio,duration,callback_url,external_task_id", "", "paid"},
|
||||
{"text2video.get", "GET", "/v1/videos/text2video/{id}", "", "none", "", "", "read"},
|
||||
{"image2video.create", "POST", "/v1/videos/image2video", "", "json", "model_name,image,image_tail,prompt,negative_prompt,cfg_scale,mode,duration,callback_url,external_task_id", "", "paid"},
|
||||
{"image2video.get", "GET", "/v1/videos/image2video/{id}", "", "none", "", "", "read"},
|
||||
{"images.create", "POST", "/v1/images/generations", "", "json", "model_name,prompt,negative_prompt,image,image_reference,image_fidelity,n,aspect_ratio,callback_url", "", "paid"},
|
||||
{"images.get", "GET", "/v1/images/generations/{id}", "", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& KlingCalls::table() {
|
||||
static const CallTable t("kling", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t KlingCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* KlingCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct KlingCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+156
@@ -0,0 +1,156 @@
|
||||
#include "vendors/media/kling/kling.h"
|
||||
|
||||
#include <ctime>
|
||||
#include <functional>
|
||||
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "security/crypto/Hmac.h"
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
std::shared_ptr<llm::ApiClient> build_client(const std::string& access, const std::string& secret, long now,
|
||||
const std::function<std::unique_ptr<clients::Transport>()>& factory) {
|
||||
std::string token = KlingClient::credentials_present(access, secret) ? KlingJwt::make(access, secret, now) : "";
|
||||
if (factory) return ClientFactory::with(factory(), KlingClient::config(), token);
|
||||
return ClientFactory::real(KlingClient::config(), token);
|
||||
}
|
||||
|
||||
CallArgs with_body(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string KlingJwt::make(const std::string& access_key, const std::string& secret_key, long now) {
|
||||
Json header = Json::object();
|
||||
header.set("alg", Json("HS256"));
|
||||
header.set("typ", Json("JWT"));
|
||||
Json payload = Json::object();
|
||||
payload.set("iss", Json(access_key));
|
||||
payload.set("exp", Json(static_cast<double>(now + kTtlSeconds)));
|
||||
payload.set("nbf", Json(static_cast<double>(now - 5)));
|
||||
std::string signing = security::crypto::Base64::encode_url(header.dump()) + "." +
|
||||
security::crypto::Base64::encode_url(payload.dump());
|
||||
auto mac = security::crypto::Hmac::sha256(secret_key, signing);
|
||||
return signing + "." + security::crypto::Base64::encode_url(reinterpret_cast<const std::uint8_t*>(mac.data()), mac.size());
|
||||
}
|
||||
|
||||
KlingClient::KlingClient(std::string access_key, std::string secret_key,
|
||||
std::function<std::unique_ptr<clients::Transport>()> transport)
|
||||
: MediaService(build_client(access_key, secret_key, static_cast<long>(std::time(nullptr)), transport),
|
||||
KlingCalls::table()),
|
||||
access_(std::move(access_key)),
|
||||
secret_(std::move(secret_key)),
|
||||
transport_(std::move(transport)),
|
||||
minted_at_(static_cast<long>(std::time(nullptr))) {}
|
||||
|
||||
std::shared_ptr<KlingClient> KlingClient::from_env() {
|
||||
return std::make_shared<KlingClient>(ClientFactory::env({kAccessEnv}), ClientFactory::env({kSecretEnv}));
|
||||
}
|
||||
|
||||
void KlingClient::renew() {
|
||||
long now = static_cast<long>(std::time(nullptr));
|
||||
if (!credentials_present(access_, secret_) || now - minted_at_ < KlingJwt::kRenewAfterSeconds) return;
|
||||
client_ = build_client(access_, secret_, now, transport_);
|
||||
minted_at_ = now;
|
||||
}
|
||||
|
||||
void KlingClient::check_envelope(const Json& doc) {
|
||||
const Json* code = doc.is_object() ? doc.find("code") : nullptr;
|
||||
if (!code || !code->is_number() || code->as_int() == 0) return;
|
||||
throw llm::ApiError("kling", 400, std::to_string(code->as_int()), AdapterSupport::str(doc, "message"),
|
||||
AdapterSupport::str(doc, "request_id"));
|
||||
}
|
||||
|
||||
PreparedCall KlingClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
|
||||
|
||||
Json KlingClient::submit(const std::string& key, const Json& body, bool confirm) {
|
||||
renew();
|
||||
Json out = call_json(key, with_body(body), confirm);
|
||||
check_envelope(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
Json KlingClient::get(const std::string& key, const std::string& task_id) {
|
||||
renew();
|
||||
CallArgs a;
|
||||
a.path["id"] = task_id;
|
||||
Json out = call_json(key, a);
|
||||
check_envelope(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
JobState KlingVideoGenerator::state_of(const std::string& status) {
|
||||
if (status == "succeed") return JobState::kSucceeded;
|
||||
if (status == "failed") return JobState::kFailed;
|
||||
if (status == "processing") return JobState::kRunning;
|
||||
return JobState::kQueued; // submitted
|
||||
}
|
||||
|
||||
Json KlingVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
b.set("model_name", Json(r.model.empty() ? "kling-v1-6" : r.model));
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds)));
|
||||
AdapterSupport::put(b, "image", r.image_url);
|
||||
AdapterSupport::put(b, "image_tail", r.end_image_url);
|
||||
if (r.seed >= 0) throw llm::ApiError::invalid("kling", "no seed parameter is documented");
|
||||
if (!r.resolution.empty()) throw llm::ApiError::invalid("kling", "resolution is set through mode (std / pro): pass mode in extra");
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json KlingVideoGenerator::plan(const VideoRequest& request) const {
|
||||
return client_->plan(request.image_url.empty() ? "text2video.create" : "image2video.create", body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket KlingVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "kling", "generate");
|
||||
const bool image = !request.image_url.empty();
|
||||
Json out = client_->submit(image ? "image2video.create" : "text2video.create", b, true);
|
||||
const Json* data = out.find("data");
|
||||
JobTicket t;
|
||||
t.provider = "kling";
|
||||
t.model = b.find("model_name")->as_string();
|
||||
t.kind = MediaKind::kVideo;
|
||||
t.id = data ? AdapterSupport::str(*data, "task_id") : "";
|
||||
if (t.id.empty()) throw llm::ApiError("kling", 502, "BAD_RESPONSE", "no data.task_id in the reply");
|
||||
t.meta = Json::object();
|
||||
t.meta.set("route", Json(image ? "image2video" : "text2video"));
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus KlingVideoGenerator::poll(const JobTicket& ticket) {
|
||||
std::string route = AdapterSupport::str(ticket.meta, "route");
|
||||
if (route != "image2video" && route != "text2video") throw llm::ApiError::invalid("kling", "ticket has no route");
|
||||
Json out = client_->get(route + ".get", ticket.id);
|
||||
const Json* data = out.find("data");
|
||||
JobStatus s;
|
||||
if (!data) {
|
||||
s.state = JobState::kQueued;
|
||||
return s;
|
||||
}
|
||||
s.raw_state = AdapterSupport::str(*data, "task_status");
|
||||
s.state = state_of(s.raw_state);
|
||||
if (s.state == JobState::kFailed) s.error = AdapterSupport::str(*data, "task_status_msg");
|
||||
if (s.state == JobState::kSucceeded) {
|
||||
if (const Json* r = data->find("task_result")) s.assets = OutputScanner::scan(*r, MediaKind::kVideo);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "succeed without videos";
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+80
@@ -0,0 +1,80 @@
|
||||
#pragma once
|
||||
|
||||
// Kling AI (Kuaishou) API. Kling publishes its developer docs as a JavaScript
|
||||
// app with no machine-readable spec, so this adapter is built from community
|
||||
// sources and live probes only: see SPEC "Unverified". Auth is a short-lived
|
||||
// HS256 JWT built from KLING_ACCESS_KEY (iss) and KLING_SECRET_KEY. Hosts
|
||||
// api-singapore.klingai.com and api.klingai.com both answered the documented
|
||||
// path with `{"code":1001,"message":"Authorization is empty"}` unauthenticated.
|
||||
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/kling/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class KlingJwt {
|
||||
public:
|
||||
static constexpr long kTtlSeconds = 1800; // token lifetime
|
||||
static constexpr long kRenewAfterSeconds = 1500; // the client rebuilds before this
|
||||
// header {"alg":"HS256","typ":"JWT"}, payload {iss, exp, nbf}; base64url, no padding.
|
||||
static std::string make(const std::string& access_key, const std::string& secret_key, long now_seconds);
|
||||
};
|
||||
|
||||
class KlingClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api-singapore.klingai.com";
|
||||
static constexpr const char* kAccessEnv = "KLING_ACCESS_KEY";
|
||||
static constexpr const char* kSecretEnv = "KLING_SECRET_KEY";
|
||||
|
||||
// The token is fixed per ApiClient, so a KlingClient holds a factory and
|
||||
// renews the ApiClient when the token nears expiry.
|
||||
KlingClient(std::string access_key, std::string secret_key,
|
||||
std::function<std::unique_ptr<clients::Transport>()> transport = nullptr);
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("kling", kHost, kAccessEnv); }
|
||||
static std::shared_ptr<KlingClient> from_env();
|
||||
static bool credentials_present(const std::string& access, const std::string& secret) {
|
||||
return !access.empty() && !secret.empty();
|
||||
}
|
||||
|
||||
// Paid submits: text2video.create, image2video.create, images.create.
|
||||
Json submit(const std::string& key, const Json& body, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& body) const;
|
||||
Json get(const std::string& key, const std::string& task_id); // *.get rows
|
||||
// Throws ApiError when the envelope's code is not 0.
|
||||
static void check_envelope(const Json& doc);
|
||||
|
||||
private:
|
||||
void renew();
|
||||
std::string access_, secret_;
|
||||
std::function<std::unique_ptr<clients::Transport>()> transport_;
|
||||
long minted_at_ = 0;
|
||||
};
|
||||
|
||||
class KlingVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit KlingVideoGenerator(std::shared_ptr<KlingClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "kling"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"kling-v1", "kling-v1-6", "kling-v2-master", "kling-v2-1", "kling-v2-5-turbo"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // not documented
|
||||
int suggested_poll_ms() const override { return 6000; }
|
||||
|
||||
// submitted / processing / succeed / failed (community sources).
|
||||
static JobState state_of(const std::string& status);
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<KlingClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+30
@@ -0,0 +1,30 @@
|
||||
#include "vendors/media/leonardo/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"v2.generations.create", "POST", "/v2/generations", "createGeneration", "json", "model,parameters,public", "", "paid"},
|
||||
{"v2.generations.sync", "POST", "/v2/generationssync", "createSyncGeneration", "json", "model,parameters,public", "", "paid"},
|
||||
{"v2.models", "GET", "/v2/models", "getModels", "none", "", "", "read"},
|
||||
{"v1.generations.create", "POST", "/v1/generations", "", "json", "prompt,modelId,width,height,num_images,alchemy,ultra,contrast,styleUUID,negative_prompt,seed,public", "", "paid"},
|
||||
{"v1.generations.get", "GET", "/v1/generations/{id}", "getGenerationById", "none", "", "", "read"},
|
||||
{"v1.me", "GET", "/v1/me", "getUserSelf", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& LeonardoCalls::table() {
|
||||
static const CallTable t("leonardo", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t LeonardoCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* LeonardoCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct LeonardoCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
#include "vendors/media/leonardo/leonardo.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs with_body(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall LeonardoClient::plan(const Json& body) const { return prepare("v2.generations.create", with_body(body)); }
|
||||
Json LeonardoClient::create(const Json& body, bool confirm) {
|
||||
return call_json("v2.generations.create", with_body(body), confirm);
|
||||
}
|
||||
Json LeonardoClient::create_sync(const Json& body, bool confirm) {
|
||||
return call_json("v2.generations.sync", with_body(body), confirm);
|
||||
}
|
||||
Json LeonardoClient::models() { return call_json("v2.models"); }
|
||||
Json LeonardoClient::generation(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["id"] = id;
|
||||
return call_json("v1.generations.get", a);
|
||||
}
|
||||
Json LeonardoClient::me() { return call_json("v1.me"); }
|
||||
|
||||
Json LeonardoJobs::body(const std::string& model, const Json& parameters, bool is_public) {
|
||||
Json b = Json::object();
|
||||
b.set("model", Json(model));
|
||||
b.set("parameters", parameters);
|
||||
b.set("public", Json(is_public)); // never publish to the community feed by default
|
||||
return b;
|
||||
}
|
||||
|
||||
JobTicket LeonardoJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) {
|
||||
JobTicket t;
|
||||
t.provider = "leonardo";
|
||||
t.model = model;
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(reply, "generationId");
|
||||
if (t.id.empty()) throw llm::ApiError("leonardo", 502, "BAD_RESPONSE", "no generationId in the reply");
|
||||
t.meta = Json::object();
|
||||
AdapterSupport::put(t.meta, "apiCreditCost", AdapterSupport::str(reply, "apiCreditCost"));
|
||||
return t;
|
||||
}
|
||||
|
||||
// v1 GET /generations/{id}: {"generations_by_pk": {"status": "PENDING|COMPLETE|FAILED",
|
||||
// "generated_images": [{"url": ...}], ...}} (status words from the FAQ; shape unverified).
|
||||
JobStatus LeonardoJobs::status_of(const Json& doc, MediaKind kind) {
|
||||
const Json* g = doc.find("generations_by_pk");
|
||||
const Json& gen = g && g->is_object() ? *g : doc;
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(gen, "status");
|
||||
if (s.raw_state == "COMPLETE") {
|
||||
s.state = JobState::kSucceeded;
|
||||
for (const char* key : {"generated_images", "generated_videos", "generated_assets"})
|
||||
if (const Json* a = gen.find(key)) {
|
||||
auto found = OutputScanner::scan(*a, kind);
|
||||
s.assets.insert(s.assets.end(), found.begin(), found.end());
|
||||
}
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "COMPLETE without generated images";
|
||||
}
|
||||
} else if (s.raw_state == "FAILED") {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "FAILED";
|
||||
} else {
|
||||
s.state = JobState::kRunning; // PENDING
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
JobStatus LeonardoJobs::poll(LeonardoClient& client, const JobTicket& ticket) {
|
||||
return status_of(client.generation(ticket.id), ticket.kind);
|
||||
}
|
||||
|
||||
Json LeonardoImageGenerator::body(const ImageRequest& r) const {
|
||||
r.validate();
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("leonardo", "model is required (see LeonardoClient::models)");
|
||||
Json p = Json::object();
|
||||
AdapterSupport::put(p, "prompt", r.prompt);
|
||||
AdapterSupport::put_int(p, "width", r.width);
|
||||
AdapterSupport::put_int(p, "height", r.height);
|
||||
p.set("quantity", Json(r.count));
|
||||
AdapterSupport::put_seed(p, "seed", r.seed);
|
||||
if (!r.aspect_ratio.empty() || !r.negative_prompt.empty() || !r.source_url.empty())
|
||||
throw llm::ApiError::invalid("leonardo", "aspect_ratio / negative_prompt / source images are model-specific: pass them in extra (parameters)");
|
||||
AdapterSupport::merge_extra(p, r.extra);
|
||||
return LeonardoJobs::body(r.model, p, false);
|
||||
}
|
||||
|
||||
Json LeonardoImageGenerator::plan(const ImageRequest& request) const { return client_->plan(body(request)).summary(); }
|
||||
|
||||
JobTicket LeonardoImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "leonardo", "generate");
|
||||
return LeonardoJobs::ticket_from(client_->create(b, true), request.model, MediaKind::kImage);
|
||||
}
|
||||
|
||||
Json LeonardoVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("leonardo", "model is required (see LeonardoClient::models)");
|
||||
Json p = Json::object();
|
||||
AdapterSupport::put(p, "prompt", r.prompt);
|
||||
AdapterSupport::put_int(p, "duration", r.duration_seconds);
|
||||
AdapterSupport::put(p, "resolution", r.resolution);
|
||||
AdapterSupport::put_seed(p, "seed", r.seed);
|
||||
if (!r.aspect_ratio.empty() || !r.negative_prompt.empty() || !r.image_url.empty() || !r.end_image_url.empty())
|
||||
throw llm::ApiError::invalid("leonardo", "frames / aspect_ratio / negative_prompt are model-specific: pass them in extra (parameters)");
|
||||
AdapterSupport::merge_extra(p, r.extra);
|
||||
return LeonardoJobs::body(r.model, p, false);
|
||||
}
|
||||
|
||||
Json LeonardoVideoGenerator::plan(const VideoRequest& request) const { return client_->plan(body(request)).summary(); }
|
||||
|
||||
JobTicket LeonardoVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "leonardo", "generate");
|
||||
return LeonardoJobs::ticket_from(client_->create(b, true), request.model, MediaKind::kVideo);
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
#pragma once
|
||||
|
||||
// Leonardo.Ai production API. v2 (https://cloud.leonardo.ai/api/rest/v2):
|
||||
// POST /generations (async, any image / video / audio / 3D model by id),
|
||||
// POST /generationssync, GET /models. Polling uses the documented v1
|
||||
// GET /generations/{id} (the v1 reference pages are gone from the docs; the
|
||||
// shape is described in prose only, see SPEC Unverified). Bearer auth with
|
||||
// LEONARDO_API_KEY.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/leonardo/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class LeonardoClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://cloud.leonardo.ai/api/rest";
|
||||
static constexpr const char* kKeyEnv = "LEONARDO_API_KEY";
|
||||
|
||||
explicit LeonardoClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), LeonardoCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("leonardo", kHost, kKeyEnv); }
|
||||
static std::shared_ptr<LeonardoClient> from_env() {
|
||||
return std::make_shared<LeonardoClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<LeonardoClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<LeonardoClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// body = {"model": "<id>", "parameters": {...}, "public": false}
|
||||
Json create(const Json& body, bool confirm);
|
||||
PreparedCall plan(const Json& body) const;
|
||||
Json create_sync(const Json& body, bool confirm); // up to ~27 s, finished result in the reply
|
||||
Json models(); // v2, parameter specs per model
|
||||
Json generation(const std::string& id); // v1
|
||||
Json me(); // v1, account and token balance
|
||||
};
|
||||
|
||||
// Shared by the image and video adapters (the API does not tell them apart).
|
||||
class LeonardoJobs {
|
||||
public:
|
||||
static Json body(const std::string& model, const Json& parameters, bool is_public);
|
||||
static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind);
|
||||
static JobStatus poll(LeonardoClient& client, const JobTicket& ticket);
|
||||
static JobStatus status_of(const Json& generation, MediaKind kind);
|
||||
};
|
||||
|
||||
class LeonardoImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit LeonardoImageGenerator(std::shared_ptr<LeonardoClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "leonardo"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"flux-pro-2.0", "flux-kontext-pro", "phoenix-v1.0", "lucid-origin", "ideogram-v3.0", "gpt-image-2",
|
||||
"nano-banana-2", "seedream-4.5"};
|
||||
}
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override { return LeonardoJobs::poll(*client_, ticket); }
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the v2 spec
|
||||
int suggested_poll_ms() const override { return 3000; }
|
||||
|
||||
private:
|
||||
Json body(const ImageRequest& request) const;
|
||||
std::shared_ptr<LeonardoClient> client_;
|
||||
};
|
||||
|
||||
class LeonardoVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit LeonardoVideoGenerator(std::shared_ptr<LeonardoClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "leonardo"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"motion_2.0", "veo-3.1-fast-generate-001", "kling-3.0", "hailuo-03", "seedance-2.0", "wan-2.7"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override { return LeonardoJobs::poll(*client_, ticket); }
|
||||
bool cancel(const JobTicket&, bool) override { return false; }
|
||||
int suggested_poll_ms() const override { return 5000; }
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<LeonardoClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
#include "vendors/media/luma/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"generations.create_video", "POST", "/generations/video", "", "json", "model,aspect_ratio,callback_url,concepts,duration,generation_type,keyframes,loop,prompt,resolution", "", "paid"},
|
||||
{"generations.create_image", "POST", "/generations/image", "", "json", "model,aspect_ratio,callback_url,character_ref,format,generation_type,image_ref,modify_image_ref,prompt,style_ref,sync,sync_timeout", "", "paid"},
|
||||
{"generations.reframe_image", "POST", "/generations/image/reframe", "", "json", "aspect_ratio,generation_type,media,model,callback_url,format,grid_position_x,grid_position_y,prompt,resized_height,resized_width,x_end,x_start,y_end,y_start", "", "paid"},
|
||||
{"generations.reframe_video", "POST", "/generations/video/reframe", "", "json", "aspect_ratio,generation_type,media,model,callback_url,first_frame,grid_position_x,grid_position_y,prompt,resized_height,resized_width,x_end,x_start,y_end,y_start", "", "paid"},
|
||||
{"generations.modify_video", "POST", "/generations/video/modify", "", "json", "generation_type,media,mode,model,callback_url,first_frame,prompt", "", "paid"},
|
||||
{"generations.upscale", "POST", "/generations/{id}/upscale", "", "json", "callback_url,generation_type,resolution", "", "paid"},
|
||||
{"generations.audio", "POST", "/generations/{id}/audio", "", "json", "callback_url,generation_type,negative_prompt,prompt", "", "paid"},
|
||||
{"generations.get", "GET", "/generations/{id}", "", "none", "", "", "read"},
|
||||
{"generations.list", "GET", "/generations", "", "none", "", "limit,offset", "read"},
|
||||
{"generations.delete", "DELETE", "/generations/{id}", "", "none", "", "", "delete"},
|
||||
{"concepts.list", "GET", "/generations/concepts/list", "", "none", "", "", "read"},
|
||||
{"credits.get", "GET", "/credits", "", "none", "", "", "read"},
|
||||
{"ping", "GET", "/ping", "", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"Generation", "id,state,failure_reason,created_at,assets,generation_type,model,request,image,video,progress_video"},
|
||||
{"CreditGetResponse", "credit_balance"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& LumaCalls::table() {
|
||||
static const CallTable t("luma", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t LumaCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* LumaCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct LumaCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+174
@@ -0,0 +1,174 @@
|
||||
#include "vendors/media/luma/luma.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs with_body(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs by_id(const std::string& id, const Json& body = Json()) {
|
||||
CallArgs a;
|
||||
a.path["id"] = id;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall LumaClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
|
||||
Json LumaClient::create(const std::string& key, const Json& body, bool confirm) {
|
||||
return call_json(key, with_body(body), confirm);
|
||||
}
|
||||
Json LumaClient::upscale(const std::string& id, const Json& body, bool confirm) {
|
||||
return call_json("generations.upscale", by_id(id, body), confirm);
|
||||
}
|
||||
Json LumaClient::add_audio(const std::string& id, const Json& body, bool confirm) {
|
||||
return call_json("generations.audio", by_id(id, body), confirm);
|
||||
}
|
||||
Json LumaClient::generation(const std::string& id) { return call_json("generations.get", by_id(id)); }
|
||||
|
||||
Json LumaClient::list(int limit, int offset) {
|
||||
CallArgs a;
|
||||
if (limit > 0) a.query["limit"] = std::to_string(limit);
|
||||
if (offset > 0) a.query["offset"] = std::to_string(offset);
|
||||
return call_json("generations.list", a);
|
||||
}
|
||||
|
||||
Json LumaClient::remove(const std::string& id, bool confirm) { return call_json("generations.delete", by_id(id), confirm); }
|
||||
Json LumaClient::credits() { return call_json("credits.get"); }
|
||||
Json LumaClient::concepts() { return call_json("concepts.list"); }
|
||||
Json LumaClient::ping() { return call_json("ping"); }
|
||||
|
||||
JobTicket LumaJobs::ticket_from(const Json& g, const std::string& model, MediaKind kind) {
|
||||
JobTicket t;
|
||||
t.provider = "luma";
|
||||
t.model = model;
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(g, "id");
|
||||
if (t.id.empty()) throw llm::ApiError("luma", 502, "BAD_RESPONSE", "no id in the generation");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus LumaJobs::status_of(const Json& g, MediaKind kind) {
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(g, "state");
|
||||
if (s.raw_state == "completed") {
|
||||
s.state = JobState::kSucceeded;
|
||||
if (const Json* a = g.find("assets")) s.assets = OutputScanner::scan(*a, kind);
|
||||
// progress_video is a preview, not the result
|
||||
std::vector<MediaAsset> keep;
|
||||
for (const MediaAsset& x : s.assets)
|
||||
if (x.url.find("progress") == std::string::npos) keep.push_back(x);
|
||||
if (!keep.empty()) s.assets = keep;
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "completed without assets";
|
||||
}
|
||||
} else if (s.raw_state == "failed") {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = AdapterSupport::str(g, "failure_reason");
|
||||
} else if (s.raw_state == "dreaming") {
|
||||
s.state = JobState::kRunning;
|
||||
} else {
|
||||
s.state = JobState::kQueued;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
JobStatus LumaJobs::poll(LumaClient& client, const JobTicket& ticket) {
|
||||
return status_of(client.generation(ticket.id), ticket.kind);
|
||||
}
|
||||
|
||||
Json LumaVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
b.set("model", Json(r.model.empty() ? "ray-2" : r.model));
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put(b, "resolution", r.resolution);
|
||||
if (r.duration_seconds > 0) b.set("duration", Json(std::to_string(r.duration_seconds) + "s"));
|
||||
if (!r.image_url.empty() || !r.end_image_url.empty()) {
|
||||
Json kf = Json::object();
|
||||
if (!r.image_url.empty()) {
|
||||
Json f0 = Json::object();
|
||||
f0.set("type", Json("image"));
|
||||
f0.set("url", Json(r.image_url));
|
||||
kf.set("frame0", f0);
|
||||
}
|
||||
if (!r.end_image_url.empty()) {
|
||||
Json f1 = Json::object();
|
||||
f1.set("type", Json("image"));
|
||||
f1.set("url", Json(r.end_image_url));
|
||||
kf.set("frame1", f1);
|
||||
}
|
||||
b.set("keyframes", kf);
|
||||
}
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json LumaVideoGenerator::plan(const VideoRequest& request) const {
|
||||
return client_->plan("generations.create_video", body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket LumaVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "luma", "generate");
|
||||
return LumaJobs::ticket_from(client_->create("generations.create_video", b, true),
|
||||
b.find("model")->as_string(), MediaKind::kVideo);
|
||||
}
|
||||
|
||||
JobStatus LumaVideoGenerator::poll(const JobTicket& ticket) { return LumaJobs::poll(*client_, ticket); }
|
||||
|
||||
bool LumaVideoGenerator::cancel(const JobTicket& ticket, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "luma", "delete generation");
|
||||
client_->remove(ticket.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
Json LumaImageGenerator::body(const ImageRequest& r) const {
|
||||
r.validate();
|
||||
if (r.count != 1) throw llm::ApiError::invalid("luma", "one image per call (count must be 1)");
|
||||
if (r.width || r.height) throw llm::ApiError::invalid("luma", "use aspect_ratio, not width / height");
|
||||
Json b = Json::object();
|
||||
b.set("model", Json(r.model.empty() ? "photon-1" : r.model));
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put(b, "aspect_ratio", r.aspect_ratio);
|
||||
if (!r.source_url.empty()) {
|
||||
Json ref = Json::array();
|
||||
Json one = Json::object();
|
||||
one.set("url", Json(r.source_url));
|
||||
ref.push(one);
|
||||
b.set("image_ref", ref);
|
||||
}
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json LumaImageGenerator::plan(const ImageRequest& request) const {
|
||||
return client_->plan("generations.create_image", body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket LumaImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "luma", "generate");
|
||||
return LumaJobs::ticket_from(client_->create("generations.create_image", b, true),
|
||||
b.find("model")->as_string(), MediaKind::kImage);
|
||||
}
|
||||
|
||||
JobStatus LumaImageGenerator::poll(const JobTicket& ticket) { return LumaJobs::poll(*client_, ticket); }
|
||||
|
||||
bool LumaImageGenerator::cancel(const JobTicket& ticket, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "luma", "delete generation");
|
||||
client_->remove(ticket.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+82
@@ -0,0 +1,82 @@
|
||||
#pragma once
|
||||
|
||||
// Luma Dream Machine API (https://api.lumalabs.ai/dream-machine/v1): video
|
||||
// (Ray models), images (Photon), reframe, modify, upscale, audio, concepts,
|
||||
// credits. Bearer auth with LUMA_API_KEY. States: queued, dreaming,
|
||||
// completed, failed (from the official SDK types).
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/luma/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class LumaClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.lumalabs.ai/dream-machine/v1";
|
||||
static constexpr const char* kKeyEnv = "LUMA_API_KEY";
|
||||
|
||||
explicit LumaClient(std::shared_ptr<llm::ApiClient> client) : MediaService(std::move(client), LumaCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("luma", kHost, kKeyEnv); }
|
||||
static std::shared_ptr<LumaClient> from_env() {
|
||||
return std::make_shared<LumaClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<LumaClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<LumaClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// key: generations.create_video | create_image | reframe_image | reframe_video | modify_video.
|
||||
Json create(const std::string& key, const Json& body, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& body) const;
|
||||
Json upscale(const std::string& generation_id, const Json& body, bool confirm);
|
||||
Json add_audio(const std::string& generation_id, const Json& body, bool confirm);
|
||||
Json generation(const std::string& id);
|
||||
Json list(int limit, int offset);
|
||||
Json remove(const std::string& id, bool confirm); // destructive, no default
|
||||
Json credits();
|
||||
Json concepts();
|
||||
Json ping();
|
||||
};
|
||||
|
||||
class LumaVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit LumaVideoGenerator(std::shared_ptr<LumaClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "luma"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override { return {"ray-2", "ray-flash-2"}; }
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override; // deletes the generation
|
||||
int suggested_poll_ms() const override { return 4000; }
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<LumaClient> client_;
|
||||
};
|
||||
|
||||
class LumaImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit LumaImageGenerator(std::shared_ptr<LumaClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "luma"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override { return {"photon-1", "photon-flash-1"}; }
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 2000; }
|
||||
|
||||
private:
|
||||
Json body(const ImageRequest& request) const;
|
||||
std::shared_ptr<LumaClient> client_;
|
||||
};
|
||||
|
||||
struct LumaJobs {
|
||||
static JobTicket ticket_from(const Json& generation, const std::string& model, MediaKind kind);
|
||||
static JobStatus poll(LumaClient& client, const JobTicket& ticket);
|
||||
static JobStatus status_of(const Json& generation, MediaKind kind);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+31
@@ -0,0 +1,31 @@
|
||||
#include "vendors/media/minimax/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"video.create", "POST", "/v1/video_generation", "videoGeneration", "json", "model,prompt,first_frame_image,last_frame_image,subject_reference,prompt_optimizer,fast_pretreatment,duration,resolution,callback_url", "", "paid"},
|
||||
{"video.query", "GET", "/v1/query/video_generation", "", "none", "", "task_id", "read"},
|
||||
{"files.retrieve", "GET", "/v1/files/retrieve", "", "none", "", "file_id", "read"},
|
||||
{"v2.video.create", "POST", "/v2/video_generation", "", "json", "model,content,resolution,duration,ratio,extra,callback_url", "", "paid"},
|
||||
{"v2.video.query", "GET", "/v2/query/video_generation/{task_id}", "", "none", "", "", "read"},
|
||||
{"v2.video.list", "GET", "/v2/query/video_generation", "", "none", "", "page_num,page_size", "read"},
|
||||
{"v2.video.delete", "DELETE", "/v2/video_generation/{task_id}", "", "none", "", "", "delete"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& MinimaxCalls::table() {
|
||||
static const CallTable t("minimax", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t MinimaxCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* MinimaxCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct MinimaxCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
#include "vendors/media/minimax/minimax.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs with_body(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs task(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["task_id"] = id;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void MinimaxClient::check_base_resp(const Json& doc) {
|
||||
const Json* br = doc.is_object() ? doc.find("base_resp") : nullptr;
|
||||
if (!br || !br->is_object()) return;
|
||||
const Json* code = br->find("status_code");
|
||||
if (!code || !code->is_number() || code->as_int() == 0) return;
|
||||
throw llm::ApiError("minimax", 400, std::to_string(code->as_int()), AdapterSupport::str(*br, "status_msg"));
|
||||
}
|
||||
|
||||
PreparedCall MinimaxClient::plan_video(const Json& body) const { return prepare("video.create", with_body(body)); }
|
||||
|
||||
Json MinimaxClient::create_video(const Json& body, bool confirm) {
|
||||
Json out = call_json("video.create", with_body(body), confirm);
|
||||
check_base_resp(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
Json MinimaxClient::query_video(const std::string& task_id) {
|
||||
CallArgs a;
|
||||
a.query["task_id"] = task_id;
|
||||
Json out = call_json("video.query", a);
|
||||
check_base_resp(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
Json MinimaxClient::retrieve_file(const std::string& file_id) {
|
||||
CallArgs a;
|
||||
a.query["file_id"] = file_id;
|
||||
Json out = call_json("files.retrieve", a);
|
||||
check_base_resp(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
Json MinimaxClient::create_video_v2(const Json& body, bool confirm) {
|
||||
return call_json("v2.video.create", with_body(body), confirm);
|
||||
}
|
||||
Json MinimaxClient::query_video_v2(const std::string& id) { return call_json("v2.video.query", task(id)); }
|
||||
|
||||
Json MinimaxClient::list_video_v2(int page_num, int page_size) {
|
||||
CallArgs a;
|
||||
if (page_num > 0) a.query["page_num"] = std::to_string(page_num);
|
||||
if (page_size > 0) a.query["page_size"] = std::to_string(page_size);
|
||||
return call_json("v2.video.list", a);
|
||||
}
|
||||
|
||||
Json MinimaxClient::delete_video_v2(const std::string& id, bool confirm) {
|
||||
return call_json("v2.video.delete", task(id), confirm);
|
||||
}
|
||||
|
||||
JobState MinimaxVideoGenerator::state_of(const std::string& status) {
|
||||
if (status == "Success") return JobState::kSucceeded;
|
||||
if (status == "Fail") return JobState::kFailed;
|
||||
if (status == "Processing") return JobState::kRunning;
|
||||
return JobState::kQueued; // Preparing, Queueing
|
||||
}
|
||||
|
||||
Json MinimaxVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
b.set("model", Json(r.model.empty() ? "MiniMax-Hailuo-2.3" : r.model));
|
||||
AdapterSupport::put(b, "prompt", r.prompt);
|
||||
AdapterSupport::put_int(b, "duration", r.duration_seconds);
|
||||
if (!r.resolution.empty()) {
|
||||
std::string res = r.resolution;
|
||||
for (char& c : res) c = static_cast<char>(std::toupper(static_cast<unsigned char>(c)));
|
||||
b.set("resolution", Json(res)); // docs: 768P / 1080P
|
||||
}
|
||||
AdapterSupport::put(b, "first_frame_image", r.image_url);
|
||||
AdapterSupport::put(b, "last_frame_image", r.end_image_url);
|
||||
if (!r.aspect_ratio.empty()) throw llm::ApiError::invalid("minimax", "v1 video_generation has no aspect_ratio");
|
||||
if (r.seed >= 0) throw llm::ApiError::invalid("minimax", "v1 video_generation has no seed");
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json MinimaxVideoGenerator::plan(const VideoRequest& request) const {
|
||||
return client_->plan_video(body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket MinimaxVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "minimax", "generate");
|
||||
Json out = client_->create_video(b, true);
|
||||
JobTicket t;
|
||||
t.provider = "minimax";
|
||||
t.model = b.find("model")->as_string();
|
||||
t.kind = MediaKind::kVideo;
|
||||
t.id = AdapterSupport::str(out, "task_id");
|
||||
if (t.id.empty()) throw llm::ApiError("minimax", 502, "BAD_RESPONSE", "no task_id in the reply");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus MinimaxVideoGenerator::poll(const JobTicket& ticket) {
|
||||
Json q = client_->query_video(ticket.id);
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(q, "status");
|
||||
s.state = state_of(s.raw_state);
|
||||
if (s.state == JobState::kFailed) s.error = "Fail";
|
||||
if (s.state == JobState::kSucceeded) {
|
||||
std::string file_id = AdapterSupport::str(q, "file_id");
|
||||
if (file_id.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "Success without a file_id";
|
||||
return s;
|
||||
}
|
||||
Json f = client_->retrieve_file(file_id);
|
||||
if (const Json* file = f.find("file")) s.assets = OutputScanner::scan(*file, MediaKind::kVideo);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "file retrieved without a download_url";
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+66
@@ -0,0 +1,66 @@
|
||||
#pragma once
|
||||
|
||||
// MiniMax (Hailuo) video API on https://api.minimax.io: v1 video_generation
|
||||
// (text, image, first/last frame, subject reference), the query endpoint, file
|
||||
// retrieval for the download URL, and the v2 content-array endpoints.
|
||||
// Bearer auth with MINIMAX_API_KEY. Errors can arrive as HTTP 200 with
|
||||
// base_resp.status_code != 0: MinimaxClient turns those into ApiError.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/minimax/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class MinimaxClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.minimax.io";
|
||||
static constexpr const char* kKeyEnv = "MINIMAX_API_KEY";
|
||||
|
||||
explicit MinimaxClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), MinimaxCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("minimax", kHost, kKeyEnv); }
|
||||
static std::shared_ptr<MinimaxClient> from_env() {
|
||||
return std::make_shared<MinimaxClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<MinimaxClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<MinimaxClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
Json create_video(const Json& body, bool confirm); // v1, paid
|
||||
PreparedCall plan_video(const Json& body) const;
|
||||
Json query_video(const std::string& task_id); // v1
|
||||
Json retrieve_file(const std::string& file_id); // v1
|
||||
Json create_video_v2(const Json& body, bool confirm);
|
||||
Json query_video_v2(const std::string& task_id);
|
||||
Json list_video_v2(int page_num, int page_size);
|
||||
Json delete_video_v2(const std::string& task_id, bool confirm); // no default
|
||||
|
||||
// Throws ApiError when base_resp.status_code is present and not 0.
|
||||
static void check_base_resp(const Json& doc);
|
||||
};
|
||||
|
||||
class MinimaxVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit MinimaxVideoGenerator(std::shared_ptr<MinimaxClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "minimax"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"MiniMax-Hailuo-2.3", "MiniMax-Hailuo-02", "T2V-01-Director", "T2V-01"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // v1 has no cancel
|
||||
int suggested_poll_ms() const override { return 8000; }
|
||||
|
||||
// Preparing / Queueing / Processing / Success / Fail (v1 query).
|
||||
static JobState state_of(const std::string& status);
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<MinimaxClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+48
@@ -0,0 +1,48 @@
|
||||
#include "vendors/media/recraft/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"images.creativeUpscale", "POST", "/v1/images/creativeUpscale", "CreativeUpscale", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"},
|
||||
{"images.crispUpscale", "POST", "/v1/images/crispUpscale", "CrispUpscale", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"},
|
||||
{"images.enhance", "POST", "/v1/images/enhance", "EnhanceImage", "either", "image_format,refinement,response_format,zdr,image_url,image", "", "paid"},
|
||||
{"images.eraseRegion", "POST", "/v1/images/eraseRegion", "EraseRegion", "either", "image_format,response_format,zdr,image_url,mask_url,image,mask", "", "paid"},
|
||||
{"images.generateBackground", "POST", "/v1/images/generateBackground", "GenerateBackground", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,mask_url,image,mask", "", "paid"},
|
||||
{"images.generations", "POST", "/v1/images/generations", "GenerateImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"},
|
||||
{"images.generations.raster", "POST", "/v1/images/generations/raster", "GenerateImageRaster", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"},
|
||||
{"images.generations.vector", "POST", "/v1/images/generations/vector", "GenerateImageVector", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,size,style,style_id,style_match,text_layout,upscale,zdr,style_reference_urls,style_references", "", "paid"},
|
||||
{"images.imageToImage", "POST", "/v1/images/imageToImage", "ImageToImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,strength,image", "", "paid"},
|
||||
{"images.inpaint", "POST", "/v1/images/inpaint", "InpaintImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,mask_url,image,mask", "", "paid"},
|
||||
{"images.outpaint", "POST", "/v1/images/outpaint", "OutpaintImage", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,expand_bottom,expand_left,expand_right,expand_top,size,zoom_out_percentage,image_url,image", "", "paid"},
|
||||
{"images.removeBackground", "POST", "/v1/images/removeBackground", "RemoveBackground", "either", "image_format,response_format,upscale,zdr,image_url,image", "", "paid"},
|
||||
{"images.replaceBackground", "POST", "/v1/images/replaceBackground", "ReplaceBackground", "either", "controls,image_format,model,n,negative_prompt,prompt,random_seed,response_format,style,style_id,style_match,text_layout,zdr,image_url,image", "", "paid"},
|
||||
{"images.variateImage", "POST", "/v1/images/variateImage", "VariateImage", "either", "image_format,model,n,random_seed,response_format,size,zdr,image_url,image", "", "paid"},
|
||||
{"images.vectorize", "POST", "/v1/images/vectorize", "VectorizeImage", "either", "image_format,response_format,upscale,zdr,color_reduction,limit_num_shapes,max_num_colors,max_num_shapes,return_gradients,shape_stacking,small_shape_filter,strict_color_palette,svg_compression,image_url,image", "", "paid"},
|
||||
{"styles", "GET", "/v1/styles", "ListStyles", "none", "", "", "read"},
|
||||
{"styles.create", "POST", "/v1/styles", "CreateStyle", "either", "image_weights,match,mix_policy,model,palette,private,prompt,source_style_weights,source_styles,style,image_urls,images", "", "paid"},
|
||||
{"styles.id.delete", "DELETE", "/v1/styles/{style_id}", "DeleteStyle", "none", "", "", "delete"},
|
||||
{"styles.id", "GET", "/v1/styles/{style_id}", "GetStyle", "none", "", "", "read"},
|
||||
{"styles.basic", "GET", "/v1/styles/basic", "ListBasicStyles", "none", "", "", "read"},
|
||||
{"users.me", "GET", "/v1/users/me", "GetCurrentUser", "none", "", "", "read"},
|
||||
{"prompts.enhance", "POST", "/v1/prompts/enhance", "EnhancePrompt", "json", "prompt", "", "paid"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"GenerateImageResponse", "created,data,credits,style_id"},
|
||||
{"User", "id,email,name,credits"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& RecraftCalls::table() {
|
||||
static const CallTable t("recraft", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t RecraftCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* RecraftCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct RecraftCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
#include "vendors/media/recraft/recraft.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs args(const Json& fields, const std::vector<FilePart>& files) {
|
||||
CallArgs a;
|
||||
a.body = fields;
|
||||
a.files = files;
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs style_id(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["style_id"] = id;
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall RecraftClient::plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files) const {
|
||||
return prepare(key, args(fields, files));
|
||||
}
|
||||
|
||||
Json RecraftClient::run(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm) {
|
||||
return call_json(key, args(fields, files), confirm);
|
||||
}
|
||||
|
||||
Json RecraftClient::me() { return call_json("users.me"); }
|
||||
Json RecraftClient::styles() { return call_json("styles"); }
|
||||
Json RecraftClient::basic_styles() { return call_json("styles.basic"); }
|
||||
Json RecraftClient::style(const std::string& id) { return call_json("styles.id", style_id(id)); }
|
||||
Json RecraftClient::create_style(const Json& fields, const std::vector<FilePart>& files, bool confirm) {
|
||||
return call_json("styles.create", args(fields, files), confirm);
|
||||
}
|
||||
Json RecraftClient::delete_style(const std::string& id, bool confirm) {
|
||||
return call_json("styles.id.delete", style_id(id), confirm);
|
||||
}
|
||||
|
||||
Json RecraftImageGenerator::fields(const ImageRequest& r) const {
|
||||
r.validate();
|
||||
Json f = Json::object();
|
||||
AdapterSupport::put(f, "prompt", r.prompt);
|
||||
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put(f, "model", r.model);
|
||||
if (r.count > 1) f.set("n", Json(r.count));
|
||||
if (r.width && r.height) f.set("size", Json(std::to_string(r.width) + "x" + std::to_string(r.height)));
|
||||
AdapterSupport::put_seed(f, "random_seed", r.seed);
|
||||
if (!r.aspect_ratio.empty())
|
||||
throw llm::ApiError::invalid("recraft", "Recraft sizes images with size (WxH), not aspect_ratio");
|
||||
if (!r.source_url.empty())
|
||||
throw llm::ApiError::invalid("recraft", "source images use the images.* edit rows: RecraftClient::run with image_url");
|
||||
AdapterSupport::merge_extra(f, r.extra);
|
||||
return f;
|
||||
}
|
||||
|
||||
Json RecraftImageGenerator::plan(const ImageRequest& request) const {
|
||||
return client_->plan("images.generations", fields(request), {}).summary();
|
||||
}
|
||||
|
||||
JobTicket RecraftImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
Json f = fields(request);
|
||||
AdapterSupport::need_confirm(confirm, "recraft", "generate");
|
||||
Json out = client_->run("images.generations", f, {}, true);
|
||||
JobTicket t;
|
||||
t.provider = "recraft";
|
||||
t.model = request.model;
|
||||
t.kind = MediaKind::kImage;
|
||||
t.id = "immediate";
|
||||
t.immediate = true;
|
||||
if (const Json* d = out.find("data"); d && d->is_array()) {
|
||||
for (const Json& item : d->as_array()) {
|
||||
std::string b64 = AdapterSupport::str(item, "b64_json");
|
||||
if (!b64.empty()) {
|
||||
MediaAsset a;
|
||||
a.kind = MediaKind::kImage;
|
||||
a.data_base64 = b64;
|
||||
a.mime = "image/webp"; // Recraft's default raster format is lossless WebP
|
||||
t.assets.push_back(a);
|
||||
}
|
||||
}
|
||||
if (t.assets.empty()) t.assets = OutputScanner::scan(*d, MediaKind::kImage);
|
||||
}
|
||||
if (t.assets.empty()) throw llm::ApiError("recraft", 502, "BAD_RESPONSE", "no image in the reply");
|
||||
t.meta = Json::object();
|
||||
AdapterSupport::put(t.meta, "credits", AdapterSupport::str(out, "credits"));
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus RecraftImageGenerator::poll(const JobTicket& ticket) {
|
||||
JobStatus s;
|
||||
s.state = JobState::kSucceeded;
|
||||
s.raw_state = "immediate";
|
||||
s.assets = ticket.assets;
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
#pragma once
|
||||
|
||||
// Recraft external API: generation (general, raster, vector), image tools
|
||||
// (vectorize, background, upscale, erase, remix, inpaint, outpaint, enhance),
|
||||
// styles, prompt enhancement and the user endpoint. Bearer auth with
|
||||
// RECRAFT_API_TOKEN, base https://external.api.recraft.ai. Every image call
|
||||
// is synchronous: the reply carries the result, so tickets are immediate.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/recraft/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class RecraftClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://external.api.recraft.ai";
|
||||
static constexpr const char* kKeyEnv = "RECRAFT_API_TOKEN";
|
||||
|
||||
explicit RecraftClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), RecraftCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("recraft", kHost, kKeyEnv); }
|
||||
static std::shared_ptr<RecraftClient> from_env() {
|
||||
return std::make_shared<RecraftClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<RecraftClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<RecraftClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// Paid image rows (images.*) and prompts.enhance. JSON bodies take image_url
|
||||
// fields; multipart is used when files are given.
|
||||
Json run(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files) const;
|
||||
Json me(); // credits + identity: the free read
|
||||
Json styles();
|
||||
Json basic_styles();
|
||||
Json style(const std::string& style_id);
|
||||
Json create_style(const Json& fields, const std::vector<FilePart>& files, bool confirm); // $0.005
|
||||
Json delete_style(const std::string& style_id, bool confirm); // destructive
|
||||
};
|
||||
|
||||
class RecraftImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit RecraftImageGenerator(std::shared_ptr<RecraftClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "recraft"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"recraftv4_1", "recraftv4_1_pro", "recraftv4", "recraftv4_pro", "recraftv3", "recraftv3_vector", "recraftv2"};
|
||||
}
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // synchronous API
|
||||
|
||||
private:
|
||||
Json fields(const ImageRequest& request) const;
|
||||
std::shared_ptr<RecraftClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
#include "vendors/media/replicate/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"account.get", "GET", "/account", "account.get", "none", "", "", "read"},
|
||||
{"collections.list", "GET", "/collections", "collections.list", "none", "", "", "read"},
|
||||
{"collections.get", "GET", "/collections/{collection_slug}", "collections.get", "none", "", "", "read"},
|
||||
{"deployments.list", "GET", "/deployments", "deployments.list", "none", "", "", "read"},
|
||||
{"deployments.create", "POST", "/deployments", "deployments.create", "json", "hardware,max_instances,min_instances,model,name,version", "", "write"},
|
||||
{"deployments.delete", "DELETE", "/deployments/{deployment_owner}/{deployment_name}", "deployments.delete", "none", "", "", "delete"},
|
||||
{"deployments.get", "GET", "/deployments/{deployment_owner}/{deployment_name}", "deployments.get", "none", "", "", "read"},
|
||||
{"deployments.update", "PATCH", "/deployments/{deployment_owner}/{deployment_name}", "deployments.update", "json", "hardware,max_instances,min_instances,version", "", "write"},
|
||||
{"deployments.predictions.create", "POST", "/deployments/{deployment_owner}/{deployment_name}/predictions", "deployments.predictions.create", "json", "input,stream,webhook,webhook_events_filter", "", "paid"},
|
||||
{"files.list", "GET", "/files", "files.list", "none", "", "", "read"},
|
||||
{"files.create", "POST", "/files", "files.create", "multipart", "content,filename,metadata,type", "", "write"},
|
||||
{"files.delete", "DELETE", "/files/{file_id}", "files.delete", "none", "", "", "delete"},
|
||||
{"files.get", "GET", "/files/{file_id}", "files.get", "none", "", "", "read"},
|
||||
{"files.download", "GET", "/files/{file_id}/download", "files.download", "none", "", "owner,expiry,signature", "read"},
|
||||
{"hardware.list", "GET", "/hardware", "hardware.list", "none", "", "", "read"},
|
||||
{"models.list", "GET", "/models", "models.list", "none", "", "sort_by,sort_direction", "read"},
|
||||
{"models.create", "POST", "/models", "models.create", "json", "cover_image_url,description,github_url,hardware,license_url,name,owner,paper_url,visibility", "", "write"},
|
||||
{"models.delete", "DELETE", "/models/{model_owner}/{model_name}", "models.delete", "none", "", "", "delete"},
|
||||
{"models.get", "GET", "/models/{model_owner}/{model_name}", "models.get", "none", "", "", "read"},
|
||||
{"models.update", "PATCH", "/models/{model_owner}/{model_name}", "models.update", "json", "description,github_url,license_url,paper_url,readme,weights_url", "", "write"},
|
||||
{"models.examples.list", "GET", "/models/{model_owner}/{model_name}/examples", "models.examples.list", "none", "", "", "read"},
|
||||
{"models.predictions.create", "POST", "/models/{model_owner}/{model_name}/predictions", "models.predictions.create", "json", "input,stream,webhook,webhook_events_filter", "", "paid"},
|
||||
{"models.readme.get", "GET", "/models/{model_owner}/{model_name}/readme", "models.readme.get", "none", "", "", "read"},
|
||||
{"models.versions.list", "GET", "/models/{model_owner}/{model_name}/versions", "models.versions.list", "none", "", "", "read"},
|
||||
{"models.versions.delete", "DELETE", "/models/{model_owner}/{model_name}/versions/{version_id}", "models.versions.delete", "none", "", "", "delete"},
|
||||
{"models.versions.get", "GET", "/models/{model_owner}/{model_name}/versions/{version_id}", "models.versions.get", "none", "", "", "read"},
|
||||
{"trainings.create", "POST", "/models/{model_owner}/{model_name}/versions/{version_id}/trainings", "trainings.create", "json", "destination,input,webhook,webhook_events_filter", "", "paid"},
|
||||
{"predictions.list", "GET", "/predictions", "predictions.list", "none", "", "created_after,created_before,source", "read"},
|
||||
{"predictions.create", "POST", "/predictions", "predictions.create", "json", "input,stream,version,webhook,webhook_events_filter", "", "paid"},
|
||||
{"predictions.get", "GET", "/predictions/{prediction_id}", "predictions.get", "none", "", "", "read"},
|
||||
{"predictions.cancel", "POST", "/predictions/{prediction_id}/cancel", "predictions.cancel", "none", "", "", "write"},
|
||||
{"search", "GET", "/search", "search", "none", "", "query,limit", "read"},
|
||||
{"trainings.list", "GET", "/trainings", "trainings.list", "none", "", "", "read"},
|
||||
{"trainings.get", "GET", "/trainings/{training_id}", "trainings.get", "none", "", "", "read"},
|
||||
{"trainings.cancel", "POST", "/trainings/{training_id}/cancel", "trainings.cancel", "none", "", "", "write"},
|
||||
{"webhooks.default.secret.get", "GET", "/webhooks/default/secret", "webhooks.default.secret.get", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"schemas_prediction_response", "id,model,version,input,output,status,error,logs,metrics,created_at,started_at,completed_at,urls,get,cancel"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& ReplicateCalls::table() {
|
||||
static const CallTable t("replicate", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t ReplicateCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* ReplicateCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct ReplicateCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+278
@@ -0,0 +1,278 @@
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <sstream>
|
||||
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "security/crypto/Hmac.h"
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs args_for(const ReplicateClient::ModelRef& m, const Json& input, const Json& options) {
|
||||
CallArgs a;
|
||||
Json body = Json::object();
|
||||
body.set("input", input.is_null() ? Json::object() : input);
|
||||
if (options.is_object())
|
||||
for (const auto& [k, v] : options.as_object()) body.set(k, v); // webhook, webhook_events_filter, stream
|
||||
if (m.version.empty()) {
|
||||
a.path["model_owner"] = m.owner;
|
||||
a.path["model_name"] = m.name;
|
||||
} else {
|
||||
body.set("version", Json(m.version));
|
||||
}
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs one(const char* name, const std::string& value) {
|
||||
CallArgs a;
|
||||
a.path[name] = value;
|
||||
return a;
|
||||
}
|
||||
|
||||
Json mark_truncated(Json page) {
|
||||
const Json* next = page.is_object() ? page.find("next") : nullptr;
|
||||
if (next && next->is_string() && !next->as_string().empty()) page.set("truncated", Json(true));
|
||||
return page;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ReplicateClient::ModelRef ReplicateClient::ModelRef::parse(const std::string& ref) {
|
||||
ModelRef m;
|
||||
std::string head = ref;
|
||||
std::size_t colon = ref.find(':');
|
||||
if (colon != std::string::npos) {
|
||||
head = ref.substr(0, colon);
|
||||
m.version = ref.substr(colon + 1);
|
||||
}
|
||||
std::size_t slash = head.find('/');
|
||||
if (slash == std::string::npos || head.find('/', slash + 1) != std::string::npos)
|
||||
throw llm::ApiError::invalid("replicate", "model must look like owner/name or owner/name:version");
|
||||
m.owner = head.substr(0, slash);
|
||||
m.name = head.substr(slash + 1);
|
||||
if (!CallTable::valid_segment(m.owner) || !CallTable::valid_segment(m.name) ||
|
||||
(colon != std::string::npos && !CallTable::valid_segment(m.version)))
|
||||
throw llm::ApiError::invalid("replicate", "bad model reference");
|
||||
return m;
|
||||
}
|
||||
|
||||
PreparedCall ReplicateClient::plan_prediction(const ModelRef& m, const Json& input, const Json& options) const {
|
||||
return prepare(m.version.empty() ? "models.predictions.create" : "predictions.create", args_for(m, input, options));
|
||||
}
|
||||
|
||||
Json ReplicateClient::create_prediction(const ModelRef& m, const Json& input, const Json& options, bool confirm) {
|
||||
return call_json(m.version.empty() ? "models.predictions.create" : "predictions.create",
|
||||
args_for(m, input, options), confirm);
|
||||
}
|
||||
|
||||
Json ReplicateClient::create_deployment_prediction(const std::string& owner, const std::string& name,
|
||||
const Json& input, const Json& options, bool confirm) {
|
||||
CallArgs a;
|
||||
a.path["deployment_owner"] = owner;
|
||||
a.path["deployment_name"] = name;
|
||||
Json body = Json::object();
|
||||
body.set("input", input);
|
||||
if (options.is_object())
|
||||
for (const auto& [k, v] : options.as_object()) body.set(k, v);
|
||||
a.body = body;
|
||||
return call_json("deployments.predictions.create", a, confirm);
|
||||
}
|
||||
|
||||
Json ReplicateClient::get_prediction(const std::string& id) { return call_json("predictions.get", one("prediction_id", id)); }
|
||||
Json ReplicateClient::cancel_prediction(const std::string& id, bool confirm) {
|
||||
return call_json("predictions.cancel", one("prediction_id", id), confirm);
|
||||
}
|
||||
Json ReplicateClient::list_predictions() { return mark_truncated(call_json("predictions.list")); }
|
||||
Json ReplicateClient::account() { return call_json("account.get"); }
|
||||
Json ReplicateClient::hardware() { return call_json("hardware.list"); }
|
||||
Json ReplicateClient::models_list() { return mark_truncated(call_json("models.list")); }
|
||||
|
||||
Json ReplicateClient::model_get(const std::string& owner, const std::string& name) {
|
||||
CallArgs a;
|
||||
a.path["model_owner"] = owner;
|
||||
a.path["model_name"] = name;
|
||||
return call_json("models.get", a);
|
||||
}
|
||||
|
||||
Json ReplicateClient::versions_list(const std::string& owner, const std::string& name) {
|
||||
CallArgs a;
|
||||
a.path["model_owner"] = owner;
|
||||
a.path["model_name"] = name;
|
||||
return mark_truncated(call_json("models.versions.list", a));
|
||||
}
|
||||
|
||||
Json ReplicateClient::deployments_list() { return mark_truncated(call_json("deployments.list")); }
|
||||
|
||||
Json ReplicateClient::deployment_get(const std::string& owner, const std::string& name) {
|
||||
CallArgs a;
|
||||
a.path["deployment_owner"] = owner;
|
||||
a.path["deployment_name"] = name;
|
||||
return call_json("deployments.get", a);
|
||||
}
|
||||
|
||||
Json ReplicateClient::files_list() { return mark_truncated(call_json("files.list")); }
|
||||
Json ReplicateClient::file_get(const std::string& id) { return call_json("files.get", one("file_id", id)); }
|
||||
Json ReplicateClient::file_delete(const std::string& id, bool confirm) {
|
||||
return call_json("files.delete", one("file_id", id), confirm);
|
||||
}
|
||||
|
||||
Json ReplicateClient::file_upload(const std::string& filename, const std::string& content_type,
|
||||
const std::string& data, bool confirm) {
|
||||
CallArgs a;
|
||||
a.body = Json::object();
|
||||
a.body.set("filename", Json(filename));
|
||||
a.files.push_back(FilePart{"content", filename, content_type, data});
|
||||
return call_json("files.create", a, confirm);
|
||||
}
|
||||
|
||||
Json ReplicateClient::collections_list() { return mark_truncated(call_json("collections.list")); }
|
||||
|
||||
Json ReplicateClient::search(const std::string& query) {
|
||||
CallArgs a;
|
||||
a.query["query"] = query;
|
||||
return call_json("search", a);
|
||||
}
|
||||
|
||||
Json ReplicateClient::webhook_secret() { return call_json("webhooks.default.secret.get"); }
|
||||
Json ReplicateClient::trainings_list() { return mark_truncated(call_json("trainings.list")); }
|
||||
|
||||
// ---------------------------------------------------------------- webhooks
|
||||
|
||||
std::string ReplicateWebhook::sign(const std::string& secret, const std::string& id, const std::string& timestamp,
|
||||
const std::string& body) {
|
||||
std::string b64 = secret.rfind("whsec_", 0) == 0 ? secret.substr(6) : secret;
|
||||
std::string key;
|
||||
if (b64.empty() || !security::crypto::Base64::decode(b64, key) || key.empty()) return "";
|
||||
if (body.size() > MediaCaps::kMaxWebhookBodyBytes) return "";
|
||||
auto mac = security::crypto::Hmac::sha256(key, id + "." + timestamp + "." + body);
|
||||
return security::crypto::Base64::encode(reinterpret_cast<const std::uint8_t*>(mac.data()), mac.size());
|
||||
}
|
||||
|
||||
bool ReplicateWebhook::verify(const std::string& secret, const std::string& id, const std::string& timestamp,
|
||||
const std::string& signature_header, const std::string& body, long now_seconds) {
|
||||
if (id.empty() || timestamp.empty() || timestamp.size() > 12 || signature_header.size() > 1024) return false;
|
||||
for (char c : timestamp)
|
||||
if (c < '0' || c > '9') return false;
|
||||
long ts = std::strtol(timestamp.c_str(), nullptr, 10);
|
||||
long skew = now_seconds > ts ? now_seconds - ts : ts - now_seconds;
|
||||
if (skew > MediaCaps::kWebhookToleranceSeconds) return false;
|
||||
std::string expected = sign(secret, id, timestamp, body);
|
||||
if (expected.empty()) return false;
|
||||
std::istringstream in(signature_header);
|
||||
std::string part;
|
||||
while (in >> part) {
|
||||
if (part.rfind("v1,", 0) != 0) continue;
|
||||
std::string got = part.substr(3);
|
||||
if (got.size() != expected.size()) continue;
|
||||
unsigned char diff = 0; // constant time over the whole string
|
||||
for (std::size_t i = 0; i < got.size(); ++i) diff |= static_cast<unsigned char>(got[i] ^ expected[i]);
|
||||
if (diff == 0) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- jobs
|
||||
|
||||
JobTicket ReplicateJobs::ticket_from(const Json& p, const std::string& model, MediaKind kind) const {
|
||||
JobTicket t;
|
||||
t.provider = "replicate";
|
||||
t.model = model;
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(p, "id");
|
||||
if (t.id.empty()) throw llm::ApiError("replicate", 502, "BAD_RESPONSE", "no id in the prediction");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus ReplicateJobs::poll(const JobTicket& ticket, MediaKind kind) {
|
||||
Json p = client_->get_prediction(ticket.id);
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(p, "status");
|
||||
if (s.raw_state == "succeeded") {
|
||||
s.state = JobState::kSucceeded;
|
||||
if (const Json* o = p.find("output")) s.assets = OutputScanner::scan(*o, kind);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "succeeded without a downloadable output (non-URL output)";
|
||||
}
|
||||
} else if (s.raw_state == "failed") {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = AdapterSupport::str(p, "error");
|
||||
} else if (s.raw_state == "canceled" || s.raw_state == "aborted") {
|
||||
s.state = JobState::kCancelled;
|
||||
} else if (s.raw_state == "processing") {
|
||||
s.state = JobState::kRunning;
|
||||
} else {
|
||||
s.state = JobState::kQueued;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
bool ReplicateJobs::cancel(const JobTicket& ticket, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "replicate", "cancel");
|
||||
client_->cancel_prediction(ticket.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
Json ReplicateImageGenerator::input(const ImageRequest& r) const {
|
||||
r.validate();
|
||||
Json in = Json::object();
|
||||
AdapterSupport::put(in, "prompt", r.prompt);
|
||||
AdapterSupport::put(in, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put_int(in, "width", r.width);
|
||||
AdapterSupport::put_int(in, "height", r.height);
|
||||
AdapterSupport::put(in, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put_seed(in, "seed", r.seed);
|
||||
if (r.count > 1) in.set("num_outputs", Json(r.count));
|
||||
AdapterSupport::put(in, "image", r.source_url);
|
||||
AdapterSupport::merge_extra(in, r.extra);
|
||||
return in;
|
||||
}
|
||||
|
||||
Json ReplicateImageGenerator::plan(const ImageRequest& r) const {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
|
||||
return jobs_.client().plan_prediction(ReplicateClient::ModelRef::parse(r.model), input(r), Json::object()).summary();
|
||||
}
|
||||
|
||||
JobTicket ReplicateImageGenerator::submit(const ImageRequest& r, bool confirm) {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
|
||||
auto ref = ReplicateClient::ModelRef::parse(r.model);
|
||||
Json in = input(r);
|
||||
AdapterSupport::need_confirm(confirm, "replicate", "create prediction");
|
||||
return jobs_.ticket_from(jobs_.client().create_prediction(ref, in, Json::object(), true), r.model, MediaKind::kImage);
|
||||
}
|
||||
|
||||
Json ReplicateVideoGenerator::input(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json in = Json::object();
|
||||
AdapterSupport::put(in, "prompt", r.prompt);
|
||||
AdapterSupport::put(in, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put_int(in, "duration", r.duration_seconds);
|
||||
AdapterSupport::put(in, "aspect_ratio", r.aspect_ratio);
|
||||
AdapterSupport::put(in, "resolution", r.resolution);
|
||||
AdapterSupport::put_seed(in, "seed", r.seed);
|
||||
AdapterSupport::put(in, "image", r.image_url);
|
||||
AdapterSupport::put(in, "last_frame_image", r.end_image_url);
|
||||
AdapterSupport::merge_extra(in, r.extra);
|
||||
return in;
|
||||
}
|
||||
|
||||
Json ReplicateVideoGenerator::plan(const VideoRequest& r) const {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
|
||||
return jobs_.client().plan_prediction(ReplicateClient::ModelRef::parse(r.model), input(r), Json::object()).summary();
|
||||
}
|
||||
|
||||
JobTicket ReplicateVideoGenerator::submit(const VideoRequest& r, bool confirm) {
|
||||
if (r.model.empty()) throw llm::ApiError::invalid("replicate", "model is required (owner/name)");
|
||||
auto ref = ReplicateClient::ModelRef::parse(r.model);
|
||||
Json in = input(r);
|
||||
AdapterSupport::need_confirm(confirm, "replicate", "create prediction");
|
||||
return jobs_.ticket_from(jobs_.client().create_prediction(ref, in, Json::object(), true), r.model, MediaKind::kVideo);
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
#pragma once
|
||||
|
||||
// Replicate: predictions (official models, versions, deployments), models,
|
||||
// files, account, hardware, collections, trainings and webhook verification.
|
||||
// Bearer auth, base https://api.replicate.com/v1.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/replicate/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class ReplicateClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.replicate.com/v1"; // the spec server url; table paths are relative to it
|
||||
static constexpr const char* kKeyEnv = "REPLICATE_API_TOKEN";
|
||||
|
||||
explicit ReplicateClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), ReplicateCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("replicate", kHost, kKeyEnv); }
|
||||
static std::shared_ptr<ReplicateClient> from_env() {
|
||||
return std::make_shared<ReplicateClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<ReplicateClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<ReplicateClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// "owner/name" (official model) or "owner/name:version" (pinned version).
|
||||
struct ModelRef {
|
||||
std::string owner, name, version;
|
||||
static ModelRef parse(const std::string& ref); // throws ApiError::invalid
|
||||
};
|
||||
|
||||
// Paid. Official model -> models.predictions.create; with version -> predictions.create.
|
||||
Json create_prediction(const ModelRef& model, const Json& input, const Json& options, bool confirm);
|
||||
PreparedCall plan_prediction(const ModelRef& model, const Json& input, const Json& options) const;
|
||||
Json create_deployment_prediction(const std::string& owner, const std::string& name, const Json& input,
|
||||
const Json& options, bool confirm);
|
||||
Json get_prediction(const std::string& id);
|
||||
Json cancel_prediction(const std::string& id, bool confirm); // write, no default
|
||||
// First page only; "truncated" is set when the server has a next page.
|
||||
Json list_predictions();
|
||||
|
||||
Json account();
|
||||
Json hardware();
|
||||
Json models_list();
|
||||
Json model_get(const std::string& owner, const std::string& name);
|
||||
Json versions_list(const std::string& owner, const std::string& name);
|
||||
Json deployments_list();
|
||||
Json deployment_get(const std::string& owner, const std::string& name);
|
||||
Json files_list();
|
||||
Json file_get(const std::string& id);
|
||||
Json file_upload(const std::string& filename, const std::string& content_type, const std::string& data, bool confirm);
|
||||
Json file_delete(const std::string& id, bool confirm); // destructive, no default
|
||||
Json collections_list();
|
||||
Json search(const std::string& query);
|
||||
Json webhook_secret();
|
||||
Json trainings_list();
|
||||
};
|
||||
|
||||
// Standard Webhooks verification (id, timestamp, signature headers).
|
||||
class ReplicateWebhook {
|
||||
public:
|
||||
// secret = the "whsec_..." value from webhooks.default.secret.get.
|
||||
// signature_header = "v1,<base64> v1,<base64>" (space separated).
|
||||
// now_seconds: injected clock. Returns false on any mismatch, stale or
|
||||
// malformed input; never throws on attacker-controlled data.
|
||||
static bool verify(const std::string& secret, const std::string& webhook_id, const std::string& timestamp,
|
||||
const std::string& signature_header, const std::string& body, long now_seconds);
|
||||
// base64(HMAC-SHA256(key, id + "." + timestamp + "." + body)); "" on a bad secret.
|
||||
static std::string sign(const std::string& secret, const std::string& webhook_id, const std::string& timestamp,
|
||||
const std::string& body);
|
||||
};
|
||||
|
||||
// kind-parametrised adapter core shared by the image and video generators.
|
||||
class ReplicateJobs {
|
||||
public:
|
||||
explicit ReplicateJobs(std::shared_ptr<ReplicateClient> c) : client_(std::move(c)) {}
|
||||
JobStatus poll(const JobTicket& ticket, MediaKind kind);
|
||||
bool cancel(const JobTicket& ticket, bool confirm);
|
||||
JobTicket ticket_from(const Json& prediction, const std::string& model, MediaKind kind) const;
|
||||
ReplicateClient& client() { return *client_; }
|
||||
const ReplicateClient& client() const { return *client_; }
|
||||
|
||||
private:
|
||||
std::shared_ptr<ReplicateClient> client_;
|
||||
};
|
||||
|
||||
class ReplicateImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit ReplicateImageGenerator(std::shared_ptr<ReplicateClient> c) : jobs_(std::move(c)) {}
|
||||
std::string provider() const override { return "replicate"; }
|
||||
bool configured() const override { return jobs_.client().configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"black-forest-labs/flux-schnell", "black-forest-labs/flux-dev", "black-forest-labs/flux-1.1-pro",
|
||||
"stability-ai/stable-diffusion-3.5-large", "ideogram-ai/ideogram-v3-turbo", "recraft-ai/recraft-v3"};
|
||||
}
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override { return jobs_.poll(ticket, MediaKind::kImage); }
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override { return jobs_.cancel(ticket, confirm); }
|
||||
int suggested_poll_ms() const override { return 1500; }
|
||||
|
||||
private:
|
||||
Json input(const ImageRequest& request) const;
|
||||
ReplicateJobs jobs_;
|
||||
};
|
||||
|
||||
class ReplicateVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit ReplicateVideoGenerator(std::shared_ptr<ReplicateClient> c) : jobs_(std::move(c)) {}
|
||||
std::string provider() const override { return "replicate"; }
|
||||
bool configured() const override { return jobs_.client().configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"minimax/video-01", "luma/ray", "kwaivgi/kling-v1.6-standard", "wan-video/wan-2.1-t2v-480p"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override { return jobs_.poll(ticket, MediaKind::kVideo); }
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override { return jobs_.cancel(ticket, confirm); }
|
||||
int suggested_poll_ms() const override { return 3000; }
|
||||
|
||||
private:
|
||||
Json input(const VideoRequest& request) const;
|
||||
ReplicateJobs jobs_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+91
@@ -0,0 +1,91 @@
|
||||
#include "vendors/media/runway/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"text_to_image", "POST", "/v1/text_to_image", "", "json", "model,promptText,ratio,referenceImages,contentModeration,seed,background,outputCount,quality,outputFormat,grounding,edit,maskUri,preciseEdit,thinkingLevel,backgroundColor,colors", "", "paid"},
|
||||
{"text_to_video", "POST", "/v1/text_to_video", "", "json", "duration,model,promptText,ratio,contentModeration,outputFormat,proresProfile,seed,audio,negativePrompt,providerSettings,referenceAudio,references,referenceVideos,resolution,cfgScale,promptExpansionMode,draft,draftTaskId", "", "paid"},
|
||||
{"image_to_video", "POST", "/v1/image_to_video", "", "json", "duration,model,promptImage,promptText,ratio,contentModeration,outputFormat,proresProfile,seed,audio,negativePrompt,providerSettings,references,referenceAudio,resolution,cfgScale,referenceVideos,promptExpansionMode,draft", "", "paid"},
|
||||
{"video_to_video", "POST", "/v1/video_to_video", "", "json", "model,videoUri,contentModeration,keyframes,outputFormat,promptText,proresProfile,ratio,seed,targetAspectRatio,promptVideo,providerSettings,references,duration,referenceAudio,referenceVideos,resolution,audio,mode,draft", "", "paid"},
|
||||
{"image_upscale", "POST", "/v1/image_upscale", "", "json", "imageUri,model,flavor,scaleFactor,sharpen,smartGrain,ultraDetail", "", "paid"},
|
||||
{"video_upscale", "POST", "/v1/video_upscale", "", "json", "model,videoUri,creativity,flavor,fpsBoost,resolution,sharpen,smartGrain,targetFramerate", "", "paid"},
|
||||
{"video_to_hdr", "POST", "/v1/video_to_hdr", "", "json", "model,videoUri,outputFormat,proresProfile", "", "paid"},
|
||||
{"video_to_audio", "POST", "/v1/video_to_audio", "", "json", "duration,model,videoUri,format,negativePrompt,numVariants,preserveSpeech,promptText,seed", "", "paid"},
|
||||
{"character_performance", "POST", "/v1/character_performance", "", "json", "type,uri,publicFigureThreshold,character,model,reference,bodyControl,contentModeration,expressionIntensity,ratio,seed", "", "paid"},
|
||||
{"sound_effect", "POST", "/v1/sound_effect", "", "json", "model,promptText,loudnessRate,outputFormat,pitchRate,referenceAudios,sampleRate,speechRate,duration,format,negativePrompt,numVariants,seed,loop", "", "paid"},
|
||||
{"speech_to_speech", "POST", "/v1/speech_to_speech", "", "json", "type,uri,presetId,media,model,voice,removeBackgroundNoise", "", "paid"},
|
||||
{"text_to_speech", "POST", "/v1/text_to_speech", "", "json", "model,promptText,loudnessRate,outputFormat,pitchRate,sampleRate,speechRate,voice,languageCode,nextText,previousText,seed,similarityBoost,speed,stability,style,useSpeakerBoost,applyTextNormalization", "", "paid"},
|
||||
{"voice_dubbing", "POST", "/v1/voice_dubbing", "", "json", "audioUri,model,targetLang,disableVoiceCloning,dropBackgroundAudio,numSpeakers", "", "paid"},
|
||||
{"voice_isolation", "POST", "/v1/voice_isolation", "", "json", "audioUri,model", "", "paid"},
|
||||
{"avatar_videos", "POST", "/v1/avatar_videos", "", "json", "presetId,type,avatarId,audio,text,voice,avatar,model,speech", "", "paid"},
|
||||
{"generate.image", "POST", "/v1/generate/image", "", "json", "promptText,aspectRatio,contentModeration,outputCount,referenceImages,resolution,seed,configId,input", "", "paid"},
|
||||
{"generate.video", "POST", "/v1/generate/video", "", "json", "aspectRatio,audio,contentModeration,duration,keyframes,negativePrompt,promptText,referenceAudio,referenceImages,referenceVideos,resolution,seed,configId,input", "", "paid"},
|
||||
{"generate.audio", "POST", "/v1/generate/audio", "", "json", "promptText,type,duration,loop,referenceAudios,voice,configId,input", "", "paid"},
|
||||
{"recipes.ad_localization", "POST", "/v1/recipes/ad_localization", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"recipes.marketing_stock_image", "POST", "/v1/recipes/marketing_stock_image", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"recipes.multi_shot_video", "POST", "/v1/recipes/multi_shot_video", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"recipes.product_ad", "POST", "/v1/recipes/product_ad", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"recipes.product_campaign_image", "POST", "/v1/recipes/product_campaign_image", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"recipes.product_swap", "POST", "/v1/recipes/product_swap", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"recipes.product_ugc", "POST", "/v1/recipes/product_ugc", "", "json", "uri,mode,prompt,version,audio,duration,firstFrame,ratio,shots,view,referenceImage,targetLanguage,outputCount,quality,productImages,productInfo,styleImages,userConcept,image,newProductImages,originalProductImage,referenceVideo,resolution,characterImage,productImage", "", "paid"},
|
||||
{"tasks.get", "GET", "/v1/tasks/{id}", "", "none", "", "", "read"},
|
||||
{"tasks.delete", "DELETE", "/v1/tasks/{id}", "", "none", "", "", "delete"},
|
||||
{"organization.get", "GET", "/v1/organization", "", "none", "", "", "read"},
|
||||
{"organization.usage", "POST", "/v1/organization/usage", "", "json", "beforeDate,startDate", "", "read"},
|
||||
{"uploads.create", "POST", "/v1/uploads", "", "json", "file,fileMetadata", "", "write"},
|
||||
{"avatars.create", "POST", "/v1/avatars", "", "json", "presetId,type,id,name,personality,referenceImage,voice,documentIds,imageProcessing,startScript,endDate,startDate", "", "write"},
|
||||
{"avatars.list", "GET", "/v1/avatars", "", "none", "", "", "read"},
|
||||
{"avatars.get", "GET", "/v1/avatars/{id}", "", "none", "", "", "read"},
|
||||
{"avatars.update", "PATCH", "/v1/avatars/{id}", "", "json", "presetId,type,id,name,personality,referenceImage,voice,documentIds,imageProcessing,startScript,endDate,startDate", "", "write"},
|
||||
{"avatars.delete", "DELETE", "/v1/avatars/{id}", "", "none", "", "", "delete"},
|
||||
{"avatar_usage", "GET", "/v1/avatar_usage", "", "none", "", "", "read"},
|
||||
{"avatar_conversations.list", "GET", "/v1/avatar_conversations", "", "none", "", "", "read"},
|
||||
{"avatar_conversations.get", "GET", "/v1/avatar_conversations/{id}", "", "none", "", "", "read"},
|
||||
{"avatar_conversations.delete", "DELETE", "/v1/avatar_conversations/{id}", "", "none", "", "", "delete"},
|
||||
{"documents.create", "POST", "/v1/documents", "", "json", "content,name", "", "write"},
|
||||
{"documents.list", "GET", "/v1/documents", "", "none", "", "", "read"},
|
||||
{"documents.get", "GET", "/v1/documents/{id}", "", "none", "", "", "read"},
|
||||
{"documents.update", "PATCH", "/v1/documents/{id}", "", "json", "content,name", "", "write"},
|
||||
{"documents.delete", "DELETE", "/v1/documents/{id}", "", "none", "", "", "delete"},
|
||||
{"voices.create", "POST", "/v1/voices", "", "json", "audio,type,model,prompt,from,name,description", "", "write"},
|
||||
{"voices.preview", "POST", "/v1/voices/preview", "", "json", "audio,type,model,prompt,from,name,description", "", "paid"},
|
||||
{"voices.list", "GET", "/v1/voices", "", "none", "", "", "read"},
|
||||
{"voices.get", "GET", "/v1/voices/{id}", "", "none", "", "", "read"},
|
||||
{"voices.update", "PATCH", "/v1/voices/{id}", "", "json", "audio,type,model,prompt,from,name,description", "", "write"},
|
||||
{"voices.delete", "DELETE", "/v1/voices/{id}", "", "none", "", "", "delete"},
|
||||
{"realtime_sessions.create", "POST", "/v1/realtime_sessions", "", "json", "presetId,type,avatarId,signedUrl,token,roomName,url,agentIdentity,description,name,parameters,timeoutSeconds,avatar,model,integration,livekit,maxDuration,personality,startScript,tools", "", "paid"},
|
||||
{"realtime_sessions.get", "GET", "/v1/realtime_sessions/{id}", "", "none", "", "", "read"},
|
||||
{"realtime_sessions.delete", "DELETE", "/v1/realtime_sessions/{id}", "", "none", "", "", "delete"},
|
||||
{"routers.create", "POST", "/v1/routers", "", "json", "fallback,maxCreditsPerGeneration,models,optimizeFor,schemaVersion,slug,description,name,settings", "", "write"},
|
||||
{"routers.list", "GET", "/v1/routers", "", "none", "", "", "read"},
|
||||
{"routers.get", "GET", "/v1/routers/{id}", "", "none", "", "", "read"},
|
||||
{"routers.update", "PATCH", "/v1/routers/{id}", "", "json", "fallback,maxCreditsPerGeneration,models,optimizeFor,schemaVersion,slug,description,name,settings", "", "write"},
|
||||
{"routers.delete", "DELETE", "/v1/routers/{id}", "", "none", "", "", "delete"},
|
||||
{"routers.requests", "GET", "/v1/routers/{id}/requests", "", "none", "", "", "read"},
|
||||
{"workflows.list", "GET", "/v1/workflows", "", "none", "", "", "read"},
|
||||
{"workflows.get", "GET", "/v1/workflows/{id}", "", "none", "", "", "read"},
|
||||
{"workflows.invoke", "POST", "/v1/workflows/{id}", "", "json", "type,value,uri,nodeOutputs", "", "paid"},
|
||||
{"workflow_invocations.get", "GET", "/v1/workflow_invocations/{id}", "", "none", "", "", "read"},
|
||||
{"audit_logs.list", "GET", "/v1/organization/webapp/audit_logs", "", "none", "", "", "read"},
|
||||
{"audit_logs.get", "GET", "/v1/organization/webapp/audit_logs/{id}", "", "none", "", "", "read"},
|
||||
{"webapp_usage.list", "GET", "/v1/organization/webapp/usage", "", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"TaskRetrieveResponse", "id,status,createdAt,output,failure,failureCode,progress"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& RunwayCalls::table() {
|
||||
static const CallTable t("runway", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t RunwayCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* RunwayCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct RunwayCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+163
@@ -0,0 +1,163 @@
|
||||
#include "vendors/media/runway/runway.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs with_body(const Json& body) {
|
||||
CallArgs a;
|
||||
a.body = body;
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs by_id(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["id"] = id;
|
||||
return a;
|
||||
}
|
||||
|
||||
std::string ratio_of(const std::string& aspect, int width, int height) {
|
||||
if (!aspect.empty()) return aspect;
|
||||
if (width && height) return std::to_string(width) + ":" + std::to_string(height);
|
||||
return "";
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall RunwayClient::plan(const std::string& key, const Json& body) const { return prepare(key, with_body(body)); }
|
||||
Json RunwayClient::submit(const std::string& key, const Json& body, bool confirm) {
|
||||
return call_json(key, with_body(body), confirm);
|
||||
}
|
||||
Json RunwayClient::task(const std::string& id) { return call_json("tasks.get", by_id(id)); }
|
||||
Json RunwayClient::cancel_or_delete_task(const std::string& id, bool confirm) {
|
||||
return call_json("tasks.delete", by_id(id), confirm);
|
||||
}
|
||||
Json RunwayClient::organization() { return call_json("organization.get"); }
|
||||
|
||||
Json RunwayClient::organization_usage(const Json& body) {
|
||||
PreparedCall c = prepare("organization.usage", with_body(body));
|
||||
c.effect = Effect::kRead; // a usage query: the POST creates nothing and costs nothing
|
||||
return parse_json("runway", execute(c, false));
|
||||
}
|
||||
|
||||
JobTicket RunwayJobs::ticket_from(const Json& reply, const std::string& model, MediaKind kind) {
|
||||
JobTicket t;
|
||||
t.provider = "runway";
|
||||
t.model = model;
|
||||
t.kind = kind;
|
||||
t.id = AdapterSupport::str(reply, "id");
|
||||
if (t.id.empty()) throw llm::ApiError("runway", 502, "BAD_RESPONSE", "no id in the task reply");
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus RunwayJobs::status_of(const Json& task, MediaKind kind) {
|
||||
JobStatus s;
|
||||
s.raw_state = AdapterSupport::str(task, "status");
|
||||
if (s.raw_state == "SUCCEEDED") {
|
||||
s.state = JobState::kSucceeded;
|
||||
if (const Json* o = task.find("output")) s.assets = OutputScanner::scan(*o, kind);
|
||||
if (s.assets.empty()) {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = "succeeded without output URLs";
|
||||
}
|
||||
} else if (s.raw_state == "FAILED") {
|
||||
s.state = JobState::kFailed;
|
||||
s.error = AdapterSupport::str(task, "failure");
|
||||
std::string code = AdapterSupport::str(task, "failureCode");
|
||||
if (!code.empty()) s.error += " (" + code + ")";
|
||||
} else if (s.raw_state == "CANCELLED") {
|
||||
s.state = JobState::kCancelled;
|
||||
} else if (s.raw_state == "RUNNING") {
|
||||
s.state = JobState::kRunning;
|
||||
if (const Json* p = task.find("progress"); p && p->is_number()) s.progress = p->as_number();
|
||||
} else {
|
||||
s.state = JobState::kQueued; // PENDING, THROTTLED
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
JobStatus RunwayJobs::poll(RunwayClient& client, const JobTicket& ticket) {
|
||||
return status_of(client.task(ticket.id), ticket.kind);
|
||||
}
|
||||
|
||||
Json RunwayImageGenerator::body(const ImageRequest& r) const {
|
||||
r.validate();
|
||||
if (r.count != 1) throw llm::ApiError::invalid("runway", "one image per task (count must be 1)");
|
||||
Json b = Json::object();
|
||||
b.set("model", Json(r.model.empty() ? "gen4_image" : r.model));
|
||||
AdapterSupport::put(b, "promptText", r.prompt);
|
||||
AdapterSupport::put(b, "ratio", ratio_of(r.aspect_ratio, r.width, r.height));
|
||||
AdapterSupport::put_seed(b, "seed", r.seed);
|
||||
if (!r.source_url.empty()) {
|
||||
Json refs = Json::array();
|
||||
Json one = Json::object();
|
||||
one.set("uri", Json(r.source_url));
|
||||
one.set("tag", Json("source"));
|
||||
refs.push(one);
|
||||
b.set("referenceImages", refs);
|
||||
}
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json RunwayImageGenerator::plan(const ImageRequest& request) const {
|
||||
return client_->plan("text_to_image", body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket RunwayImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "runway", "generate");
|
||||
return RunwayJobs::ticket_from(client_->submit("text_to_image", b, true), b.find("model")->as_string(), MediaKind::kImage);
|
||||
}
|
||||
|
||||
JobStatus RunwayImageGenerator::poll(const JobTicket& t) { return RunwayJobs::poll(*client_, t); }
|
||||
|
||||
bool RunwayImageGenerator::cancel(const JobTicket& t, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "runway", "cancel task");
|
||||
client_->cancel_or_delete_task(t.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string RunwayVideoGenerator::key_for(const VideoRequest& r) {
|
||||
return r.image_url.empty() ? "text_to_video" : "image_to_video";
|
||||
}
|
||||
|
||||
Json RunwayVideoGenerator::body(const VideoRequest& r) const {
|
||||
r.validate();
|
||||
Json b = Json::object();
|
||||
b.set("model", Json(r.model.empty() ? (r.image_url.empty() ? "gen4.5" : "gen4_turbo") : r.model));
|
||||
AdapterSupport::put(b, "promptText", r.prompt);
|
||||
AdapterSupport::put(b, "ratio", r.aspect_ratio);
|
||||
AdapterSupport::put_int(b, "duration", r.duration_seconds);
|
||||
AdapterSupport::put_seed(b, "seed", r.seed);
|
||||
AdapterSupport::put(b, "negativePrompt", r.negative_prompt);
|
||||
AdapterSupport::put(b, "resolution", r.resolution);
|
||||
AdapterSupport::put(b, "promptImage", r.image_url);
|
||||
if (!r.end_image_url.empty())
|
||||
throw llm::ApiError::invalid("runway", "end frames use per-model promptImage arrays: pass them in extra");
|
||||
AdapterSupport::merge_extra(b, r.extra);
|
||||
return b;
|
||||
}
|
||||
|
||||
Json RunwayVideoGenerator::plan(const VideoRequest& request) const {
|
||||
return client_->plan(key_for(request), body(request)).summary();
|
||||
}
|
||||
|
||||
JobTicket RunwayVideoGenerator::submit(const VideoRequest& request, bool confirm) {
|
||||
Json b = body(request);
|
||||
AdapterSupport::need_confirm(confirm, "runway", "generate");
|
||||
return RunwayJobs::ticket_from(client_->submit(key_for(request), b, true), b.find("model")->as_string(), MediaKind::kVideo);
|
||||
}
|
||||
|
||||
JobStatus RunwayVideoGenerator::poll(const JobTicket& t) { return RunwayJobs::poll(*client_, t); }
|
||||
|
||||
bool RunwayVideoGenerator::cancel(const JobTicket& t, bool confirm) {
|
||||
AdapterSupport::need_confirm(confirm, "runway", "cancel task");
|
||||
client_->cancel_or_delete_task(t.id, true);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+93
@@ -0,0 +1,93 @@
|
||||
#pragma once
|
||||
|
||||
// Runway developer API (https://api.dev.runwayml.com): text / image / video
|
||||
// generation, tasks, uploads, organization, avatars, voices, documents,
|
||||
// routers, workflows, realtime sessions. Bearer auth plus the required
|
||||
// X-Runway-Version header. The key is RUNWAY_API_KEY (the official SDK reads
|
||||
// RUNWAYML_API_SECRET; both are accepted). Runway publishes no OpenAPI
|
||||
// document: the table is checked against the official SDK source.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/runway/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class RunwayClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.dev.runwayml.com";
|
||||
static constexpr const char* kKeyEnv = "RUNWAY_API_KEY";
|
||||
static constexpr const char* kKeyEnvSdk = "RUNWAYML_API_SECRET";
|
||||
static constexpr const char* kApiVersion = "2024-11-06"; // SDK default X-Runway-Version
|
||||
|
||||
explicit RunwayClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), RunwayCalls::table()) {}
|
||||
static llm::ApiClientConfig config() {
|
||||
llm::ApiClientConfig c = ClientFactory::bearer("runway", kHost, kKeyEnv);
|
||||
c.default_headers["X-Runway-Version"] = kApiVersion;
|
||||
return c;
|
||||
}
|
||||
static std::shared_ptr<RunwayClient> from_env() {
|
||||
return std::make_shared<RunwayClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv, kKeyEnvSdk})));
|
||||
}
|
||||
static std::shared_ptr<RunwayClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<RunwayClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// Paid generation rows (text_to_image, text_to_video, image_to_video, ...). Returns {id, ...}.
|
||||
Json submit(const std::string& key, const Json& body, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& body) const;
|
||||
Json task(const std::string& id);
|
||||
Json cancel_or_delete_task(const std::string& id, bool confirm); // no default
|
||||
Json organization();
|
||||
Json organization_usage(const Json& body); // POST but a query: effect read
|
||||
};
|
||||
|
||||
class RunwayImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit RunwayImageGenerator(std::shared_ptr<RunwayClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "runway"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"gen4_image", "gen4_image_turbo", "gemini_2.5_flash", "ideogram_4_5", "gpt_image_2", "seedream5_pro"};
|
||||
}
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 3000; }
|
||||
|
||||
private:
|
||||
Json body(const ImageRequest& request) const;
|
||||
std::shared_ptr<RunwayClient> client_;
|
||||
};
|
||||
|
||||
class RunwayVideoGenerator : public VideoGenerator {
|
||||
public:
|
||||
explicit RunwayVideoGenerator(std::shared_ptr<RunwayClient> c) : client_(std::move(c)) {}
|
||||
std::string provider() const override { return "runway"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override {
|
||||
return {"gen4.5", "gen4_turbo", "veo3.1", "veo3.1_fast", "kling3.0_pro", "seedance2", "hailuo3"};
|
||||
}
|
||||
Json plan(const VideoRequest& request) const override;
|
||||
JobTicket submit(const VideoRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket& ticket, bool confirm) override;
|
||||
int suggested_poll_ms() const override { return 5000; }
|
||||
|
||||
static std::string key_for(const VideoRequest& request);
|
||||
|
||||
private:
|
||||
Json body(const VideoRequest& request) const;
|
||||
std::shared_ptr<RunwayClient> client_;
|
||||
};
|
||||
|
||||
struct RunwayJobs {
|
||||
static JobStatus poll(RunwayClient& client, const JobTicket& ticket);
|
||||
static JobStatus status_of(const Json& task, MediaKind kind);
|
||||
static JobTicket ticket_from(const Json& reply, const std::string& model, MediaKind kind);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
#include "vendors/media/stability/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// key, http method, path, spec operation id, body kind, body fields, query params, effect)
|
||||
static const CallRow kRows[] = {
|
||||
{"3d.stable-fast-3d", "POST", "/v2beta/3d/stable-fast-3d", "", "multipart", "image,texture_resolution,foreground_ratio,remesh,vertex_count", "", "paid"},
|
||||
{"3d.stable-point-aware-3d", "POST", "/v2beta/3d/stable-point-aware-3d", "", "multipart", "image,texture_resolution,foreground_ratio,remesh,target_type,target_count,guidance_scale,seed", "", "paid"},
|
||||
{"results.id", "GET", "/v2beta/results/{id}", "", "none", "", "", "read"},
|
||||
{"audio.stable-audio-2.text-to-audio", "POST", "/v2beta/audio/stable-audio-2/text-to-audio", "", "multipart", "prompt,duration,seed,steps,cfg_scale,model,output_format", "", "paid"},
|
||||
{"audio.stable-audio-2.audio-to-audio", "POST", "/v2beta/audio/stable-audio-2/audio-to-audio", "", "multipart", "prompt,duration,seed,steps,cfg_scale,model,output_format,audio,strength", "", "paid"},
|
||||
{"audio.stable-audio-2.inpaint", "POST", "/v2beta/audio/stable-audio-2/inpaint", "", "multipart", "prompt,duration,seed,steps,output_format,audio,mask_start,mask_end", "", "paid"},
|
||||
{"audio.stable-audio.text-to-audio", "POST", "/v2beta/audio/stable-audio/text-to-audio", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format", "", "paid"},
|
||||
{"audio.stable-audio.audio-to-audio", "POST", "/v2beta/audio/stable-audio/audio-to-audio", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format,strength,audio", "", "paid"},
|
||||
{"audio.stable-audio.inpaint", "POST", "/v2beta/audio/stable-audio/inpaint", "", "multipart", "prompt,model,duration,seed,steps,cfg_scale,output_format,mask_start,mask_end,audio", "", "paid"},
|
||||
{"audio.results.id", "GET", "/v2beta/audio/results/{id}", "fetchAudioResult", "none", "", "", "read"},
|
||||
{"stable-image.upscale.conservative", "POST", "/v2beta/stable-image/upscale/conservative", "", "multipart", "image,prompt,negative_prompt,seed,output_format,creativity", "", "paid"},
|
||||
{"stable-image.upscale.creative", "POST", "/v2beta/stable-image/upscale/creative", "", "multipart", "image,prompt,negative_prompt,output_format,seed,creativity,style_preset", "", "paid"},
|
||||
{"stable-image.upscale.creative.result.id", "GET", "/v2beta/stable-image/upscale/creative/result/{id}", "", "none", "", "", "read"},
|
||||
{"stable-image.upscale.fast", "POST", "/v2beta/stable-image/upscale/fast", "", "multipart", "image,output_format", "", "paid"},
|
||||
{"stable-image.edit.erase", "POST", "/v2beta/stable-image/edit/erase", "", "multipart", "image,mask,grow_mask,seed,output_format", "", "paid"},
|
||||
{"stable-image.edit.inpaint", "POST", "/v2beta/stable-image/edit/inpaint", "", "multipart", "image,prompt,negative_prompt,mask,grow_mask,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.edit.outpaint", "POST", "/v2beta/stable-image/edit/outpaint", "", "multipart", "image,left,right,up,down,creativity,prompt,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.edit.search-and-replace", "POST", "/v2beta/stable-image/edit/search-and-replace", "", "multipart", "image,prompt,search_prompt,negative_prompt,grow_mask,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.edit.search-and-recolor", "POST", "/v2beta/stable-image/edit/search-and-recolor", "", "multipart", "image,prompt,select_prompt,negative_prompt,grow_mask,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.edit.remove-background", "POST", "/v2beta/stable-image/edit/remove-background", "", "multipart", "image,output_format", "", "paid"},
|
||||
{"stable-image.edit.replace-background-and-relight", "POST", "/v2beta/stable-image/edit/replace-background-and-relight", "", "multipart", "subject_image,background_reference,background_prompt,foreground_prompt,negative_prompt,preserve_original_subject,original_background_depth,keep_original_background,light_source_direction,light_reference,light_source_strength,seed,output_format", "", "paid"},
|
||||
{"stable-image.generate.ultra", "POST", "/v2beta/stable-image/generate/ultra", "", "multipart", "prompt,negative_prompt,aspect_ratio,seed,output_format,image,style_preset,strength", "", "paid"},
|
||||
{"stable-image.generate.core", "POST", "/v2beta/stable-image/generate/core", "", "multipart", "prompt,aspect_ratio,negative_prompt,seed,style_preset,output_format", "", "paid"},
|
||||
{"stable-image.generate.sd3", "POST", "/v2beta/stable-image/generate/sd3", "", "multipart", "prompt,mode,image,strength,aspect_ratio,model,seed,output_format,style_preset,negative_prompt,cfg_scale", "", "paid"},
|
||||
{"stable-image.control.sketch", "POST", "/v2beta/stable-image/control/sketch", "", "multipart", "prompt,image,control_strength,negative_prompt,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.control.structure", "POST", "/v2beta/stable-image/control/structure", "", "multipart", "prompt,image,control_strength,negative_prompt,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.control.style", "POST", "/v2beta/stable-image/control/style", "", "multipart", "prompt,image,negative_prompt,aspect_ratio,fidelity,seed,output_format,style_preset", "", "paid"},
|
||||
{"stable-image.control.style-transfer", "POST", "/v2beta/stable-image/control/style-transfer", "", "multipart", "init_image,style_image,prompt,negative_prompt,seed,style_strength,composition_fidelity,change_strength,output_format", "", "paid"},
|
||||
{"v1.generation.id.text-to-image", "POST", "/v1/generation/{engine_id}/text-to-image", "textToImage", "json", "height,width,text_prompts,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"},
|
||||
{"v1.generation.id.image-to-image", "POST", "/v1/generation/{engine_id}/image-to-image", "imageToImage", "multipart", "text_prompts,init_image,init_image_mode,image_strength,step_schedule_start,step_schedule_end,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"},
|
||||
{"v1.generation.id.image-to-image.masking", "POST", "/v1/generation/{engine_id}/image-to-image/masking", "masking", "multipart", "init_image,mask_source,mask_image,text_prompts,cfg_scale,clip_guidance_preset,sampler,samples,seed,steps,style_preset,extras", "", "paid"},
|
||||
{"v1.engines.list", "GET", "/v1/engines/list", "listEngines", "none", "", "", "read"},
|
||||
{"v1.user.account", "GET", "/v1/user/account", "userAccount", "none", "", "", "read"},
|
||||
{"v1.user.balance", "GET", "/v1/user/balance", "userBalance", "none", "", "", "read"},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
// FIELDS-BEGIN (parsed by tools/check_spec.py; one row per line: schema, fields we read)
|
||||
static const char* const kFields[][2] = {
|
||||
{"AsyncGenerationResponse", "id"},
|
||||
{"ImageJSONResponse", "image,finish_reason,seed"},
|
||||
{"AudioJSONResponse", "audio,finish_reason,seed"},
|
||||
};
|
||||
// FIELDS-END
|
||||
|
||||
const CallTable& StabilityCalls::table() {
|
||||
static const CallTable t("stability", kRows, sizeof(kRows) / sizeof(kRows[0]));
|
||||
return t;
|
||||
}
|
||||
|
||||
std::size_t StabilityCalls::field_count() { return sizeof(kFields) / sizeof(kFields[0]); }
|
||||
const char* const* StabilityCalls::field_row(std::size_t i) { return kFields[i]; }
|
||||
|
||||
} // namespace molo::media
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of REST calls for this provider. tools/check_spec.py checks
|
||||
// every row against the published spec. Code builds every path from here.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
#include "vendors/media/common/call_table.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
struct StabilityCalls {
|
||||
static const CallTable& table();
|
||||
// Response fields our parsers read, per schema (checked by check_spec.py).
|
||||
static std::size_t field_count();
|
||||
static const char* const* field_row(std::size_t index);
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
#include "vendors/media/stability/stability.h"
|
||||
|
||||
#include "vendors/media/common/adapter_support.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
namespace {
|
||||
|
||||
CallArgs form(const Json& fields, const std::vector<FilePart>& files) {
|
||||
CallArgs a;
|
||||
a.body = fields;
|
||||
a.files = files;
|
||||
a.headers["Accept"] = "application/json";
|
||||
return a;
|
||||
}
|
||||
|
||||
CallArgs by_id(const std::string& id) {
|
||||
CallArgs a;
|
||||
a.path["id"] = id;
|
||||
a.headers["Accept"] = "application/json";
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PreparedCall StabilityClient::plan(const std::string& key, const Json& fields,
|
||||
const std::vector<FilePart>& files) const {
|
||||
return prepare(key, form(fields, files));
|
||||
}
|
||||
|
||||
Json StabilityClient::submit(const std::string& key, const Json& fields, const std::vector<FilePart>& files,
|
||||
bool confirm) {
|
||||
return call_json(key, form(fields, files), confirm);
|
||||
}
|
||||
|
||||
Json StabilityClient::result(const std::string& id) { return call_json("results.id", by_id(id)); }
|
||||
Json StabilityClient::audio_result(const std::string& id) { return call_json("audio.results.id", by_id(id)); }
|
||||
Json StabilityClient::creative_upscale_result(const std::string& id) {
|
||||
return call_json("stable-image.upscale.creative.result.id", by_id(id));
|
||||
}
|
||||
Json StabilityClient::engines() { return call_json("v1.engines.list"); }
|
||||
Json StabilityClient::account() { return call_json("v1.user.account"); }
|
||||
Json StabilityClient::balance() { return call_json("v1.user.balance"); }
|
||||
|
||||
Json StabilityClient::v1_text_to_image(const std::string& engine_id, const Json& body, bool confirm) {
|
||||
CallArgs a;
|
||||
a.path["engine_id"] = engine_id;
|
||||
a.body = body;
|
||||
a.headers["Accept"] = "application/json";
|
||||
return call_json("v1.generation.id.text-to-image", a, confirm);
|
||||
}
|
||||
|
||||
std::vector<std::string> StabilityImageGenerator::models() const {
|
||||
return {"core", "ultra", "sd3.5-large", "sd3.5-large-turbo", "sd3.5-medium", "sd3.5-flash"};
|
||||
}
|
||||
|
||||
std::string StabilityImageGenerator::key_for(const std::string& model) {
|
||||
if (model.empty() || model == "core") return "stable-image.generate.core";
|
||||
if (model == "ultra") return "stable-image.generate.ultra";
|
||||
if (model.rfind("sd3", 0) == 0) return "stable-image.generate.sd3";
|
||||
throw llm::ApiError::invalid("stability", "unknown model '" + model + "' (core, ultra, sd3*)");
|
||||
}
|
||||
|
||||
Json StabilityImageGenerator::fields(const ImageRequest& r, std::string& key) const {
|
||||
r.validate();
|
||||
key = key_for(r.model);
|
||||
if (r.count != 1) throw llm::ApiError::invalid("stability", "one image per call (count must be 1)");
|
||||
if (r.width || r.height) throw llm::ApiError::invalid("stability", "use aspect_ratio, not width / height");
|
||||
if (!r.source_url.empty())
|
||||
throw llm::ApiError::invalid("stability", "source images are uploaded as files: use StabilityClient::submit with a FilePart");
|
||||
Json f = Json::object();
|
||||
AdapterSupport::put(f, "prompt", r.prompt);
|
||||
AdapterSupport::put(f, "negative_prompt", r.negative_prompt);
|
||||
AdapterSupport::put(f, "aspect_ratio", r.aspect_ratio);
|
||||
if (r.seed >= 0) f.set("seed", Json(static_cast<double>(r.seed > 4294967294L ? 4294967294L : r.seed)));
|
||||
if (key == "stable-image.generate.sd3" && r.model != "sd3") f.set("model", Json(r.model));
|
||||
f.set("output_format", Json("png"));
|
||||
AdapterSupport::merge_extra(f, r.extra);
|
||||
return f;
|
||||
}
|
||||
|
||||
Json StabilityImageGenerator::plan(const ImageRequest& request) const {
|
||||
std::string key;
|
||||
Json f = fields(request, key);
|
||||
return client_->plan(key, f, {}).summary();
|
||||
}
|
||||
|
||||
JobTicket StabilityImageGenerator::submit(const ImageRequest& request, bool confirm) {
|
||||
std::string key;
|
||||
Json f = fields(request, key);
|
||||
AdapterSupport::need_confirm(confirm, "stability", "generate");
|
||||
Json out = client_->submit(key, f, {}, true);
|
||||
JobTicket t;
|
||||
t.provider = "stability";
|
||||
t.model = request.model.empty() ? "core" : request.model;
|
||||
t.kind = MediaKind::kImage;
|
||||
if (const Json* id = out.find("id"); id && id->is_string()) {
|
||||
t.id = id->as_string();
|
||||
return t;
|
||||
}
|
||||
std::string b64 = AdapterSupport::str(out, "image");
|
||||
if (b64.empty()) throw llm::ApiError("stability", 502, "BAD_RESPONSE", "no image and no id in the reply");
|
||||
MediaAsset a;
|
||||
a.kind = MediaKind::kImage;
|
||||
a.data_base64 = b64;
|
||||
a.mime = "image/" + (f.find("output_format") ? f.find("output_format")->as_string() : std::string("png"));
|
||||
a.label = "seed=" + AdapterSupport::str(out, "seed") + " finish_reason=" + AdapterSupport::str(out, "finish_reason");
|
||||
t.id = "immediate";
|
||||
t.immediate = true;
|
||||
t.assets.push_back(a);
|
||||
return t;
|
||||
}
|
||||
|
||||
JobStatus StabilityImageGenerator::poll(const JobTicket& ticket) {
|
||||
JobStatus s;
|
||||
if (ticket.immediate) {
|
||||
s.state = JobState::kSucceeded;
|
||||
s.raw_state = "immediate";
|
||||
s.assets = ticket.assets;
|
||||
return s;
|
||||
}
|
||||
Json out = client_->result(ticket.id);
|
||||
std::string b64 = AdapterSupport::str(out, "image");
|
||||
if (b64.empty()) {
|
||||
s.state = JobState::kRunning;
|
||||
s.raw_state = AdapterSupport::str(out, "status");
|
||||
if (s.raw_state.empty()) s.raw_state = "in-progress";
|
||||
return s;
|
||||
}
|
||||
MediaAsset a;
|
||||
a.kind = MediaKind::kImage;
|
||||
a.data_base64 = b64;
|
||||
a.mime = "image/png";
|
||||
a.label = "seed=" + AdapterSupport::str(out, "seed") + " finish_reason=" + AdapterSupport::str(out, "finish_reason");
|
||||
s.state = JobState::kSucceeded;
|
||||
s.raw_state = "complete";
|
||||
s.assets.push_back(a);
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace molo::media
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
#pragma once
|
||||
|
||||
// Stability AI: v2beta Stable Image (generate, edit, upscale, control), 3D,
|
||||
// Stable Audio, and the v1 engines / account surface. Every v2beta call is
|
||||
// multipart/form-data; we always ask for JSON (base64 images) so errors and
|
||||
// results parse the same way.
|
||||
|
||||
#include "vendors/media/common/client_factory.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
#include "vendors/media/stability/calls.h"
|
||||
|
||||
namespace molo::media {
|
||||
|
||||
class StabilityClient : public MediaService {
|
||||
public:
|
||||
static constexpr const char* kHost = "https://api.stability.ai";
|
||||
static constexpr const char* kKeyEnv = "STABILITY_API_KEY";
|
||||
|
||||
explicit StabilityClient(std::shared_ptr<llm::ApiClient> client)
|
||||
: MediaService(std::move(client), StabilityCalls::table()) {}
|
||||
static llm::ApiClientConfig config() { return ClientFactory::bearer("stability", kHost, kKeyEnv); }
|
||||
static std::shared_ptr<StabilityClient> from_env() {
|
||||
return std::make_shared<StabilityClient>(ClientFactory::real(config(), ClientFactory::env({kKeyEnv})));
|
||||
}
|
||||
static std::shared_ptr<StabilityClient> with(std::unique_ptr<clients::Transport> t, const std::string& key) {
|
||||
return std::make_shared<StabilityClient>(ClientFactory::with(std::move(t), config(), key));
|
||||
}
|
||||
|
||||
// Any v2beta POST row (stable-image.*, 3d.*, audio.*). Paid: confirm required.
|
||||
// Returns {image|audio, finish_reason, seed} or, for async rows, {id}.
|
||||
Json submit(const std::string& key, const Json& fields, const std::vector<FilePart>& files, bool confirm);
|
||||
PreparedCall plan(const std::string& key, const Json& fields, const std::vector<FilePart>& files) const;
|
||||
|
||||
// Polls: HTTP 202 {id,status} while running, 200 with the result when done.
|
||||
Json result(const std::string& id); // /v2beta/results/{id}
|
||||
Json audio_result(const std::string& id); // /v2beta/audio/results/{id}
|
||||
Json creative_upscale_result(const std::string& id); // legacy creative upscale path
|
||||
|
||||
// v1 reads.
|
||||
Json engines();
|
||||
Json account();
|
||||
Json balance();
|
||||
// v1 text-to-image (JSON body, artifacts[] with base64).
|
||||
Json v1_text_to_image(const std::string& engine_id, const Json& body, bool confirm);
|
||||
};
|
||||
|
||||
class StabilityImageGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit StabilityImageGenerator(std::shared_ptr<StabilityClient> client) : client_(std::move(client)) {}
|
||||
std::string provider() const override { return "stability"; }
|
||||
bool configured() const override { return client_->configured(); }
|
||||
std::vector<std::string> models() const override;
|
||||
Json plan(const ImageRequest& request) const override;
|
||||
JobTicket submit(const ImageRequest& request, bool confirm) override;
|
||||
JobStatus poll(const JobTicket& ticket) override;
|
||||
bool cancel(const JobTicket&, bool) override { return false; } // no cancel in the API
|
||||
|
||||
// model -> table key (ultra, core, sd3*), "" model = core.
|
||||
static std::string key_for(const std::string& model);
|
||||
|
||||
private:
|
||||
Json fields(const ImageRequest& request, std::string& key) const;
|
||||
std::shared_ptr<StabilityClient> client_;
|
||||
};
|
||||
|
||||
} // namespace molo::media
|
||||
+578
@@ -0,0 +1,578 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check the media call tables against what each vendor publishes.
|
||||
|
||||
For every provider the checker reads the rows between CALLS-BEGIN / CALLS-END
|
||||
in src/vendors/media/<provider>/calls.cpp (and the response fields between
|
||||
FIELDS-BEGIN / FIELDS-END) and compares them with the vendor's own source:
|
||||
|
||||
openapi Stability (v2beta + v1), Black Forest Labs, Replicate, fal (queue
|
||||
spec from the docs, Platform API), Ideogram v2, Recraft, Higgsfield
|
||||
sdk Runway and Luma: the official npm package (resources/*.js for the
|
||||
paths, resources/*.d.ts for the request fields)
|
||||
docs Leonardo v2 (the OpenAPI document embedded in three reference
|
||||
pages), MiniMax (the OpenAPI embedded in the API reference pages)
|
||||
probe Kling (no machine-readable source exists): the path must answer
|
||||
401/400/403 unauthenticated, never 404. Reported as UNVERIFIED.
|
||||
|
||||
It fails (exit 1) when
|
||||
row a declared (method, path) is not an operation of the source
|
||||
field a request body field or query parameter we send is not in the source
|
||||
kind the declared body kind disagrees with the source content types
|
||||
schema a response field we read is not reachable from the named schema
|
||||
coverage a source operation is neither in the table nor in EXCLUSIONS
|
||||
|
||||
Python 3 standard library only. No credentials. Network required.
|
||||
|
||||
python3 -I src/vendors/media/tools/check_spec.py [provider ...]
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.normpath(os.path.join(HERE, ".."))
|
||||
STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
|
||||
UA = {"User-Agent": "molo-media-spec-check/1"}
|
||||
|
||||
# (provider, source operation) -> reason. Everything else in a source must be in the table.
|
||||
EXCLUSIONS = {
|
||||
"stability": {
|
||||
("POST", "/v2alpha/generation/stable-image/upscale"): "alpha endpoint, superseded by v2beta upscale",
|
||||
("GET", "/v2alpha/generation/stable-image/upscale/result/{id}"): "alpha endpoint",
|
||||
("POST", "/v2alpha/generation/stable-image/inpaint"): "alpha endpoint, superseded by v2beta edit/inpaint",
|
||||
},
|
||||
"fal": {
|
||||
("GET", "/{owner}/{app}/requests/{request_id}/status/stream"): "server-sent events; ApiClient streams POST only (gap)",
|
||||
},
|
||||
}
|
||||
# fal Platform API operations outside model inference: out of scope for package media.
|
||||
FAL_PLATFORM_SKIP_PREFIXES = ("/workflows", "/assets", "/storage", "/serverless", "/compute", "/keys", "/account",
|
||||
"/organization")
|
||||
# Rows whose path is generic by design (the vendor says its OpenAPI file is supplementary).
|
||||
GENERIC_ROWS = {"higgsfield": {"model.submit"}}
|
||||
# Rows checked by documentation prose only (the reference page was removed). Listed, never silent.
|
||||
PROSE_ONLY = {"leonardo": {"v1.generations.create", "v1.generations.get", "v1.me"}}
|
||||
|
||||
problems = []
|
||||
notes = []
|
||||
cache = {}
|
||||
|
||||
|
||||
def fail(provider, kind, msg):
|
||||
problems.append("%s: %s: %s" % (provider, kind, msg))
|
||||
|
||||
|
||||
def fetch(url, binary=False):
|
||||
if url not in cache:
|
||||
req = urllib.request.Request(url, headers=UA)
|
||||
with urllib.request.urlopen(req, timeout=90) as r:
|
||||
cache[url] = r.read()
|
||||
data = cache[url]
|
||||
return data if binary else data.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def rows_of(provider):
|
||||
text = open(os.path.join(ROOT, provider, "calls.cpp")).read()
|
||||
a, b = text.index("// CALLS-BEGIN"), text.index("// CALLS-END")
|
||||
rows = []
|
||||
for line in text[a:b].splitlines()[2:]:
|
||||
s = line.strip()
|
||||
if s.startswith("{") and '"' in s:
|
||||
f = STR.findall(s)
|
||||
assert len(f) == 8, (provider, s)
|
||||
rows.append(dict(zip(("key", "method", "path", "op", "kind", "body", "query", "effect"), f)))
|
||||
c, d = text.index("// FIELDS-BEGIN"), text.index("// FIELDS-END")
|
||||
fields = []
|
||||
for line in text[c:d].splitlines()[1:]:
|
||||
s = line.strip()
|
||||
if s.startswith("{") and '"' in s:
|
||||
fields.append(STR.findall(s))
|
||||
for r in rows:
|
||||
r["body"] = [x for x in r["body"].split(",") if x]
|
||||
r["query"] = [x for x in r["query"].split(",") if x]
|
||||
return rows, fields
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- OpenAPI ----
|
||||
class Oas:
|
||||
def __init__(self, doc):
|
||||
self.d = doc
|
||||
|
||||
def ref(self, node):
|
||||
n = 0
|
||||
while isinstance(node, dict) and "$ref" in node and n < 25:
|
||||
cur = self.d
|
||||
for part in node["$ref"].lstrip("#/").split("/"):
|
||||
cur = cur[part.replace("~1", "/").replace("~0", "~")]
|
||||
node = cur
|
||||
n += 1
|
||||
return node
|
||||
|
||||
def props(self, schema, depth=0):
|
||||
schema = self.ref(schema)
|
||||
out = []
|
||||
if not isinstance(schema, dict) or depth > 6:
|
||||
return out
|
||||
for name in schema.get("properties") or {}:
|
||||
out.append(name)
|
||||
for key in ("allOf", "oneOf", "anyOf"):
|
||||
for sub in schema.get(key) or []:
|
||||
out.extend(self.props(sub, depth + 1))
|
||||
return out
|
||||
|
||||
def reachable(self, schema, depth=0, seen=None):
|
||||
"""Every property name reachable from a schema (for response fields)."""
|
||||
seen = seen if seen is not None else set()
|
||||
out = set()
|
||||
if depth > 6:
|
||||
return out
|
||||
if isinstance(schema, dict) and "$ref" in schema:
|
||||
if schema["$ref"] in seen:
|
||||
return out
|
||||
seen.add(schema["$ref"])
|
||||
schema = self.ref(schema)
|
||||
if not isinstance(schema, dict):
|
||||
return out
|
||||
for name, sub in (schema.get("properties") or {}).items():
|
||||
out.add(name)
|
||||
out |= self.reachable(sub, depth + 1, seen)
|
||||
for key in ("allOf", "oneOf", "anyOf"):
|
||||
for sub in schema.get(key) or []:
|
||||
out |= self.reachable(sub, depth + 1, seen)
|
||||
if "items" in schema:
|
||||
out |= self.reachable(schema["items"], depth + 1, seen)
|
||||
ap = schema.get("additionalProperties")
|
||||
if isinstance(ap, dict):
|
||||
out |= self.reachable(ap, depth + 1, seen)
|
||||
return out
|
||||
|
||||
def ops(self):
|
||||
for path, item in self.d["paths"].items():
|
||||
for m, op in item.items():
|
||||
if m in ("get", "post", "put", "patch", "delete"):
|
||||
yield m.upper(), path, op
|
||||
|
||||
def op(self, method, path):
|
||||
item = self.d["paths"].get(path)
|
||||
return item.get(method.lower()) if item else None
|
||||
|
||||
def params(self, path, op, where):
|
||||
out = []
|
||||
for p in (self.d["paths"][path].get("parameters") or []) + (op.get("parameters") or []):
|
||||
p = self.ref(p)
|
||||
if p.get("in") == where:
|
||||
out.append(p["name"])
|
||||
return out
|
||||
|
||||
def body(self, op):
|
||||
rb = self.ref(op.get("requestBody") or {})
|
||||
content = rb.get("content") or {}
|
||||
kinds, fields = [], []
|
||||
for ct, k in (("application/json", "json"), ("multipart/form-data", "multipart"),
|
||||
("application/x-www-form-urlencoded", "form")):
|
||||
if ct in content:
|
||||
kinds.append(k)
|
||||
fields.extend(self.props(content[ct].get("schema") or {}))
|
||||
return kinds, fields
|
||||
|
||||
|
||||
def check_rows_against_oas(provider, rows, oas, strip="", only=lambda r: True, path_filter=lambda p: True):
|
||||
"""Compare table rows with one OpenAPI document. Returns the (method, path) set covered."""
|
||||
covered = set()
|
||||
for r in rows:
|
||||
if not only(r):
|
||||
continue
|
||||
if r["key"] in GENERIC_ROWS.get(provider, ()):
|
||||
notes.append("%s: row %s is generic by design (not in the spec)" % (provider, r["key"]))
|
||||
continue
|
||||
path = r["path"][len(strip):] if strip and r["path"].startswith(strip) else r["path"]
|
||||
op = oas.op(r["method"], path)
|
||||
if op is None:
|
||||
fail(provider, "row", "%s %s (%s) is not in the spec" % (r["method"], r["path"], r["key"]))
|
||||
continue
|
||||
covered.add((r["method"], path))
|
||||
if r["op"] and op.get("operationId") != r["op"]:
|
||||
fail(provider, "row", "%s declares operation id %s, spec says %s" % (r["key"], r["op"], op.get("operationId")))
|
||||
kinds, fields = oas.body(op)
|
||||
want = r["kind"]
|
||||
have = "none" if not kinds else (kinds[0] if len(kinds) == 1 else "either")
|
||||
if want != have:
|
||||
fail(provider, "kind", "%s declares body kind %s, spec has %s" % (r["key"], want, have))
|
||||
for f in r["body"]:
|
||||
if f != "*" and f not in fields:
|
||||
fail(provider, "field", "%s sends body field %s, absent from the spec" % (r["key"], f))
|
||||
q = oas.params(path, op, "query")
|
||||
for f in r["query"]:
|
||||
if f not in q:
|
||||
fail(provider, "field", "%s sends query parameter %s, absent from the spec" % (r["key"], f))
|
||||
return covered
|
||||
|
||||
|
||||
def check_coverage(provider, oas, covered, strip="", path_filter=lambda p: True):
|
||||
excl = EXCLUSIONS.get(provider, {})
|
||||
n = 0
|
||||
for m, p, _op in oas.ops():
|
||||
if not path_filter(p):
|
||||
continue
|
||||
n += 1
|
||||
if (m, p) in covered or (m, p) in excl:
|
||||
continue
|
||||
fail(provider, "coverage", "spec operation %s %s is neither in the table nor excluded" % (m, p))
|
||||
return n
|
||||
|
||||
|
||||
def check_fields(provider, fields, oas_list):
|
||||
for schema, flist in fields:
|
||||
want = [x for x in flist.split(",") if x]
|
||||
ok = False
|
||||
for oas in oas_list:
|
||||
schemas = oas.d.get("components", {}).get("schemas", {})
|
||||
if schema in schemas:
|
||||
have = oas.reachable({"$ref": "#/components/schemas/" + schema})
|
||||
missing = [f for f in want if f not in have]
|
||||
for f in missing:
|
||||
fail(provider, "schema", "response field %s is not reachable from schema %s" % (f, schema))
|
||||
ok = True
|
||||
break
|
||||
if not ok:
|
||||
fail(provider, "schema", "schema %s not found in the spec" % schema)
|
||||
|
||||
|
||||
def summarize(provider, rows, ops, kind):
|
||||
nb = sum(len(r["body"]) + len(r["query"]) for r in rows)
|
||||
print(" %-11s %-8s rows=%-3d fields=%-4d source ops=%s" % (provider, kind, len(rows), nb, ops))
|
||||
|
||||
|
||||
# ------------------------------------------------------------- providers ----
|
||||
def stability():
|
||||
rows, fields = rows_of("stability")
|
||||
v2 = Oas(json.loads(fetch("https://docs.stability.ai/openapi.json")))
|
||||
v1 = Oas(json.loads(fetch("https://docs.stability.ai/openapi-v1.json")))
|
||||
c2 = check_rows_against_oas("stability", rows, v2, only=lambda r: r["path"].startswith("/v2beta"))
|
||||
c1 = check_rows_against_oas("stability", rows, v1, only=lambda r: r["path"].startswith("/v1"))
|
||||
n = check_coverage("stability", v2, c2) + check_coverage("stability", v1, c1)
|
||||
check_fields("stability", fields, [v2, v1])
|
||||
summarize("stability", rows, n, "openapi")
|
||||
|
||||
|
||||
def bfl():
|
||||
rows, fields = rows_of("bfl")
|
||||
oas = Oas(json.loads(fetch("https://api.bfl.ai/openapi.json")))
|
||||
cov = check_rows_against_oas("bfl", rows, oas)
|
||||
n = check_coverage("bfl", oas, cov)
|
||||
check_fields("bfl", fields, [oas])
|
||||
summarize("bfl", rows, n, "openapi")
|
||||
|
||||
|
||||
def replicate():
|
||||
rows, fields = rows_of("replicate")
|
||||
oas = Oas(json.loads(fetch("https://api.replicate.com/openapi.json")))
|
||||
servers = [s.get("url") for s in oas.d.get("servers", [])]
|
||||
if servers != ["https://api.replicate.com/v1"]:
|
||||
fail("replicate", "row", "spec server url changed: %s (kHost assumes https://api.replicate.com/v1)" % servers)
|
||||
cov = check_rows_against_oas("replicate", rows, oas)
|
||||
n = check_coverage("replicate", oas, cov)
|
||||
check_fields("replicate", fields, [oas])
|
||||
summarize("replicate", rows, n, "openapi")
|
||||
|
||||
|
||||
def fal():
|
||||
rows, fields = rows_of("fal")
|
||||
queue = Oas(json.loads(fetch("https://fal.ai/docs/api-reference/inference-api/openapi/queue.json")))
|
||||
plat = Oas(json.loads(fetch("https://api.fal.ai/v1/openapi.json")))
|
||||
cq = check_rows_against_oas("fal", rows, queue, only=lambda r: r["key"].startswith("queue."))
|
||||
cp = check_rows_against_oas("fal", rows, plat, strip="/v1", only=lambda r: r["key"].startswith("platform."))
|
||||
n = check_coverage("fal", queue, cq) + check_coverage(
|
||||
"fal", plat, cp, path_filter=lambda p: not p.startswith(FAL_PLATFORM_SKIP_PREFIXES))
|
||||
skipped = [p for _m, p, _o in plat.ops() if p.startswith(FAL_PLATFORM_SKIP_PREFIXES)]
|
||||
notes.append("fal: %d Platform API operations outside model inference are out of scope (assets, storage, "
|
||||
"serverless, compute, keys, account, organization, workflows)" % len(skipped))
|
||||
check_fields("fal", fields, [queue, plat])
|
||||
summarize("fal", rows, n, "openapi")
|
||||
|
||||
|
||||
def ideogram():
|
||||
rows, fields = rows_of("ideogram")
|
||||
oas = Oas(json.loads(fetch("https://developer.ideogram.ai/v2/openapi.json")))
|
||||
cov = check_rows_against_oas("ideogram", rows, oas)
|
||||
n = check_coverage("ideogram", oas, cov)
|
||||
check_fields("ideogram", fields, [oas])
|
||||
summarize("ideogram", rows, n, "openapi")
|
||||
|
||||
|
||||
def recraft():
|
||||
rows, fields = rows_of("recraft")
|
||||
oas = Oas(json.loads(fetch("https://www.recraft.ai/docs/api-reference/openapi.json")))
|
||||
cov = check_rows_against_oas("recraft", rows, oas)
|
||||
n = check_coverage("recraft", oas, cov)
|
||||
check_fields("recraft", fields, [oas])
|
||||
summarize("recraft", rows, n, "openapi")
|
||||
|
||||
|
||||
def higgsfield():
|
||||
rows, fields = rows_of("higgsfield")
|
||||
oas = Oas(json.loads(fetch("https://docs.higgsfield.ai/docs/openapi.json")))
|
||||
cov = check_rows_against_oas("higgsfield", rows, oas)
|
||||
n = check_coverage("higgsfield", oas, cov)
|
||||
check_fields("higgsfield", fields, [oas])
|
||||
summarize("higgsfield", rows, n, "openapi")
|
||||
|
||||
|
||||
# ------------------------------------------------------- docs-embedded ----
|
||||
def md_openapi(url):
|
||||
text = fetch(url)
|
||||
m = re.search(r"```json\n(.*?)\n```", text, re.S)
|
||||
return Oas(json.loads(m.group(1)))
|
||||
|
||||
|
||||
def leonardo():
|
||||
rows, fields = rows_of("leonardo")
|
||||
pages = {"createGeneration": "creategeneration", "createSyncGeneration": "createsyncgeneration", "getModels": "getmodels"}
|
||||
docs = [md_openapi("https://docs.leonardo.ai/reference/%s.md" % p) for p in pages.values()]
|
||||
covered = set()
|
||||
n = 0
|
||||
for oas in docs:
|
||||
for m, p, _op in oas.ops():
|
||||
n += 1
|
||||
covered.add((m, "/v2" + p))
|
||||
prose = PROSE_ONLY["leonardo"]
|
||||
for r in rows:
|
||||
if r["key"] in prose:
|
||||
continue
|
||||
hit = None
|
||||
for oas in docs:
|
||||
op = oas.op(r["method"], r["path"][len("/v2"):])
|
||||
if op is not None:
|
||||
hit = (oas, op)
|
||||
if hit is None:
|
||||
fail("leonardo", "row", "%s %s (%s) is not in the v2 reference" % (r["method"], r["path"], r["key"]))
|
||||
continue
|
||||
oas, op = hit
|
||||
if r["op"] and op.get("operationId") != r["op"]:
|
||||
fail("leonardo", "row", "%s operation id %s != %s" % (r["key"], r["op"], op.get("operationId")))
|
||||
kinds, have = oas.body(op)
|
||||
if r["body"] and not (set(r["body"]) <= set(have) | {"model", "parameters", "public"}):
|
||||
fail("leonardo", "field", "%s body fields %s" % (r["key"], r["body"]))
|
||||
for key in sorted(prose):
|
||||
notes.append("leonardo: row %s is UNVERIFIED against a spec (v1 reference pages were removed); it is "
|
||||
"documented in the quick start (POST /v1/generations) and FAQ prose (GET generation by id)" % key)
|
||||
quick = fetch("https://docs.leonardo.ai/docs/getting-started.md")
|
||||
if "api/rest/v1/generations" not in quick:
|
||||
fail("leonardo", "row", "quick start no longer shows POST /api/rest/v1/generations")
|
||||
summarize("leonardo", rows, n, "docs")
|
||||
|
||||
|
||||
MINIMAX_PAGES = ["video-generation-t2v", "video-generation-i2v", "video-generation-fl2v", "video-generation-s2v",
|
||||
"video-generation-query", "video-generation-v2-create", "video-generation-v2-query",
|
||||
"video-generation-v2-list", "video-generation-v2-delete"]
|
||||
MINIMAX_FILE_PAGE = "file-management-retrieve"
|
||||
MINIMAX_UNSUPPORTED_PAGES = {
|
||||
"video-generation-v2-h3-context-ir": "H3-Context-IR task (gap)",
|
||||
"video-generation-v2-regeneration": "video regeneration task (gap)",
|
||||
"video-agent-create": "Video Agent templates (gap)",
|
||||
"video-agent-query": "Video Agent templates (gap)",
|
||||
"video-generation-download": "download helper page, covered by files.retrieve",
|
||||
}
|
||||
|
||||
|
||||
def minimax():
|
||||
rows, _fields = rows_of("minimax")
|
||||
base = "https://platform.minimax.io/docs/api-reference/"
|
||||
texts = {p: fetch(base + p + ".md") for p in MINIMAX_PAGES + [MINIMAX_FILE_PAGE]}
|
||||
heads = set()
|
||||
page_text = {}
|
||||
for p, t in texts.items():
|
||||
for m in re.finditer(r"^````yaml \S+ (get|post|put|patch|delete) (/\S+)$", t, re.M | re.I):
|
||||
heads.add((m.group(1).upper(), m.group(2)))
|
||||
page_text.setdefault((m.group(1).upper(), m.group(2)), "")
|
||||
page_text[(m.group(1).upper(), m.group(2))] += t
|
||||
for r in rows:
|
||||
key = (r["method"], r["path"])
|
||||
if key not in heads:
|
||||
fail("minimax", "row", "%s %s (%s) is not an operation in the API reference pages" % (r["method"], r["path"], r["key"]))
|
||||
continue
|
||||
t = page_text[key]
|
||||
for f in r["body"]:
|
||||
if not re.search(r"^\s+%s:\s*$" % re.escape(f), t, re.M):
|
||||
fail("minimax", "field", "%s body field %s not found in the reference" % (r["key"], f))
|
||||
for f in r["query"]:
|
||||
if not re.search(r"- name: %s\b" % re.escape(f), t):
|
||||
fail("minimax", "field", "%s query parameter %s not found in the reference" % (r["key"], f))
|
||||
declared = {(r["method"], r["path"]) for r in rows}
|
||||
for op in sorted(heads):
|
||||
if op not in declared and op not in EXCLUSIONS["minimax"]:
|
||||
fail("minimax", "coverage", "reference operation %s %s is neither in the table nor excluded" % op)
|
||||
index = fetch("https://platform.minimax.io/docs/llms.txt")
|
||||
for page in re.findall(r"api-reference/((?:video-generation|video-agent)[a-z0-9-]*)\.md", index):
|
||||
if page in texts or page in MINIMAX_UNSUPPORTED_PAGES:
|
||||
continue
|
||||
fail("minimax", "coverage", "index lists reference page %s that is neither checked nor excluded" % page)
|
||||
for page, why in sorted(MINIMAX_UNSUPPORTED_PAGES.items()):
|
||||
notes.append("minimax: reference page %s not implemented: %s" % (page, why))
|
||||
summarize("minimax", rows, len(heads), "docs")
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- SDK ----
|
||||
def npm_tarball(pkg):
|
||||
meta = json.loads(fetch("https://registry.npmjs.org/%s/latest" % pkg))
|
||||
data = fetch(meta["dist"]["tarball"], binary=True)
|
||||
files = {}
|
||||
with tarfile.open(fileobj=io.BytesIO(data)) as tf:
|
||||
for m in tf.getmembers():
|
||||
if m.isfile() and m.name.startswith("package/") and m.size < 4_000_000:
|
||||
files[m.name[len("package/"):]] = tf.extractfile(m).read().decode("utf-8", "replace")
|
||||
return meta["version"], files
|
||||
|
||||
|
||||
def sdk_ops(files, prefix, js_ext=".js"):
|
||||
"""(METHOD, normalized path) -> resource file, from `_client.<verb>(...)` calls."""
|
||||
out = {}
|
||||
pat = re.compile(r"_client\.(get|post|put|patch|delete|getAPIList)\(\s*(?:\(0,\s*path_1\.path\)\s*`([^`]*)`|'([^']*)'|`([^`]*)`)")
|
||||
for name, text in files.items():
|
||||
if not name.startswith(prefix) or not name.endswith(js_ext) or name.endswith(".d.ts"):
|
||||
continue
|
||||
for m in pat.finditer(text):
|
||||
verb = m.group(1)
|
||||
p = m.group(2) or m.group(3) or m.group(4)
|
||||
p = re.sub(r"\$\{[^}]*\}", "{id}", p)
|
||||
out[("GET" if verb == "getAPIList" else verb.upper(), p)] = name
|
||||
return out
|
||||
|
||||
|
||||
def norm(path):
|
||||
return re.sub(r"\{[^}]*\}", "{id}", path)
|
||||
|
||||
|
||||
def runway_fields(text):
|
||||
out = []
|
||||
for m in re.finditer(r"export declare namespace (\w+Params) \{\n(.*?)\n\}\n", text, re.S):
|
||||
out += re.findall(r"^ (\w+)\??:", m.group(2), re.M)
|
||||
for m in re.finditer(r"export interface (\w+Params) \{\n(.*?)\n\}\n", text, re.S):
|
||||
out += re.findall(r"^ (\w+)\??:", m.group(2), re.M)
|
||||
return set(out)
|
||||
|
||||
|
||||
def luma_fields(text):
|
||||
out = []
|
||||
for m in re.finditer(r"export interface (\w+Params) \{\n(.*?)\n\}\n", text, re.S):
|
||||
out += re.findall(r"^ (\w+)\??:", m.group(2), re.M)
|
||||
return set(out)
|
||||
|
||||
|
||||
def sdk_check(provider, pkg, rows, prefix, base_strip, field_fn):
|
||||
version, files = npm_tarball(pkg)
|
||||
ops = sdk_ops(files, prefix)
|
||||
notes.append("%s: official SDK %s %s (%d operations found)" % (provider, pkg, version, len(ops)))
|
||||
declared = set()
|
||||
for r in rows:
|
||||
path = r["path"]
|
||||
key = (r["method"], norm(path))
|
||||
declared.add(key)
|
||||
if key not in ops:
|
||||
fail(provider, "row", "%s %s (%s) is not called by the SDK" % (r["method"], path, r["key"]))
|
||||
continue
|
||||
if r["body"] and r["body"] != ["*"]:
|
||||
js = ops[key]
|
||||
dts = files.get(js[:-3] + ".d.ts", "")
|
||||
have = field_fn(dts)
|
||||
for f in r["body"]:
|
||||
if f not in have:
|
||||
fail(provider, "field", "%s sends body field %s, absent from %s" % (r["key"], f, js[:-3] + ".d.ts"))
|
||||
for key in sorted(ops):
|
||||
if key not in declared and key not in EXCLUSIONS.get(provider, {}):
|
||||
fail(provider, "coverage", "SDK operation %s %s is neither in the table nor excluded" % key)
|
||||
return version, ops, files
|
||||
|
||||
|
||||
def runway():
|
||||
rows, fields = rows_of("runway")
|
||||
version, ops, files = sdk_check("runway", "@runwayml/sdk", rows, "resources/", "", runway_fields)
|
||||
tasks = files.get("resources/tasks.d.ts", "")
|
||||
for schema, flist in fields:
|
||||
for f in [x for x in flist.split(",") if x]:
|
||||
if not re.search(r"\b%s\??:" % re.escape(f), tasks):
|
||||
fail("runway", "schema", "task field %s not found in resources/tasks.d.ts" % f)
|
||||
for line in ("X-Runway-Version",):
|
||||
if not any(line in t for n, t in files.items() if n.endswith("client.js")):
|
||||
fail("runway", "row", "SDK no longer sends %s" % line)
|
||||
m = re.search(r"runwayVersion = '([^']+)'", files.get("client.js", ""))
|
||||
notes.append("runway: SDK default X-Runway-Version %s (table assumes 2024-11-06)" % (m.group(1) if m else "?"))
|
||||
if m and m.group(1) != "2024-11-06":
|
||||
fail("runway", "row", "SDK default X-Runway-Version is %s, RunwayClient::kApiVersion says 2024-11-06" % m.group(1))
|
||||
summarize("runway", rows, len(ops), "sdk")
|
||||
|
||||
|
||||
def luma():
|
||||
rows, fields = rows_of("luma")
|
||||
version, ops, files = sdk_check("luma", "lumaai", rows, "resources/", "", luma_fields)
|
||||
gen = files.get("resources/generations/generations.d.ts", "")
|
||||
for schema, flist in fields:
|
||||
for f in [x for x in flist.split(",") if x]:
|
||||
if not any(re.search(r"\b%s\??:" % re.escape(f), t) for n, t in files.items() if n.endswith(".d.ts")):
|
||||
fail("luma", "schema", "field %s not found in the SDK types" % f)
|
||||
if "dream-machine/v1" not in files.get("index.js", ""):
|
||||
fail("luma", "row", "SDK base URL is no longer .../dream-machine/v1")
|
||||
if "'queued' | 'dreaming' | 'completed' | 'failed'" not in gen:
|
||||
fail("luma", "schema", "generation states changed in the SDK types")
|
||||
summarize("luma", rows, len(ops), "sdk")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ probe ----
|
||||
def kling():
|
||||
rows, _f = rows_of("kling")
|
||||
bad = 0
|
||||
for r in rows:
|
||||
url = "https://api-singapore.klingai.com" + r["path"].replace("{id}", "0")
|
||||
data = b"{}" if r["method"] == "POST" else None
|
||||
req = urllib.request.Request(url, data=data, method=r["method"], headers=dict(UA, **{"Content-Type": "application/json"}))
|
||||
try:
|
||||
urllib.request.urlopen(req, timeout=30)
|
||||
code = 200
|
||||
except urllib.error.HTTPError as e:
|
||||
code = e.code
|
||||
except Exception as e: # network down: do not pretend
|
||||
fail("kling", "row", "probe of %s failed: %s" % (r["path"], e))
|
||||
continue
|
||||
if code == 404:
|
||||
fail("kling", "row", "%s %s answered 404 on the documented host" % (r["method"], r["path"]))
|
||||
else:
|
||||
bad += 0
|
||||
notes.append("kling: UNVERIFIED. No spec or SDK source exists; %d paths only probed (not 404 unauthenticated). "
|
||||
"Request and response fields come from community sources." % len(rows))
|
||||
summarize("kling", rows, "-", "probe")
|
||||
|
||||
|
||||
CHECKS = {"stability": stability, "bfl": bfl, "replicate": replicate, "fal": fal, "ideogram": ideogram,
|
||||
"recraft": recraft, "leonardo": leonardo, "runway": runway, "luma": luma, "minimax": minimax,
|
||||
"kling": kling, "higgsfield": higgsfield}
|
||||
|
||||
|
||||
def main():
|
||||
wanted = sys.argv[1:] or list(CHECKS)
|
||||
print("media call tables against vendor sources")
|
||||
for name in wanted:
|
||||
if name not in CHECKS:
|
||||
print("unknown provider", name)
|
||||
return 2
|
||||
try:
|
||||
CHECKS[name]()
|
||||
except Exception as e: # a vendor source that cannot be read is a failure, loudly
|
||||
fail(name, "source", "%s: %s" % (type(e).__name__, e))
|
||||
for n in notes:
|
||||
print(" note:", n)
|
||||
if problems:
|
||||
print("\nFAILED (%d)" % len(problems))
|
||||
for p in problems:
|
||||
print(" ", p)
|
||||
return 1
|
||||
print("\nOK: every declared row, field and schema matches its source; every source operation is covered or excluded")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
// Routes: GET /ai/media (page), /status, /calls (no credential needed)
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support/Assert.hpp"
|
||||
|
||||
using namespace molo::example::test;
|
||||
|
||||
int main() {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
for (const char* v : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "RUNWAYML_API_SECRET", "LUMA_API_KEY",
|
||||
"REPLICATE_API_TOKEN", "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY",
|
||||
"MINIMAX_API_KEY", "KLING_ACCESS_KEY", "KLING_SECRET_KEY", "HF_API_KEY_ID", "HF_API_KEY_SECRET"})
|
||||
unsetenv(v);
|
||||
Fixture f;
|
||||
auto page = f.client->get("/ai/media");
|
||||
check(page.status == 200, "page 200");
|
||||
check(contains(page.headers.get("Content-Type"), "text/html"), "html");
|
||||
check(contains(page.body, "Media generation") && contains(page.body, "viewport"), "title and mobile viewport");
|
||||
check(contains(page.body, "/embedded/ai_media.js"), "script wired");
|
||||
check(f.client->post("/ai/media", "{}", "application/json").status == 405, "post page is 405");
|
||||
|
||||
auto st = f.client->get("/ai/media/status");
|
||||
check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without keys");
|
||||
check(!contains(st.body, "\"configured\":true"), "no provider is configured");
|
||||
for (const char* env : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "LUMA_API_KEY", "REPLICATE_API_TOKEN",
|
||||
"FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY"})
|
||||
check(contains(st.body, env), std::string("status names ") + env);
|
||||
check(contains(st.body, "Midjourney") && contains(st.body, "No public API"), "Midjourney is listed as unavailable with the reason");
|
||||
check(contains(st.body, "Pika"), "Pika is listed as unavailable");
|
||||
check(contains(st.body, "\"evidence\":\"probe-only\"") && contains(st.body, "\"evidence\":\"openapi\""), "evidence levels shown");
|
||||
check(contains(st.body, "flux-2-pro") && contains(st.body, "fal-ai/flux/dev"), "models are listed");
|
||||
check(contains(st.body, "\"max_poll_attempts\":240"), "caps are shown");
|
||||
check(f.client->post("/ai/media/status", "{}", "application/json").status == 405, "post status is 405");
|
||||
|
||||
auto all = f.client->get("/ai/media/calls");
|
||||
check(all.status == 200 && contains(all.body, "\"provider\":\"stability\"") && contains(all.body, "\"provider\":\"higgsfield\""),
|
||||
"calls lists every provider");
|
||||
auto bfl = f.client->get("/ai/media/calls?provider=bfl");
|
||||
check(bfl.status == 200 && contains(bfl.body, "/v1/flux-2-pro") && contains(bfl.body, "\"effect\":\"paid\"") &&
|
||||
contains(bfl.body, "/v1/credits"),
|
||||
"bfl call table");
|
||||
auto fal = f.client->get("/ai/media/calls?provider=fal");
|
||||
check(fal.status == 200 && contains(fal.body, "queue.fal.run") == false && contains(fal.body, "/{owner}/{app}/requests/{request_id}/status"),
|
||||
"fal call table");
|
||||
check(f.client->get("/ai/media/calls?provider=midjourney").status == 422, "unknown provider is 422");
|
||||
return finish("ai/media/page");
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// Routes: POST /ai/media/* (405, 422, dry runs, 503 without a credential)
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support/Assert.hpp"
|
||||
|
||||
using namespace molo::example::test;
|
||||
|
||||
int main() {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
for (const char* v : {"STABILITY_API_KEY", "BFL_API_KEY", "RUNWAY_API_KEY", "RUNWAYML_API_SECRET", "LUMA_API_KEY",
|
||||
"REPLICATE_API_TOKEN", "FAL_KEY", "IDEOGRAM_API_KEY", "RECRAFT_API_TOKEN", "LEONARDO_API_KEY",
|
||||
"MINIMAX_API_KEY", "KLING_ACCESS_KEY", "KLING_SECRET_KEY", "HF_API_KEY_ID", "HF_API_KEY_SECRET"})
|
||||
unsetenv(v);
|
||||
Fixture f;
|
||||
auto post = [&](const std::string& route, const std::string& body) {
|
||||
return f.client->post("/ai/media/" + route, body, "application/json");
|
||||
};
|
||||
const char* posts[] = {"image", "video", "poll", "wait", "call", "account", "fetch", "webhook"};
|
||||
for (const char* r : posts) {
|
||||
std::string route = std::string("/ai/media/") + r;
|
||||
check(f.client->get(route).status == 405, route + " get is 405");
|
||||
check(post(r, "not json").status == 422, route + " bad json is 422");
|
||||
check(post(r, "[1]").status == 422, route + " non-object is 422");
|
||||
check(post(r, "{}").status == 422, route + " empty body is 422");
|
||||
}
|
||||
check(f.client->get("/ai/media/job").status == 405, "job get is 405");
|
||||
check(f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "not json").status == 422, "job bad json is 422");
|
||||
|
||||
// 422: input rules run before anything else
|
||||
check(post("image", "{\"provider\":\"midjourney\",\"prompt\":\"x\"}").status == 422, "midjourney has no API");
|
||||
check(post("image", "{\"provider\":\"minimax\",\"prompt\":\"x\"}").status == 422, "minimax has no image generation");
|
||||
check(post("video", "{\"provider\":\"stability\",\"prompt\":\"x\"}").status == 422, "stability has no video generation");
|
||||
check(post("image", "{\"provider\":\"bfl\"}").status == 422, "prompt is required");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":0}").status == 422, "count 0");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":9}").status == 422, "count over 8");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"width\":512}").status == 422, "width without height");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"width\":9000,\"height\":9000}").status == 422, "dimension over the cap");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"extra\":{\"bogus\":1}}").status == 422, "field not in the call table");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"model\":\"flux-9000\"}").status == 422, "model not in the call table");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"count\":2}").status == 422, "bfl takes one image per call");
|
||||
check(post("image", "{\"provider\":\"stability\",\"prompt\":\"x\",\"width\":512,\"height\":512}").status == 422, "stability sizes by aspect ratio");
|
||||
check(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\"}").status == 422, "fal needs an endpoint id");
|
||||
check(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\",\"model\":\"a/../b\"}").status == 422, "bad endpoint id");
|
||||
check(post("image", "{\"provider\":\"replicate\",\"prompt\":\"x\",\"model\":\"nope\"}").status == 422, "replicate needs owner/name");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"seed\":\"a\"}").status == 422, "seed must be a number");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"confirm\":\"yes\"}").status == 422, "confirm must be a boolean");
|
||||
check(post("video", "{\"provider\":\"runway\"}").status == 422, "video needs a prompt or an image");
|
||||
check(post("video", "{\"provider\":\"runway\",\"prompt\":\"x\",\"duration_seconds\":999}").status == 422, "duration over the cap");
|
||||
check(post("video", "{\"provider\":\"minimax\",\"prompt\":\"x\",\"aspect_ratio\":\"16:9\"}").status == 422, "minimax v1 has no aspect ratio");
|
||||
check(post("poll", "{\"ticket\":{\"provider\":\"bfl\"}}").status == 422, "ticket needs an id");
|
||||
check(post("poll", "{\"ticket\":{\"provider\":\"nope\",\"id\":\"1\"}}").status == 422, "ticket with an unknown provider");
|
||||
check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_attempts\":31}").status == 422, "demo attempt cap");
|
||||
check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_wait_seconds\":61}").status == 422, "demo wait cap");
|
||||
check(post("wait", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"},\"max_attempts\":0}").status == 422, "attempts under 1");
|
||||
check(post("call", "{\"provider\":\"bfl\",\"key\":\"nope\"}").status == 422, "unknown call key");
|
||||
check(post("call", "{\"provider\":\"bfl\",\"key\":\"get_result\",\"query\":{\"nope\":\"1\"}}").status == 422, "unknown query parameter");
|
||||
check(post("call", "{\"provider\":\"replicate\",\"key\":\"predictions.get\",\"path\":{\"prediction_id\":\"../x\"}}").status == 422, "path traversal");
|
||||
check(post("call", "{\"provider\":\"replicate\",\"key\":\"predictions.get\"}").status == 422, "missing path argument");
|
||||
check(post("call", "{\"provider\":\"bfl\",\"key\":\"credits\",\"body\":{\"x\":1}}").status == 422, "a GET takes no body");
|
||||
check(post("call", "{\"provider\":\"stability\",\"key\":\"stable-image.upscale.fast\",\"files\":[{\"field\":\"image\",\"data_base64\":\"@@\"}]}").status == 422,
|
||||
"bad base64 file");
|
||||
check(post("call", "{\"provider\":\"stability\",\"key\":\"stable-image.upscale.fast\",\"files\":[{\"field\":\"picture\",\"data_base64\":\"aGk=\"}]}").status == 422,
|
||||
"file field not in the table");
|
||||
check(post("fetch", "{\"url\":\"http://example.com/a.png\"}").status == 422, "fetch needs https");
|
||||
check(post("fetch", "{\"url\":\"https://user:pw@example.com/a.png\"}").status == 422, "fetch refuses embedded credentials");
|
||||
check(post("webhook", "{\"secret\":\"s\",\"webhook_id\":\"i\",\"timestamp\":\"1\",\"signature\":\"v1,x\"}").status == 422, "webhook needs a payload");
|
||||
|
||||
// dry runs: paid or resource-changing routes show the exact request
|
||||
auto dry = post("image", "{\"provider\":\"bfl\",\"prompt\":\"a cat\",\"width\":1024,\"height\":768}");
|
||||
check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "/v1/flux-2-pro") &&
|
||||
contains(dry.body, "\"effect\":\"paid\"") && contains(dry.body, "\"prompt\":\"a cat\""),
|
||||
"image is a dry run without confirm");
|
||||
check(contains(post("image", "{\"provider\":\"stability\",\"prompt\":\"a cat\",\"model\":\"ultra\",\"aspect_ratio\":\"16:9\"}").body,
|
||||
"/v2beta/stable-image/generate/ultra"),
|
||||
"stability dry run shows the endpoint");
|
||||
check(contains(post("image", "{\"provider\":\"fal\",\"prompt\":\"x\",\"model\":\"fal-ai/flux/dev\"}").body, "\"target\":\"/fal-ai/flux/dev\""),
|
||||
"fal dry run shows the queue target");
|
||||
check(contains(post("image", "{\"provider\":\"replicate\",\"prompt\":\"x\",\"model\":\"black-forest-labs/flux-schnell\"}").body,
|
||||
"/models/black-forest-labs/flux-schnell/predictions"),
|
||||
"replicate dry run shows the prediction target");
|
||||
check(contains(post("image", "{\"provider\":\"leonardo\",\"prompt\":\"x\",\"model\":\"flux-dev\"}").body, "\"public\":false"),
|
||||
"leonardo never publishes by default");
|
||||
check(contains(post("image", "{\"provider\":\"ideogram\",\"prompt\":\"x\",\"model\":\"ideogram-4\"}").body, "/v2/image/generate/ideogram-4"),
|
||||
"ideogram dry run");
|
||||
check(contains(post("image", "{\"provider\":\"recraft\",\"prompt\":\"x\",\"width\":1024,\"height\":1024}").body, "\"size\":\"1024x1024\""),
|
||||
"recraft dry run");
|
||||
check(contains(post("video", "{\"provider\":\"runway\",\"prompt\":\"waves\",\"duration_seconds\":5}").body, "/v1/text_to_video"),
|
||||
"runway video dry run");
|
||||
check(contains(post("video", "{\"provider\":\"luma\",\"prompt\":\"waves\",\"duration_seconds\":5}").body, "\"duration\":\"5s\""),
|
||||
"luma video dry run");
|
||||
check(contains(post("video", "{\"provider\":\"kling\",\"prompt\":\"waves\"}").body, "/v1/videos/text2video"), "kling video dry run");
|
||||
check(contains(post("video", "{\"provider\":\"higgsfield\",\"prompt\":\"waves\",\"model\":\"minimax/hailuo-2.3/standard/text-to-video\"}").body,
|
||||
"/minimax/hailuo-2.3/standard/text-to-video"),
|
||||
"higgsfield video dry run");
|
||||
check(contains(post("call", "{\"provider\":\"bfl\",\"key\":\"flux-dev\",\"body\":{\"prompt\":\"x\"}}").body, "\"dry_run\":true"),
|
||||
"a paid table call is a dry run");
|
||||
check(contains(post("call", "{\"provider\":\"replicate\",\"key\":\"files.delete\",\"path\":{\"file_id\":\"f1\"}}").body, "\"effect\":\"delete\""),
|
||||
"a destructive table call is a dry run");
|
||||
check(contains(post("call", "{\"provider\":\"bfl\",\"key\":\"delete_finetune\",\"body\":{\"finetune_id\":\"my-ft\"}}").body, "\"dry_run\":true"),
|
||||
"delete_finetune is a dry run");
|
||||
auto cancel = f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "{\"ticket\":{\"provider\":\"luma\",\"id\":\"g1\",\"kind\":\"video\"}}");
|
||||
check(cancel.status == 200 && contains(cancel.body, "\"dry_run\":true"), "cancel is a dry run");
|
||||
|
||||
// pure routes
|
||||
auto hook = post("webhook",
|
||||
"{\"secret\":\"whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw\",\"webhook_id\":\"msg_p5jXN8AQM9LWM0D4loKWxJek\","
|
||||
"\"timestamp\":\"1614265330\",\"signature\":\"v1,g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=\","
|
||||
"\"payload\":\"{\\\"test\\\": 2432232314}\"}");
|
||||
check(hook.status == 200 && contains(hook.body, "\"valid\":false"), "a vector from 2021 is stale now, so it is rejected");
|
||||
|
||||
// 503: reads and confirmed calls need the credential
|
||||
auto acct = post("account", "{\"provider\":\"bfl\"}");
|
||||
check(acct.status == 503 && contains(acct.body, "media_not_configured"), "account without key is 503");
|
||||
check(post("account", "{\"provider\":\"minimax\"}").status == 422, "minimax has no free account read");
|
||||
check(post("account", "{\"provider\":\"midjourney\"}").status == 422, "unknown provider account");
|
||||
check(post("call", "{\"provider\":\"bfl\",\"key\":\"credits\"}").status == 503, "read call without key is 503");
|
||||
check(post("call", "{\"provider\":\"fal\",\"key\":\"platform.meta\"}").status == 503, "fal platform read without key is 503");
|
||||
check(post("call", "{\"provider\":\"bfl\",\"key\":\"flux-dev\",\"body\":{\"prompt\":\"x\"},\"confirm\":true}").status == 503,
|
||||
"confirmed paid call without key is 503");
|
||||
check(post("image", "{\"provider\":\"bfl\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed image without key is 503");
|
||||
check(post("video", "{\"provider\":\"runway\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "confirmed video without key is 503");
|
||||
check(post("video", "{\"provider\":\"kling\",\"prompt\":\"x\",\"confirm\":true}").status == 503, "kling without keys is 503");
|
||||
check(post("poll", "{\"ticket\":{\"provider\":\"bfl\",\"id\":\"1\"}}").status == 503, "poll without key is 503");
|
||||
check(post("wait", "{\"ticket\":{\"provider\":\"fal\",\"id\":\"1\",\"meta\":{\"endpoint\":\"fal-ai/flux/dev\"}}}").status == 503, "wait without key is 503");
|
||||
check(f.client->request(molo::example::fw::Method::Delete, "/ai/media/job", "{\"ticket\":{\"provider\":\"luma\",\"id\":\"g1\"},\"confirm\":true}").status == 503,
|
||||
"confirmed cancel without key is 503");
|
||||
return finish("ai/media/routes");
|
||||
}
|
||||
Vendored
+57
@@ -0,0 +1,57 @@
|
||||
#pragma once
|
||||
|
||||
// Shared rig for the media tests. A MockTransport only RECORDS the requests
|
||||
// the real clients build; the few replies it serves are labelled FIXTURE
|
||||
// (verbatim vendor documentation) or EMPTY (no content, the assertion is on
|
||||
// our request). Nothing here verifies vendor behavior: see
|
||||
// src/vendors/media/SPEC.md "Unverified".
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "llm/error.h"
|
||||
#include "vendors/media/common/job.h"
|
||||
#include "vendors/media/common/media_service.h"
|
||||
|
||||
namespace media_test {
|
||||
|
||||
using molo::clients::ClientResponse;
|
||||
using molo::clients::HeaderMap;
|
||||
using molo::clients::Method;
|
||||
using molo::clients::transports::MockExpectation;
|
||||
using molo::clients::transports::MockTransport;
|
||||
using molo::llm::ApiError;
|
||||
using molo::media::Json;
|
||||
|
||||
inline ClientResponse reply(const std::string& body, int status = 200,
|
||||
const std::string& type = "application/json") {
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", type);
|
||||
return ClientResponse(status, h, body);
|
||||
}
|
||||
|
||||
// EMPTY: a 200 with `{}`; used only to let the real client finish a request
|
||||
// whose OUTGOING shape the test inspects.
|
||||
inline ClientResponse empty_ok() { return reply("{}"); }
|
||||
|
||||
inline bool has(const std::string& hay, const std::string& needle) { return hay.find(needle) != std::string::npos; }
|
||||
|
||||
// Allows a path to answer several times.
|
||||
inline void serve(MockTransport* t, Method m, const std::string& path, ClientResponse r, int times = 8) {
|
||||
MockExpectation e;
|
||||
e.method = m;
|
||||
e.path = path;
|
||||
e.response = std::move(r);
|
||||
e.times = times;
|
||||
t->expect(e);
|
||||
}
|
||||
|
||||
struct Mock {
|
||||
std::unique_ptr<MockTransport> owned = std::make_unique<MockTransport>();
|
||||
MockTransport* raw = owned.get();
|
||||
std::unique_ptr<molo::clients::Transport> take() { return std::move(owned); }
|
||||
};
|
||||
|
||||
} // namespace media_test
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
// ImageGenerator / VideoGenerator adapters: the exact request each one would
|
||||
// send (plan), loud refusal of options a provider cannot honour, and the
|
||||
// rule that submit() without confirm makes no network call.
|
||||
#include "support.h"
|
||||
#include "vendors/media/bfl/bfl.h"
|
||||
#include "vendors/media/common/registry.h"
|
||||
#include "vendors/media/fal/fal.h"
|
||||
#include "vendors/media/higgsfield/higgsfield.h"
|
||||
#include "vendors/media/ideogram/ideogram.h"
|
||||
#include "vendors/media/kling/kling.h"
|
||||
#include "vendors/media/leonardo/leonardo.h"
|
||||
#include "vendors/media/luma/luma.h"
|
||||
#include "vendors/media/minimax/minimax.h"
|
||||
#include "vendors/media/recraft/recraft.h"
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
#include "vendors/media/runway/runway.h"
|
||||
#include "vendors/media/stability/stability.h"
|
||||
|
||||
using namespace molo::media;
|
||||
using namespace media_test;
|
||||
|
||||
namespace {
|
||||
|
||||
ImageRequest image(const std::string& model = "") {
|
||||
ImageRequest r;
|
||||
r.prompt = "a lighthouse at dusk";
|
||||
r.model = model;
|
||||
return r;
|
||||
}
|
||||
|
||||
VideoRequest video(const std::string& model = "") {
|
||||
VideoRequest r;
|
||||
r.prompt = "waves rolling in";
|
||||
r.model = model;
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string target(const Json& plan) { return plan.find("target")->as_string(); }
|
||||
std::string body_of(const Json& plan) { return plan.find("body") ? plan.find("body")->dump() : ""; }
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(media_adapter, stability_picks_the_endpoint_from_the_model) {
|
||||
ASSERT_EQ(StabilityImageGenerator::key_for(""), std::string("stable-image.generate.core"));
|
||||
ASSERT_EQ(StabilityImageGenerator::key_for("ultra"), std::string("stable-image.generate.ultra"));
|
||||
ASSERT_EQ(StabilityImageGenerator::key_for("sd3.5-large"), std::string("stable-image.generate.sd3"));
|
||||
ASSERT_THROW(StabilityImageGenerator::key_for("dall-e"), ApiError);
|
||||
Mock m;
|
||||
StabilityImageGenerator g(StabilityClient::with(m.take(), "KEY"));
|
||||
ImageRequest r = image("sd3.5-large");
|
||||
r.aspect_ratio = "16:9";
|
||||
r.seed = 42;
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/v2beta/stable-image/generate/sd3"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"16:9\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"model\":\"sd3.5-large\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"seed\":42"));
|
||||
ImageRequest wh = image();
|
||||
wh.width = 512;
|
||||
wh.height = 512;
|
||||
ASSERT_THROW(g.plan(wh), ApiError); // stability sizes by aspect ratio only
|
||||
ImageRequest many = image();
|
||||
many.count = 2;
|
||||
ASSERT_THROW(g.plan(many), ApiError);
|
||||
ImageRequest src = image();
|
||||
src.source_url = "https://example.com/in.png";
|
||||
ASSERT_THROW(g.plan(src), ApiError);
|
||||
ASSERT_THROW(g.submit(image(), false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, bfl_models_and_options) {
|
||||
Mock m;
|
||||
BflImageGenerator g(BflClient::with(m.take(), "KEY"));
|
||||
ImageRequest r = image("flux-kontext-pro");
|
||||
r.aspect_ratio = "1:1";
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/v1/flux-kontext-pro"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"1:1\""));
|
||||
ImageRequest wh = image("flux-kontext-pro"); // kontext takes aspect_ratio, not width / height
|
||||
wh.width = 1024;
|
||||
wh.height = 1024;
|
||||
ASSERT_THROW(g.plan(wh), ApiError);
|
||||
ImageRequest def = image();
|
||||
def.width = 1024;
|
||||
def.height = 768;
|
||||
ASSERT_EQ(target(g.plan(def)), std::string("/v1/flux-2-pro")); // default model
|
||||
ImageRequest many = image();
|
||||
many.count = 3;
|
||||
ASSERT_THROW(g.plan(many), ApiError);
|
||||
ImageRequest unknown = image("flux-9000");
|
||||
ASSERT_THROW(g.plan(unknown), ApiError); // not in the table
|
||||
ASSERT_THROW(g.submit(def, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
BflVideoGenerator v(BflClient::with(std::make_unique<MockTransport>(), "KEY"));
|
||||
VideoRequest vr = video();
|
||||
vr.duration_seconds = 5;
|
||||
vr.aspect_ratio = "16:9";
|
||||
Json vp = v.plan(vr);
|
||||
ASSERT_EQ(target(vp), std::string("/v1/flux-3-video"));
|
||||
ASSERT_TRUE(has(body_of(vp), "\"duration\":5"));
|
||||
}
|
||||
|
||||
TEST(media_adapter, replicate_needs_owner_name_and_maps_common_fields_into_input) {
|
||||
Mock m;
|
||||
ReplicateImageGenerator g(ReplicateClient::with(m.take(), "KEY"));
|
||||
ImageRequest r = image("black-forest-labs/flux-schnell");
|
||||
r.aspect_ratio = "3:2";
|
||||
r.count = 2;
|
||||
r.seed = 9;
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/models/black-forest-labs/flux-schnell/predictions"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"num_outputs\":2"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"aspect_ratio\":\"3:2\""));
|
||||
ASSERT_THROW(g.plan(image()), ApiError);
|
||||
ASSERT_THROW(g.plan(image("flux-schnell")), ApiError);
|
||||
ASSERT_THROW(g.submit(r, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
ReplicateVideoGenerator v(ReplicateClient::with(std::make_unique<MockTransport>(), "KEY"));
|
||||
VideoRequest vr = video("minimax/video-01");
|
||||
vr.image_url = "https://example.com/a.png";
|
||||
Json vp = v.plan(vr);
|
||||
ASSERT_TRUE(has(body_of(vp), "\"image\":\"https://example.com/a.png\""));
|
||||
}
|
||||
|
||||
TEST(media_adapter, fal_takes_any_endpoint_id) {
|
||||
Mock m;
|
||||
FalImageGenerator g(FalQueue::with(m.take(), "id:secret"));
|
||||
ImageRequest r = image("fal-ai/flux/dev");
|
||||
r.width = 1024;
|
||||
r.height = 576;
|
||||
r.count = 2;
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/fal-ai/flux/dev"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"image_size\":{\"height\":576,\"width\":1024}"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"num_images\":2"));
|
||||
ASSERT_THROW(g.plan(image()), ApiError);
|
||||
ASSERT_THROW(g.submit(r, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
FalVideoGenerator v(FalQueue::with(std::make_unique<MockTransport>(), "id:secret"));
|
||||
VideoRequest vr = video("fal-ai/kling-video/v2.1/master/text-to-video");
|
||||
vr.duration_seconds = 5;
|
||||
Json vp = v.plan(vr);
|
||||
ASSERT_EQ(target(vp), std::string("/fal-ai/kling-video/v2.1/master/text-to-video"));
|
||||
ASSERT_TRUE(has(body_of(vp), "\"duration\":\"5\""));
|
||||
}
|
||||
|
||||
TEST(media_adapter, ideogram_models_map_to_rows_and_edit_inputs_are_files) {
|
||||
Mock m;
|
||||
IdeogramImageGenerator g(IdeogramClient::with(m.take(), "KEY"));
|
||||
ASSERT_EQ(IdeogramImageGenerator::key_for(""), std::string("image.generate.ideogram-3"));
|
||||
ImageRequest r = image("ideogram-4");
|
||||
r.count = 2;
|
||||
r.width = 1024;
|
||||
r.height = 1024;
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/v2/image/generate/ideogram-4"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"resolution\":\"1024x1024\""));
|
||||
ImageRequest src = image();
|
||||
src.source_url = "https://example.com/in.png";
|
||||
ASSERT_THROW(g.plan(src), ApiError);
|
||||
ASSERT_THROW(g.plan(image("ideogram-99")), ApiError);
|
||||
ASSERT_THROW(g.submit(image(), false), ApiError);
|
||||
IdeogramVideoGenerator v(IdeogramClient::with(std::make_unique<MockTransport>(), "KEY"));
|
||||
ASSERT_THROW(v.plan(video()), ApiError); // a video model is required
|
||||
Json vp = v.plan(video("kling-3-standard-text-to-video"));
|
||||
ASSERT_EQ(target(vp), std::string("/v2/video/generate/kling-3-standard-text-to-video"));
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, recraft_sizes_by_wxh) {
|
||||
Mock m;
|
||||
RecraftImageGenerator g(RecraftClient::with(m.take(), "KEY"));
|
||||
ImageRequest r = image("recraftv3");
|
||||
r.width = 1024;
|
||||
r.height = 1024;
|
||||
r.count = 2;
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/v1/images/generations"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"size\":\"1024x1024\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"n\":2"));
|
||||
ImageRequest ar = image();
|
||||
ar.aspect_ratio = "16:9";
|
||||
ASSERT_THROW(g.plan(ar), ApiError);
|
||||
ASSERT_THROW(g.submit(r, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, leonardo_wraps_parameters_and_never_publishes) {
|
||||
Mock m;
|
||||
LeonardoImageGenerator g(LeonardoClient::with(m.take(), "KEY"));
|
||||
ImageRequest r = image("flux-dev");
|
||||
r.width = 1024;
|
||||
r.height = 1024;
|
||||
r.count = 4;
|
||||
Json p = g.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/v2/generations"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"model\":\"flux-dev\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"public\":false"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"quantity\":4"));
|
||||
ASSERT_THROW(g.plan(image()), ApiError); // model required
|
||||
ImageRequest ar = image("flux-dev");
|
||||
ar.aspect_ratio = "1:1";
|
||||
ASSERT_THROW(g.plan(ar), ApiError);
|
||||
ASSERT_THROW(g.submit(r, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, runway_chooses_text_or_image_to_video) {
|
||||
Mock m;
|
||||
RunwayVideoGenerator g(RunwayClient::with(m.take(), "KEY"));
|
||||
VideoRequest t = video();
|
||||
t.duration_seconds = 5;
|
||||
t.aspect_ratio = "1280:720";
|
||||
ASSERT_EQ(target(g.plan(t)), std::string("/v1/text_to_video"));
|
||||
ASSERT_TRUE(has(body_of(g.plan(t)), "\"model\":\"gen4.5\""));
|
||||
VideoRequest i = video();
|
||||
i.image_url = "https://example.com/a.png";
|
||||
ASSERT_EQ(target(g.plan(i)), std::string("/v1/image_to_video"));
|
||||
ASSERT_TRUE(has(body_of(g.plan(i)), "\"model\":\"gen4_turbo\""));
|
||||
ASSERT_TRUE(has(body_of(g.plan(i)), "\"promptImage\""));
|
||||
RunwayImageGenerator ig(RunwayClient::with(std::make_unique<MockTransport>(), "KEY"));
|
||||
ImageRequest r = image();
|
||||
r.width = 1280;
|
||||
r.height = 720;
|
||||
Json p = ig.plan(r);
|
||||
ASSERT_EQ(target(p), std::string("/v1/text_to_image"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"ratio\":\"1280:720\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"promptText\""));
|
||||
ASSERT_THROW(g.submit(t, false), ApiError);
|
||||
ASSERT_THROW(ig.cancel(JobTicket{}, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, luma_durations_are_strings_and_keyframes_nest_frames) {
|
||||
Mock m;
|
||||
LumaVideoGenerator g(LumaClient::with(m.take(), "KEY"));
|
||||
VideoRequest v = video();
|
||||
v.duration_seconds = 5;
|
||||
v.image_url = "https://example.com/start.png";
|
||||
v.end_image_url = "https://example.com/end.png";
|
||||
Json p = g.plan(v);
|
||||
ASSERT_EQ(target(p), std::string("/generations/video"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"duration\":\"5s\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"frame0\":{\"type\":\"image\",\"url\":\"https://example.com/start.png\"}"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"frame1\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"model\":\"ray-2\""));
|
||||
LumaImageGenerator ig(LumaClient::with(std::make_unique<MockTransport>(), "KEY"));
|
||||
ImageRequest wh = image();
|
||||
wh.width = 512;
|
||||
wh.height = 512;
|
||||
ASSERT_THROW(ig.plan(wh), ApiError);
|
||||
ASSERT_EQ(target(ig.plan(image())), std::string("/generations/image"));
|
||||
ASSERT_THROW(g.cancel(JobTicket{}, false), ApiError); // cancel deletes: confirm required
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, minimax_refuses_options_v1_does_not_have) {
|
||||
Mock m;
|
||||
MinimaxVideoGenerator g(MinimaxClient::with(m.take(), "KEY"));
|
||||
VideoRequest v = video();
|
||||
v.duration_seconds = 6;
|
||||
v.resolution = "768p";
|
||||
Json p = g.plan(v);
|
||||
ASSERT_EQ(target(p), std::string("/v1/video_generation"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"resolution\":\"768P\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"model\":\"MiniMax-Hailuo-2.3\""));
|
||||
VideoRequest ar = video();
|
||||
ar.aspect_ratio = "16:9";
|
||||
ASSERT_THROW(g.plan(ar), ApiError);
|
||||
VideoRequest sd = video();
|
||||
sd.seed = 5;
|
||||
ASSERT_THROW(g.plan(sd), ApiError);
|
||||
ASSERT_THROW(g.submit(v, false), ApiError);
|
||||
ASSERT_FALSE(g.cancel(JobTicket{}, true)); // not supported, says so without a call
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, kling_and_higgsfield) {
|
||||
auto kc = std::make_shared<KlingClient>("AK", "SK", [] { return std::make_unique<MockTransport>(); });
|
||||
KlingVideoGenerator k(kc);
|
||||
VideoRequest v = video();
|
||||
v.duration_seconds = 5;
|
||||
Json p = k.plan(v);
|
||||
ASSERT_EQ(target(p), std::string("/v1/videos/text2video"));
|
||||
ASSERT_TRUE(has(body_of(p), "\"model_name\":\"kling-v1-6\""));
|
||||
ASSERT_TRUE(has(body_of(p), "\"duration\":\"5\""));
|
||||
VideoRequest i = video();
|
||||
i.image_url = "https://example.com/a.png";
|
||||
ASSERT_EQ(target(k.plan(i)), std::string("/v1/videos/image2video"));
|
||||
VideoRequest sd = video();
|
||||
sd.seed = 1;
|
||||
ASSERT_THROW(k.plan(sd), ApiError);
|
||||
ASSERT_THROW(k.submit(v, false), ApiError);
|
||||
|
||||
Mock m;
|
||||
HiggsfieldImageGenerator h(HiggsfieldClient::with(m.take(), "id", "secret"));
|
||||
Json hp = h.plan(image());
|
||||
ASSERT_EQ(target(hp), std::string("/higgsfield-ai/soul/standard"));
|
||||
ImageRequest wh = image();
|
||||
wh.width = 512;
|
||||
wh.height = 512;
|
||||
ASSERT_THROW(h.plan(wh), ApiError);
|
||||
HiggsfieldVideoGenerator hv(HiggsfieldClient::with(std::make_unique<MockTransport>(), "id", "secret"));
|
||||
ASSERT_THROW(hv.plan(video()), ApiError); // model path required
|
||||
ASSERT_EQ(target(hv.plan(video("kling-video/v2.5-turbo/pro/text-to-video"))),
|
||||
std::string("/kling-video/v2.5-turbo/pro/text-to-video"));
|
||||
ASSERT_THROW(h.cancel(JobTicket{}, false), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_adapter, registry_builds_a_generator_for_every_declared_capability) {
|
||||
MediaRegistry reg;
|
||||
for (const ProviderInfo& p : MediaRegistry::providers()) {
|
||||
if (p.image) {
|
||||
auto g = reg.image(p.id);
|
||||
ASSERT_EQ(g->provider(), p.id);
|
||||
ASSERT_FALSE(g->models().empty());
|
||||
} else {
|
||||
ASSERT_THROW(reg.image(p.id), ApiError);
|
||||
}
|
||||
if (p.video) {
|
||||
auto g = reg.video(p.id);
|
||||
ASSERT_EQ(g->provider(), p.id);
|
||||
ASSERT_FALSE(g->models().empty());
|
||||
} else {
|
||||
ASSERT_THROW(reg.video(p.id), ApiError);
|
||||
}
|
||||
ASSERT_TRUE(reg.service(p.id) != nullptr);
|
||||
}
|
||||
ASSERT_THROW(reg.image("midjourney"), ApiError);
|
||||
ASSERT_THROW(reg.service("pika"), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_adapter, every_default_model_resolves_to_a_row_in_its_table) {
|
||||
// A model named by models() must plan cleanly where the adapter derives the row from it.
|
||||
MediaRegistry reg;
|
||||
ImageRequest r = image();
|
||||
for (const std::string& model : reg.image("bfl")->models()) {
|
||||
r.model = model;
|
||||
r.width = 0;
|
||||
r.height = 0;
|
||||
if (model.rfind("flux-kontext", 0) == 0) r.aspect_ratio = "1:1";
|
||||
else r.aspect_ratio = "";
|
||||
if (model == "flux-3-image") continue; // takes images[], aspect_ratio: covered by the table test
|
||||
r.aspect_ratio = "";
|
||||
reg.image("bfl")->plan(r);
|
||||
}
|
||||
for (const std::string& model : reg.image("ideogram")->models()) {
|
||||
r.model = model;
|
||||
r.aspect_ratio = "";
|
||||
r.width = 0;
|
||||
r.height = 0;
|
||||
reg.image("ideogram")->plan(r);
|
||||
}
|
||||
for (const std::string& model : reg.video("ideogram")->models()) {
|
||||
VideoRequest v = video(model);
|
||||
reg.video("ideogram")->plan(v);
|
||||
}
|
||||
for (const std::string& model : reg.video("higgsfield")->models()) reg.video("higgsfield")->plan(video(model));
|
||||
}
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
// Call tables: structure of every provider table, CallTable rules, registry.
|
||||
#include <set>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/media/common/registry.h"
|
||||
|
||||
using namespace molo::media;
|
||||
using namespace media_test;
|
||||
|
||||
namespace {
|
||||
|
||||
struct Expect {
|
||||
const char* id;
|
||||
std::size_t rows; // rows declared from the published spec / SDK / docs (see SPEC.md)
|
||||
};
|
||||
|
||||
const Expect kExpected[] = {
|
||||
{"stability", 34}, {"bfl", 30}, {"replicate", 36}, {"fal", 16}, {"ideogram", 70}, {"recraft", 22},
|
||||
{"leonardo", 6}, {"runway", 66}, {"luma", 13}, {"minimax", 7}, {"kling", 6}, {"higgsfield", 10},
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(media_tables, every_provider_has_the_declared_row_count) {
|
||||
for (const Expect& e : kExpected) {
|
||||
const std::string id = e.id;
|
||||
const CallTable& t = MediaRegistry::table(id);
|
||||
ASSERT_EQ(t.size(), e.rows);
|
||||
ASSERT_EQ(t.vendor(), std::string(e.id));
|
||||
}
|
||||
ASSERT_EQ(MediaRegistry::providers().size(), sizeof(kExpected) / sizeof(kExpected[0]));
|
||||
}
|
||||
|
||||
TEST(media_tables, rows_are_well_formed) {
|
||||
for (const Expect& e : kExpected) {
|
||||
const std::string id = e.id;
|
||||
const CallTable& t = MediaRegistry::table(id);
|
||||
std::set<std::string> seen;
|
||||
for (std::size_t i = 0; i < t.size(); ++i) {
|
||||
const CallRow& r = t.at(i);
|
||||
ASSERT_TRUE(seen.insert(r.key).second);
|
||||
std::string method = r.http_method;
|
||||
std::string kind = r.body_kind;
|
||||
// a body-less method never declares body fields
|
||||
if (method == "GET" || method == "DELETE") ASSERT_TRUE(kind == "none");
|
||||
if (kind == "none") ASSERT_EQ(std::string(r.body_fields), std::string(""));
|
||||
if (kind != "none") ASSERT_TRUE(method != "GET");
|
||||
// destructive rows are DELETE or an explicit delete_* call
|
||||
if (CallTable::effect_of(r) == Effect::kDelete)
|
||||
ASSERT_TRUE(method == "DELETE" || std::string(r.key).find("delete") != std::string::npos);
|
||||
// every GET reads
|
||||
if (method == "GET") ASSERT_TRUE(CallTable::effect_of(r) == Effect::kRead);
|
||||
// placeholders are non-empty and unique within a path
|
||||
std::set<std::string> ph;
|
||||
for (const std::string& p : CallTable::placeholders(r.path)) {
|
||||
ASSERT_FALSE(p.empty());
|
||||
ASSERT_TRUE(ph.insert(p).second);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(media_tables, paid_rows_exist_for_every_generating_provider) {
|
||||
for (const Expect& e : kExpected) {
|
||||
const std::string id = e.id;
|
||||
const CallTable& t = MediaRegistry::table(id);
|
||||
std::size_t paid = 0;
|
||||
for (std::size_t i = 0; i < t.size(); ++i)
|
||||
if (CallTable::effect_of(t.at(i)) == Effect::kPaid) ++paid;
|
||||
ASSERT_TRUE(paid >= 1);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(media_tables, rejects_a_malformed_table) {
|
||||
static const CallRow dup[] = {{"a", "GET", "/x", "", "none", "", "", "read"}, {"a", "GET", "/y", "", "none", "", "", "read"}};
|
||||
ASSERT_THROW(CallTable("t", dup, 2), std::logic_error);
|
||||
static const CallRow badm[] = {{"a", "FETCH", "/x", "", "none", "", "", "read"}};
|
||||
ASSERT_THROW(CallTable("t", badm, 1), std::logic_error);
|
||||
static const CallRow badk[] = {{"a", "GET", "/x", "", "blob", "", "", "read"}};
|
||||
ASSERT_THROW(CallTable("t", badk, 1), std::logic_error);
|
||||
static const CallRow bade[] = {{"a", "GET", "/x", "", "none", "", "", "free"}};
|
||||
ASSERT_THROW(CallTable("t", bade, 1), std::logic_error);
|
||||
static const CallRow badp[] = {{"a", "GET", "x", "", "none", "", "", "read"}};
|
||||
ASSERT_THROW(CallTable("t", badp, 1), std::logic_error);
|
||||
}
|
||||
|
||||
TEST(media_tables, path_rendering_validates_every_argument) {
|
||||
static const CallRow rows[] = {
|
||||
{"one", "GET", "/a/{id}/b/{name}", "", "none", "", "", "read"},
|
||||
{"multi", "GET", "/{endpoint_id}/x", "", "none", "", "", "read"},
|
||||
};
|
||||
CallTable t("t", rows, 2);
|
||||
ASSERT_EQ(t.render_path(t.get("one"), {{"id", "abc-1"}, {"name", "n.v"}}), std::string("/a/abc-1/b/n.v"));
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "abc"}}), ApiError); // missing
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a"}, {"name", "b"}, {"x", "c"}}), ApiError); // extra
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "../x"}, {"name", "b"}}), ApiError); // traversal
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a/b"}, {"name", "b"}}), ApiError); // slash in a segment
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", ".."}, {"name", "b"}}), ApiError);
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", ""}, {"name", "b"}}), ApiError);
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", "a b"}, {"name", "b"}}), ApiError);
|
||||
ASSERT_THROW(t.render_path(t.get("one"), {{"id", std::string(257, 'a')}, {"name", "b"}}), ApiError);
|
||||
ASSERT_EQ(t.render_path(t.get("multi"), {{"endpoint_id", "fal-ai/flux/dev"}}), std::string("/fal-ai/flux/dev/x"));
|
||||
ASSERT_THROW(t.render_path(t.get("multi"), {{"endpoint_id", "a//b"}}), ApiError);
|
||||
ASSERT_THROW(t.render_path(t.get("multi"), {{"endpoint_id", "a/b/c/d/e/f/g"}}), ApiError); // 7 segments
|
||||
ASSERT_THROW(t.get("nope"), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_tables, segment_alphabet_matches_the_documented_id_shapes) {
|
||||
ASSERT_TRUE(CallTable::valid_segment("cneqzikepnug6xezperrr4z55o")); // Replicate file id
|
||||
ASSERT_TRUE(CallTable::valid_segment("d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff")); // Higgsfield uuid
|
||||
ASSERT_TRUE(CallTable::valid_segment("abc_DEF-123==")); // URL-safe base64 with padding
|
||||
ASSERT_FALSE(CallTable::valid_segment("a?b"));
|
||||
ASSERT_FALSE(CallTable::valid_segment("a#b"));
|
||||
ASSERT_FALSE(CallTable::valid_segment("a%2fb"));
|
||||
}
|
||||
|
||||
TEST(media_registry, lists_unsupported_products_with_a_reason) {
|
||||
const auto& u = MediaRegistry::unsupported();
|
||||
ASSERT_EQ(u.size(), static_cast<std::size_t>(2));
|
||||
ASSERT_EQ(u[0].name, std::string("Midjourney"));
|
||||
ASSERT_TRUE(has(u[0].reason, "No public API"));
|
||||
ASSERT_EQ(u[1].name, std::string("Pika"));
|
||||
for (const auto& x : u) ASSERT_TRUE(x.reason.size() > 40);
|
||||
}
|
||||
|
||||
TEST(media_registry, every_provider_declares_credentials_evidence_and_a_console) {
|
||||
for (const ProviderInfo& p : MediaRegistry::providers()) {
|
||||
ASSERT_FALSE(p.env.empty());
|
||||
ASSERT_TRUE(p.image || p.video);
|
||||
ASSERT_TRUE(p.console_url.rfind("https://", 0) == 0);
|
||||
ASSERT_TRUE(p.note.size() > 10);
|
||||
if (!p.account_key.empty()) {
|
||||
const CallRow& r = MediaRegistry::table(p.id).get(p.account_key);
|
||||
ASSERT_TRUE(CallTable::effect_of(r) == Effect::kRead); // the credential proof never costs money
|
||||
}
|
||||
}
|
||||
ASSERT_EQ(std::string(MediaRegistry::evidence_name(MediaRegistry::find("kling")->evidence)), std::string("probe-only"));
|
||||
ASSERT_EQ(std::string(MediaRegistry::evidence_name(MediaRegistry::find("bfl")->evidence)), std::string("openapi"));
|
||||
ASSERT_TRUE(MediaRegistry::find("midjourney") == nullptr);
|
||||
ASSERT_THROW(MediaRegistry::table("midjourney"), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_registry, required_environment_variables_are_the_documented_ones) {
|
||||
auto env = [](const char* id) { return MediaRegistry::find(id)->env; };
|
||||
ASSERT_EQ(env("stability")[0], std::string("STABILITY_API_KEY"));
|
||||
ASSERT_EQ(env("bfl")[0], std::string("BFL_API_KEY"));
|
||||
ASSERT_EQ(env("runway")[0], std::string("RUNWAY_API_KEY"));
|
||||
ASSERT_EQ(env("luma")[0], std::string("LUMA_API_KEY"));
|
||||
ASSERT_EQ(env("replicate")[0], std::string("REPLICATE_API_TOKEN"));
|
||||
ASSERT_EQ(env("fal")[0], std::string("FAL_KEY"));
|
||||
ASSERT_EQ(env("ideogram")[0], std::string("IDEOGRAM_API_KEY"));
|
||||
ASSERT_EQ(env("recraft")[0], std::string("RECRAFT_API_TOKEN"));
|
||||
ASSERT_EQ(env("leonardo")[0], std::string("LEONARDO_API_KEY"));
|
||||
}
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
// The neutral job seam: request validation, JobRunner caps (with a fake
|
||||
// generator: local logic only), output scanning, status mappers, downloader
|
||||
// rules. Inputs to the mappers are hand-built (LOCAL-LOGIC): they exercise our
|
||||
// code, they are not claims about what a vendor returns.
|
||||
#include <vector>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/media/bfl/bfl.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
#include "vendors/media/common/downloader.h"
|
||||
#include "vendors/media/common/job_runner.h"
|
||||
#include "vendors/media/common/output_scanner.h"
|
||||
#include "vendors/media/higgsfield/higgsfield.h"
|
||||
#include "vendors/media/kling/kling.h"
|
||||
#include "vendors/media/leonardo/leonardo.h"
|
||||
#include "vendors/media/luma/luma.h"
|
||||
#include "vendors/media/minimax/minimax.h"
|
||||
#include "vendors/media/runway/runway.h"
|
||||
|
||||
using namespace molo::media;
|
||||
using namespace media_test;
|
||||
|
||||
namespace {
|
||||
|
||||
Json obj(std::initializer_list<std::pair<const char*, Json>> items) {
|
||||
Json o = Json::object();
|
||||
for (const auto& [k, v] : items) o.set(k, v);
|
||||
return o;
|
||||
}
|
||||
|
||||
Json arr(std::initializer_list<Json> items) {
|
||||
Json a = Json::array();
|
||||
for (const Json& v : items) a.push(v);
|
||||
return a;
|
||||
}
|
||||
|
||||
// Local fake: finishes after `finish_after` polls (0 = never).
|
||||
class FakeGenerator : public ImageGenerator {
|
||||
public:
|
||||
explicit FakeGenerator(int finish_after, JobState end = JobState::kSucceeded) : finish_after_(finish_after), end_(end) {}
|
||||
std::string provider() const override { return "fake"; }
|
||||
bool configured() const override { return true; }
|
||||
std::vector<std::string> models() const override { return {"m"}; }
|
||||
Json plan(const ImageRequest&) const override { return Json::object(); }
|
||||
JobTicket submit(const ImageRequest&, bool) override { return JobTicket{}; }
|
||||
JobStatus poll(const JobTicket&) override {
|
||||
++polls;
|
||||
JobStatus s;
|
||||
if (finish_after_ > 0 && polls >= finish_after_) {
|
||||
s.state = end_;
|
||||
if (end_ == JobState::kFailed) s.error = "boom";
|
||||
MediaAsset a;
|
||||
a.url = "https://example.com/out.png";
|
||||
if (end_ == JobState::kSucceeded) s.assets.push_back(a);
|
||||
} else {
|
||||
s.state = JobState::kRunning;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
bool cancel(const JobTicket&, bool) override { return false; }
|
||||
int suggested_poll_ms() const override { return 100; } // below the floor on purpose
|
||||
int polls = 0;
|
||||
|
||||
private:
|
||||
int finish_after_;
|
||||
JobState end_;
|
||||
};
|
||||
|
||||
struct Clockwork {
|
||||
long now = 0;
|
||||
std::vector<int> slept;
|
||||
JobRunner::Sleeper sleeper() {
|
||||
return [this](int ms) {
|
||||
slept.push_back(ms);
|
||||
now += ms;
|
||||
};
|
||||
}
|
||||
JobRunner::Clock clock() {
|
||||
return [this] { return now; };
|
||||
}
|
||||
};
|
||||
|
||||
JobTicket ticket(const std::string& id = "j1") {
|
||||
JobTicket t;
|
||||
t.provider = "fake";
|
||||
t.id = id;
|
||||
return t;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ------------------------------------------------------------------ JobRunner
|
||||
|
||||
TEST(media_runner, polls_until_terminal_and_applies_the_interval_floor_and_backoff) {
|
||||
Clockwork w;
|
||||
FakeGenerator g(5);
|
||||
WaitPolicy p;
|
||||
p.backoff = 2.0;
|
||||
p.max_interval_ms = 3000;
|
||||
JobRunner r(p, w.sleeper(), w.clock());
|
||||
JobStatus s = r.wait(g, ticket());
|
||||
ASSERT_TRUE(s.state == JobState::kSucceeded);
|
||||
ASSERT_EQ(g.polls, 5);
|
||||
// suggestion 100 ms is lifted to the 500 ms floor, then doubles, capped at 3000
|
||||
ASSERT_EQ(w.slept.size(), static_cast<std::size_t>(4));
|
||||
ASSERT_EQ(w.slept[0], 500);
|
||||
ASSERT_EQ(w.slept[1], 1000);
|
||||
ASSERT_EQ(w.slept[2], 2000);
|
||||
ASSERT_EQ(w.slept[3], 3000);
|
||||
}
|
||||
|
||||
TEST(media_runner, a_failed_job_is_a_status_not_an_exception) {
|
||||
Clockwork w;
|
||||
FakeGenerator g(2, JobState::kFailed);
|
||||
JobRunner r({}, w.sleeper(), w.clock());
|
||||
JobStatus s = r.wait(g, ticket());
|
||||
ASSERT_TRUE(s.state == JobState::kFailed);
|
||||
ASSERT_EQ(s.error, std::string("boom"));
|
||||
}
|
||||
|
||||
TEST(media_runner, attempt_cap_throws_loudly_and_does_not_cancel) {
|
||||
Clockwork w;
|
||||
FakeGenerator g(0);
|
||||
WaitPolicy p;
|
||||
p.max_attempts = 3;
|
||||
JobRunner r(p, w.sleeper(), w.clock());
|
||||
try {
|
||||
r.wait(g, ticket("job-77"));
|
||||
FAIL("expected WAIT_EXPIRED");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("WAIT_EXPIRED"));
|
||||
ASSERT_EQ(e.http_status(), 504);
|
||||
ASSERT_TRUE(has(e.message(), "job-77"));
|
||||
ASSERT_TRUE(has(e.message(), "NOT cancelled"));
|
||||
}
|
||||
ASSERT_EQ(g.polls, 3);
|
||||
}
|
||||
|
||||
TEST(media_runner, total_wait_cap_stops_before_sleeping_past_the_deadline) {
|
||||
Clockwork w;
|
||||
FakeGenerator g(0);
|
||||
WaitPolicy p;
|
||||
p.max_wait_seconds = 5;
|
||||
p.max_attempts = 200;
|
||||
p.first_interval_ms = 2000;
|
||||
p.backoff = 1.0;
|
||||
JobRunner r(p, w.sleeper(), w.clock());
|
||||
ASSERT_THROW(r.wait(g, ticket()), ApiError);
|
||||
long total = 0;
|
||||
for (int s : w.slept) total += s;
|
||||
ASSERT_TRUE(total <= 5000);
|
||||
ASSERT_TRUE(g.polls <= 4);
|
||||
}
|
||||
|
||||
TEST(media_runner, immediate_tickets_need_no_poll) {
|
||||
Clockwork w;
|
||||
FakeGenerator g(0);
|
||||
JobTicket t = ticket();
|
||||
t.immediate = true;
|
||||
MediaAsset a;
|
||||
a.data_base64 = "AAAA";
|
||||
t.assets.push_back(a);
|
||||
JobStatus s = JobRunner({}, w.sleeper(), w.clock()).wait(g, t);
|
||||
ASSERT_TRUE(s.state == JobState::kSucceeded);
|
||||
ASSERT_EQ(g.polls, 0);
|
||||
ASSERT_EQ(s.assets.size(), static_cast<std::size_t>(1));
|
||||
}
|
||||
|
||||
TEST(media_runner, policy_values_outside_the_caps_are_rejected) {
|
||||
auto bad = [](auto edit) {
|
||||
WaitPolicy p;
|
||||
edit(p);
|
||||
return p;
|
||||
};
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_attempts = 0; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_attempts = MediaCaps::kMaxPollAttempts + 1; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_wait_seconds = MediaCaps::kMaxWaitSeconds + 1; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_wait_seconds = 0; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.backoff = 0.9; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.backoff = 3.5; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_interval_ms = MediaCaps::kMinPollIntervalMs - 1; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.max_interval_ms = MediaCaps::kMaxPollIntervalMs + 1; })), ApiError);
|
||||
ASSERT_THROW(JobRunner(bad([](WaitPolicy& p) { p.first_interval_ms = -1; })), ApiError);
|
||||
JobRunner ok{WaitPolicy{}};
|
||||
ASSERT_EQ(ok.policy().max_attempts, 120);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- requests
|
||||
|
||||
TEST(media_requests, image_validation) {
|
||||
ImageRequest r;
|
||||
ASSERT_THROW(r.validate(), ApiError); // no prompt
|
||||
r.prompt = "a cat";
|
||||
r.validate();
|
||||
r.count = 0;
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
r.count = MediaCaps::kMaxImagesPerRequest + 1;
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
r.count = 1;
|
||||
r.width = 512;
|
||||
ASSERT_THROW(r.validate(), ApiError); // width without height
|
||||
r.height = MediaCaps::kMaxDimension + 1;
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
r.height = 512;
|
||||
r.validate();
|
||||
r.seed = -2;
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
r.seed = 4294967296L;
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
r.seed = 7;
|
||||
r.extra = Json(5);
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
r.extra = Json::object();
|
||||
r.prompt = std::string(MediaCaps::kMaxPromptChars + 1, 'x');
|
||||
ASSERT_THROW(r.validate(), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_requests, video_validation) {
|
||||
VideoRequest v;
|
||||
ASSERT_THROW(v.validate(), ApiError);
|
||||
v.image_url = "https://example.com/a.png"; // image-to-video needs no prompt
|
||||
v.validate();
|
||||
v.duration_seconds = MediaCaps::kMaxVideoSeconds + 1;
|
||||
ASSERT_THROW(v.validate(), ApiError);
|
||||
v.duration_seconds = 5;
|
||||
v.validate();
|
||||
v.duration_seconds = -1;
|
||||
ASSERT_THROW(v.validate(), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_requests, ticket_and_status_json) {
|
||||
JobTicket t;
|
||||
t.provider = "fal";
|
||||
t.id = "r1";
|
||||
t.model = "fal-ai/flux/dev";
|
||||
t.kind = MediaKind::kVideo;
|
||||
t.meta = obj({{"endpoint", Json("fal-ai/flux/dev")}});
|
||||
JobTicket back = JobTicket::from_json(t.to_json());
|
||||
ASSERT_EQ(back.provider, std::string("fal"));
|
||||
ASSERT_EQ(back.id, std::string("r1"));
|
||||
ASSERT_TRUE(back.kind == MediaKind::kVideo);
|
||||
ASSERT_EQ(back.meta.find("endpoint")->as_string(), std::string("fal-ai/flux/dev"));
|
||||
ASSERT_THROW(JobTicket::from_json(Json(1)), ApiError);
|
||||
ASSERT_THROW(JobTicket::from_json(obj({{"provider", Json("x")}})), ApiError);
|
||||
JobStatus s;
|
||||
s.state = JobState::kRunning;
|
||||
s.progress = 0.5;
|
||||
ASSERT_TRUE(has(s.to_json().dump(), "\"state\":\"running\""));
|
||||
ASSERT_FALSE(s.terminal());
|
||||
s.state = JobState::kCancelled;
|
||||
ASSERT_TRUE(s.terminal());
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- OutputScanner
|
||||
|
||||
TEST(media_scanner, finds_urls_in_the_shapes_providers_use) {
|
||||
// fal style: images[].url and video.url
|
||||
auto fal = OutputScanner::scan(obj({{"images", arr({obj({{"url", Json("https://v3.fal.media/a.png")}})})},
|
||||
{"video", obj({{"url", Json("https://v3.fal.media/b.mp4")}})}}),
|
||||
MediaKind::kImage);
|
||||
ASSERT_EQ(fal.size(), static_cast<std::size_t>(2));
|
||||
ASSERT_TRUE(fal[0].kind == MediaKind::kImage);
|
||||
ASSERT_EQ(fal[0].mime, std::string("image/png"));
|
||||
ASSERT_TRUE(fal[1].kind == MediaKind::kVideo);
|
||||
// Replicate style: a bare string, or an array
|
||||
ASSERT_EQ(OutputScanner::scan(Json("https://replicate.delivery/x/out.webp"), MediaKind::kImage).size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(OutputScanner::scan(arr({Json("https://a/1.png"), Json("https://a/2.png")}), MediaKind::kImage).size(), static_cast<std::size_t>(2));
|
||||
// non-URL strings are ignored
|
||||
ASSERT_EQ(OutputScanner::scan(arr({Json("hello"), Json("ftp://x/y.png")}), MediaKind::kImage).size(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_scanner, skips_request_echoes_and_control_links_and_dedupes) {
|
||||
Json doc = obj({{"input", obj({{"image", Json("https://in/source.png")}})},
|
||||
{"urls", obj({{"get", Json("https://api.replicate.com/v1/predictions/p1")}})},
|
||||
{"status_url", Json("https://api.higgsfield.ai/requests/x/status")},
|
||||
{"output", arr({Json("https://out/a.png"), Json("https://out/a.png")})}});
|
||||
auto found = OutputScanner::scan(doc, MediaKind::kImage);
|
||||
ASSERT_EQ(found.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(found[0].url, std::string("https://out/a.png"));
|
||||
}
|
||||
|
||||
TEST(media_scanner, data_urls_become_inline_assets_and_caps_hold) {
|
||||
auto d = OutputScanner::scan(Json("data:image/png;base64,iVBORw0KGgo="), MediaKind::kOther);
|
||||
ASSERT_EQ(d.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(d[0].mime, std::string("image/png"));
|
||||
ASSERT_EQ(d[0].data_base64, std::string("iVBORw0KGgo="));
|
||||
ASSERT_TRUE(d[0].kind == MediaKind::kImage);
|
||||
Json many = Json::array();
|
||||
for (int i = 0; i < 100; ++i) many.push(Json("https://x/" + std::to_string(i) + ".png"));
|
||||
ASSERT_EQ(OutputScanner::scan(many, MediaKind::kImage).size(), MediaCaps::kMaxAssetsPerJob);
|
||||
Json deep = Json("https://deep/out.png");
|
||||
for (int i = 0; i < 20; ++i) deep = obj({{"k", deep}});
|
||||
ASSERT_EQ(OutputScanner::scan(deep, MediaKind::kImage).size(), static_cast<std::size_t>(0)); // depth cap
|
||||
ASSERT_TRUE(OutputScanner::kind_from_name("https://x/a.GLB?sig=1", MediaKind::kOther) == MediaKind::kModel3d);
|
||||
ASSERT_TRUE(OutputScanner::kind_from_name("https://x/a.wav", MediaKind::kOther) == MediaKind::kAudio);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ status mappers
|
||||
|
||||
TEST(media_status, bfl_words) {
|
||||
ASSERT_TRUE(BflImageGenerator::state_of("Ready") == JobState::kSucceeded);
|
||||
ASSERT_TRUE(BflImageGenerator::state_of("Pending") == JobState::kQueued);
|
||||
ASSERT_TRUE(BflImageGenerator::state_of("Generating") == JobState::kRunning);
|
||||
for (const char* w : {"Error", "Request Moderated", "Content Moderated", "Task not found", "No Card Generated"})
|
||||
ASSERT_TRUE(BflImageGenerator::state_of(w) == JobState::kFailed);
|
||||
ASSERT_TRUE(BflImageGenerator::state_of("something new") == JobState::kRunning); // unknown = keep polling, bounded by caps
|
||||
}
|
||||
|
||||
TEST(media_status, runway_tasks) {
|
||||
JobStatus ok = RunwayJobs::status_of(obj({{"status", Json("SUCCEEDED")}, {"output", arr({Json("https://r/v.mp4")})}}), MediaKind::kVideo);
|
||||
ASSERT_TRUE(ok.state == JobState::kSucceeded);
|
||||
ASSERT_TRUE(ok.assets[0].kind == MediaKind::kVideo);
|
||||
JobStatus empty = RunwayJobs::status_of(obj({{"status", Json("SUCCEEDED")}}), MediaKind::kVideo);
|
||||
ASSERT_TRUE(empty.state == JobState::kFailed);
|
||||
JobStatus run = RunwayJobs::status_of(obj({{"status", Json("RUNNING")}, {"progress", Json(0.4)}}), MediaKind::kVideo);
|
||||
ASSERT_TRUE(run.state == JobState::kRunning);
|
||||
ASSERT_TRUE(run.progress > 0.39 && run.progress < 0.41);
|
||||
JobStatus fail = RunwayJobs::status_of(obj({{"status", Json("FAILED")}, {"failure", Json("moderation")}, {"failureCode", Json("SAFETY")}}), MediaKind::kVideo);
|
||||
ASSERT_EQ(fail.error, std::string("moderation (SAFETY)"));
|
||||
ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("CANCELLED")}}), MediaKind::kVideo).state == JobState::kCancelled);
|
||||
ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("THROTTLED")}}), MediaKind::kVideo).state == JobState::kQueued);
|
||||
ASSERT_TRUE(RunwayJobs::status_of(obj({{"status", Json("PENDING")}}), MediaKind::kVideo).state == JobState::kQueued);
|
||||
}
|
||||
|
||||
TEST(media_status, luma_generations_ignore_the_progress_preview) {
|
||||
Json done = obj({{"state", Json("completed")},
|
||||
{"assets", obj({{"video", Json("https://l/final.mp4")}, {"progress_video", Json("https://l/progress.mp4")}})}});
|
||||
JobStatus s = LumaJobs::status_of(done, MediaKind::kVideo);
|
||||
ASSERT_TRUE(s.state == JobState::kSucceeded);
|
||||
ASSERT_EQ(s.assets.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(s.assets[0].url, std::string("https://l/final.mp4"));
|
||||
ASSERT_TRUE(LumaJobs::status_of(obj({{"state", Json("dreaming")}}), MediaKind::kVideo).state == JobState::kRunning);
|
||||
ASSERT_TRUE(LumaJobs::status_of(obj({{"state", Json("queued")}}), MediaKind::kVideo).state == JobState::kQueued);
|
||||
JobStatus f = LumaJobs::status_of(obj({{"state", Json("failed")}, {"failure_reason", Json("nope")}}), MediaKind::kVideo);
|
||||
ASSERT_EQ(f.error, std::string("nope"));
|
||||
}
|
||||
|
||||
TEST(media_status, word_maps_of_minimax_higgsfield_kling) {
|
||||
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Success") == JobState::kSucceeded);
|
||||
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Fail") == JobState::kFailed);
|
||||
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Processing") == JobState::kRunning);
|
||||
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Queueing") == JobState::kQueued);
|
||||
ASSERT_TRUE(MinimaxVideoGenerator::state_of("Preparing") == JobState::kQueued);
|
||||
for (const char* w : {"completed"}) ASSERT_TRUE(HiggsfieldJobs::state_of(w) == JobState::kSucceeded);
|
||||
ASSERT_TRUE(HiggsfieldJobs::state_of("nsfw") == JobState::kFailed);
|
||||
ASSERT_TRUE(HiggsfieldJobs::state_of("failed") == JobState::kFailed);
|
||||
ASSERT_TRUE(HiggsfieldJobs::state_of("canceled") == JobState::kCancelled);
|
||||
ASSERT_TRUE(HiggsfieldJobs::state_of("in_progress") == JobState::kRunning);
|
||||
ASSERT_TRUE(HiggsfieldJobs::state_of("queued") == JobState::kQueued);
|
||||
ASSERT_TRUE(KlingVideoGenerator::state_of("succeed") == JobState::kSucceeded);
|
||||
ASSERT_TRUE(KlingVideoGenerator::state_of("failed") == JobState::kFailed);
|
||||
ASSERT_TRUE(KlingVideoGenerator::state_of("processing") == JobState::kRunning);
|
||||
ASSERT_TRUE(KlingVideoGenerator::state_of("submitted") == JobState::kQueued);
|
||||
}
|
||||
|
||||
TEST(media_status, leonardo_generation_shape_is_read_from_generations_by_pk) {
|
||||
Json done = obj({{"generations_by_pk", obj({{"status", Json("COMPLETE")},
|
||||
{"generated_images", arr({obj({{"url", Json("https://cdn.leonardo.ai/a.jpg")}, {"id", Json("i1")}})})}})}});
|
||||
JobStatus s = LeonardoJobs::status_of(done, MediaKind::kImage);
|
||||
ASSERT_TRUE(s.state == JobState::kSucceeded);
|
||||
ASSERT_EQ(s.assets.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("PENDING")}})}}), MediaKind::kImage).state == JobState::kRunning);
|
||||
ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("FAILED")}})}}), MediaKind::kImage).state == JobState::kFailed);
|
||||
ASSERT_TRUE(LeonardoJobs::status_of(obj({{"generations_by_pk", obj({{"status", Json("COMPLETE")}})}}), MediaKind::kImage).state == JobState::kFailed);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------- downloader
|
||||
|
||||
TEST(media_downloader, url_rules) {
|
||||
Downloader::validate_url("https://cdn.example.com/a.png?sig=abc");
|
||||
for (const char* bad : {"", "http://example.com/a.png", "ftp://x/y", "https://", "https:///path", "https://u:p@example.com/a",
|
||||
"https://exa mple.com/a", "javascript:alert(1)", "file:///etc/passwd", "data:image/png;base64,AA=="})
|
||||
ASSERT_THROW(Downloader::validate_url(bad), ApiError);
|
||||
ASSERT_THROW(Downloader::validate_url("https://x/" + std::string(MediaCaps::kMaxUrlChars, 'a')), ApiError);
|
||||
ASSERT_THROW(Downloader::validate_url("https://x/a\nb"), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_downloader, inline_assets_decode_without_network) {
|
||||
Mock m;
|
||||
MockTransport* t = m.raw;
|
||||
Downloader d(m.take());
|
||||
MediaAsset a;
|
||||
a.data_base64 = "aGVsbG8=";
|
||||
a.mime = "image/png";
|
||||
FetchedAsset f = d.fetch(a);
|
||||
ASSERT_EQ(f.bytes, std::string("hello"));
|
||||
ASSERT_EQ(f.content_type, std::string("image/png"));
|
||||
MediaAsset bad;
|
||||
bad.data_base64 = "@@@";
|
||||
ASSERT_THROW(d.fetch(bad), ApiError);
|
||||
ASSERT_THROW(d.fetch(MediaAsset{}), ApiError);
|
||||
MediaAsset insecure;
|
||||
insecure.url = "http://example.com/x.png";
|
||||
ASSERT_THROW(d.fetch(insecure), ApiError);
|
||||
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_downloader, sends_no_credential_and_maps_failures) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/missing.png", reply("not found", 404, "text/plain"));
|
||||
MockTransport* t = m.raw;
|
||||
Downloader d(m.take());
|
||||
try {
|
||||
d.fetch_url("https://cdn.example.com/missing.png");
|
||||
FAIL("expected a download failure");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("DOWNLOAD_FAILED"));
|
||||
ASSERT_EQ(e.http_status(), 404);
|
||||
}
|
||||
auto seen = t->seen();
|
||||
ASSERT_EQ(seen.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_FALSE(seen[0].headers.get("Authorization").has_value());
|
||||
ASSERT_FALSE(seen[0].headers.get("x-key").has_value());
|
||||
}
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
// Opt-in live checks. MOLO_LIVE_NET=1 enables the network; a provider's key
|
||||
// (via llm::Config) enables its read-only calls (account, balance, models).
|
||||
// Anything that spends money additionally needs MOLO_LIVE_PAID=1 and is
|
||||
// limited to a free price quote where the provider offers one (Ideogram
|
||||
// dry_run) - this file never submits a generation. Without the variables each
|
||||
// test prints a loud SKIP. Nothing is created or deleted.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "support.h"
|
||||
#include "vendors/media/common/downloader.h"
|
||||
#include "vendors/media/common/registry.h"
|
||||
#include "vendors/media/bfl/bfl.h"
|
||||
#include "vendors/media/fal/fal.h"
|
||||
#include "vendors/media/higgsfield/higgsfield.h"
|
||||
#include "vendors/media/ideogram/ideogram.h"
|
||||
#include "vendors/media/kling/kling.h"
|
||||
#include "vendors/media/leonardo/leonardo.h"
|
||||
#include "vendors/media/luma/luma.h"
|
||||
#include "vendors/media/minimax/minimax.h"
|
||||
#include "vendors/media/recraft/recraft.h"
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
#include "vendors/media/runway/runway.h"
|
||||
#include "vendors/media/stability/stability.h"
|
||||
|
||||
using namespace molo::media;
|
||||
using namespace media_test;
|
||||
|
||||
namespace {
|
||||
|
||||
// Read once at startup: other suites in this binary unset MOLO_LIVE_NET while they run.
|
||||
const bool kLiveNet = std::getenv("MOLO_LIVE_NET") != nullptr;
|
||||
const bool kLivePaid = std::getenv("MOLO_LIVE_PAID") != nullptr;
|
||||
bool net() { return kLiveNet; }
|
||||
|
||||
void skip(const std::string& name, const std::string& why) { std::cout << "SKIP " << name << " (" << why << ")\n"; }
|
||||
|
||||
// Unauthenticated probe: a deliberately invalid key must be refused (the host
|
||||
// resolves, TLS works, the documented path exists). `accept` lists the HTTP
|
||||
// statuses seen on 2026-10-25: BFL answers 422 for a malformed key, MiniMax
|
||||
// answers 200 with base_resp 1004 (turned into an ApiError by the client).
|
||||
template <class Fn>
|
||||
void expect_refused(const std::string& name, Fn call, std::initializer_list<int> accept = {401, 403}) {
|
||||
try {
|
||||
call();
|
||||
FAIL(name << ": an invalid key was accepted");
|
||||
} catch (const ApiError& e) {
|
||||
std::cout << " " << name << ": " << e.http_status() << " " << e.type() << "\n";
|
||||
bool ok = false;
|
||||
for (int a : accept) ok = ok || e.http_status() == a;
|
||||
ASSERT_TRUE(ok);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<molo::clients::Transport> real() { return std::make_unique<molo::clients::transports::HttpTransport>(); }
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(media_live, invalid_keys_are_refused_by_every_provider) {
|
||||
if (!net()) return skip("media_live.invalid_keys_are_refused", "set MOLO_LIVE_NET=1");
|
||||
const std::string bad = "molo-invalid-probe";
|
||||
expect_refused("stability", [&] { StabilityClient::with(real(), bad)->balance(); });
|
||||
expect_refused("bfl", [&] { BflClient::with(real(), bad)->credits(); }, {401, 403, 422});
|
||||
expect_refused("replicate", [&] { ReplicateClient::with(real(), bad)->account(); });
|
||||
expect_refused("fal.platform", [&] { FalPlatform::with(real(), bad)->pricing("fal-ai/flux/dev"); });
|
||||
expect_refused("fal.queue", [&] { FalQueue::with(real(), bad)->status(FalEndpoint::parse("fal-ai/flux/dev"), "00000000-0000-0000-0000-000000000000"); });
|
||||
expect_refused("ideogram", [&] { IdeogramClient::with(real(), bad)->api_keys(); });
|
||||
expect_refused("recraft", [&] { RecraftClient::with(real(), bad)->me(); });
|
||||
expect_refused("leonardo", [&] { LeonardoClient::with(real(), bad)->models(); });
|
||||
expect_refused("runway", [&] { RunwayClient::with(real(), bad)->organization(); });
|
||||
expect_refused("luma", [&] { LumaClient::with(real(), bad)->credits(); });
|
||||
expect_refused("minimax", [&] { MinimaxClient::with(real(), bad)->query_video("1"); }, {400});
|
||||
expect_refused("kling", [&] { std::make_shared<KlingClient>("AK-invalid-probe", "SK-invalid-probe")->get("text2video.get", "0"); });
|
||||
expect_refused("higgsfield", [&] { HiggsfieldClient::with(real(), "a", "b")->status("00000000-0000-0000-0000-000000000000"); });
|
||||
}
|
||||
|
||||
TEST(media_live, read_only_account_calls_for_every_configured_provider) {
|
||||
MediaRegistry reg;
|
||||
int ran = 0;
|
||||
for (const ProviderInfo& p : MediaRegistry::providers()) {
|
||||
const std::string name = "media_live.account." + p.id;
|
||||
if (!net()) {
|
||||
skip(name, "set MOLO_LIVE_NET=1");
|
||||
continue;
|
||||
}
|
||||
if (!MediaRegistry::configured(p.id)) {
|
||||
skip(name, "set " + p.env[0]);
|
||||
continue;
|
||||
}
|
||||
if (p.account_key.empty()) {
|
||||
skip(name, p.id + " has no free account read");
|
||||
continue;
|
||||
}
|
||||
Json out = reg.account(p.id);
|
||||
std::cout << " " << p.id << " " << p.account_key << " -> " << out.dump().substr(0, 120) << "\n";
|
||||
ASSERT_TRUE(out.is_object());
|
||||
++ran;
|
||||
}
|
||||
std::cout << " account reads run: " << ran << "\n";
|
||||
}
|
||||
|
||||
TEST(media_live, paid_calls_are_not_exercised_here) {
|
||||
if (!kLivePaid) return skip("media_live.paid", "set MOLO_LIVE_PAID=1 (and read this file: nothing paid runs even then)");
|
||||
std::cout << " MOLO_LIVE_PAID is set; this file still submits no generation. Use the demo with confirm:true by hand.\n";
|
||||
}
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
// Requests built from the call tables: exact targets, bodies, headers, field
|
||||
// validation and the confirmation gate. No reply is scripted except where a
|
||||
// fixture is labelled; prepare() itself never touches a transport.
|
||||
#include "support.h"
|
||||
#include "vendors/media/common/caps.h"
|
||||
#include "vendors/media/bfl/bfl.h"
|
||||
#include "vendors/media/fal/fal.h"
|
||||
#include "vendors/media/higgsfield/higgsfield.h"
|
||||
#include "vendors/media/ideogram/ideogram.h"
|
||||
#include "vendors/media/kling/kling.h"
|
||||
#include "vendors/media/leonardo/leonardo.h"
|
||||
#include "vendors/media/luma/luma.h"
|
||||
#include "vendors/media/minimax/minimax.h"
|
||||
#include "vendors/media/recraft/recraft.h"
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
#include "vendors/media/runway/runway.h"
|
||||
#include "vendors/media/stability/stability.h"
|
||||
|
||||
using namespace molo::media;
|
||||
using namespace media_test;
|
||||
|
||||
namespace {
|
||||
Json obj(std::initializer_list<std::pair<const char*, Json>> items) {
|
||||
Json o = Json::object();
|
||||
for (const auto& [k, v] : items) o.set(k, v);
|
||||
return o;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// ------------------------------------------------------------------ Stability
|
||||
|
||||
TEST(media_prepare, stability_generate_is_multipart_with_json_accept) {
|
||||
Mock m;
|
||||
auto c = StabilityClient::with(m.take(), "KEY");
|
||||
PreparedCall p = c->plan("stable-image.generate.core", obj({{"prompt", Json("a cat")}, {"output_format", Json("png")}}), {});
|
||||
ASSERT_EQ(p.method, std::string("POST"));
|
||||
ASSERT_EQ(p.target, std::string("/v2beta/stable-image/generate/core"));
|
||||
ASSERT_TRUE(has(p.content_type, "multipart/form-data; boundary="));
|
||||
ASSERT_TRUE(has(p.body, "name=\"prompt\""));
|
||||
ASSERT_TRUE(has(p.body, "a cat"));
|
||||
ASSERT_EQ(p.headers.at("Accept"), std::string("application/json"));
|
||||
ASSERT_TRUE(p.effect == Effect::kPaid);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_prepare, stability_files_must_use_a_declared_field_and_fit_the_cap) {
|
||||
Mock m;
|
||||
auto c = StabilityClient::with(m.take(), "KEY");
|
||||
FilePart img{"image", "in.png", "image/png", "\x89PNG-bytes"};
|
||||
PreparedCall p = c->plan("stable-image.upscale.fast", obj({{"output_format", Json("webp")}}), {img});
|
||||
ASSERT_TRUE(has(p.body, "filename=\"in.png\""));
|
||||
ASSERT_TRUE(has(p.body, "Content-Type: image/png"));
|
||||
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"picture", "a.png", "image/png", "x"}}), ApiError);
|
||||
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"image", "a.png", "image/png", ""}}), ApiError);
|
||||
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(),
|
||||
{FilePart{"image", "a.png", "image/png", std::string(MediaCaps::kMaxFileBytes + 1, 'x')}}), ApiError);
|
||||
ASSERT_THROW(c->plan("stable-image.upscale.fast", Json::object(), {FilePart{"image", "a\".png", "image/png", "x"}}), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_prepare, stability_rejects_unknown_fields_and_unknown_calls) {
|
||||
Mock m;
|
||||
auto c = StabilityClient::with(m.take(), "KEY");
|
||||
ASSERT_THROW(c->plan("stable-image.generate.core", obj({{"prompt", Json("x")}, {"bogus", Json(1)}}), {}), ApiError);
|
||||
ASSERT_THROW(c->plan("stable-image.generate.nothing", Json::object(), {}), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_prepare, stability_result_ids_are_validated_before_any_traffic) {
|
||||
Mock m;
|
||||
auto c = StabilityClient::with(m.take(), "KEY");
|
||||
ASSERT_THROW(c->result("../etc"), ApiError);
|
||||
ASSERT_THROW(c->result("a/b"), ApiError);
|
||||
ASSERT_THROW(c->audio_result(""), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_prepare, stability_v1_text_to_image_is_json_with_an_engine_in_the_path) {
|
||||
Mock m;
|
||||
auto c = StabilityClient::with(m.take(), "KEY");
|
||||
CallArgs a;
|
||||
a.path["engine_id"] = "stable-diffusion-xl-1024-v1-0";
|
||||
a.body = obj({{"samples", Json(1)}, {"steps", Json(30)}});
|
||||
PreparedCall p = c->prepare("v1.generation.id.text-to-image", a);
|
||||
ASSERT_EQ(p.target, std::string("/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image"));
|
||||
ASSERT_EQ(p.content_type, std::string("application/json"));
|
||||
ASSERT_EQ(p.body, std::string("{\"samples\":1,\"steps\":30}"));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------ BFL
|
||||
|
||||
TEST(media_prepare, bfl_body_is_json_and_query_is_encoded) {
|
||||
Mock m;
|
||||
auto c = BflClient::with(m.take(), "KEY");
|
||||
PreparedCall p = c->plan("flux-2-pro", obj({{"prompt", Json("x")}, {"width", Json(1024)}, {"height", Json(768)}, {"seed", Json(7)}}));
|
||||
ASSERT_EQ(p.target, std::string("/v1/flux-2-pro"));
|
||||
ASSERT_EQ(p.body, std::string("{\"height\":768,\"prompt\":\"x\",\"seed\":7,\"width\":1024}"));
|
||||
CallArgs a;
|
||||
a.query["id"] = "a b&c";
|
||||
PreparedCall g = c->prepare("get_result", a);
|
||||
ASSERT_EQ(g.target, std::string("/v1/get_result?id=a%20b%26c"));
|
||||
ASSERT_EQ(g.body, std::string(""));
|
||||
CallArgs bad;
|
||||
bad.query["nope"] = "1";
|
||||
ASSERT_THROW(c->prepare("get_result", bad), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_prepare, bfl_sends_the_x_key_header_and_never_a_bearer) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/v1/credits", empty_ok());
|
||||
MockTransport* t = m.raw;
|
||||
auto c = BflClient::with(m.take(), "SECRET-KEY");
|
||||
c->credits();
|
||||
auto seen = t->seen();
|
||||
ASSERT_EQ(seen.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(*seen[0].headers.get("x-key"), std::string("SECRET-KEY"));
|
||||
ASSERT_FALSE(seen[0].headers.get("Authorization").has_value());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Replicate
|
||||
|
||||
TEST(media_prepare, replicate_official_model_and_pinned_version_use_different_rows) {
|
||||
Mock m;
|
||||
auto c = ReplicateClient::with(m.take(), "KEY");
|
||||
Json in = obj({{"prompt", Json("a cat")}});
|
||||
PreparedCall a = c->plan_prediction(ReplicateClient::ModelRef::parse("black-forest-labs/flux-schnell"), in, Json::object());
|
||||
ASSERT_EQ(a.target, std::string("/models/black-forest-labs/flux-schnell/predictions"));
|
||||
ASSERT_EQ(a.body, std::string("{\"input\":{\"prompt\":\"a cat\"}}"));
|
||||
PreparedCall b = c->plan_prediction(ReplicateClient::ModelRef::parse("owner/name:abc123"), in, Json::object());
|
||||
ASSERT_EQ(b.target, std::string("/predictions"));
|
||||
ASSERT_EQ(b.body, std::string("{\"input\":{\"prompt\":\"a cat\"},\"version\":\"abc123\"}"));
|
||||
Json hook = obj({{"webhook", Json("https://example.com/h")}, {"webhook_events_filter", Json::array()}});
|
||||
PreparedCall w = c->plan_prediction(ReplicateClient::ModelRef::parse("o/n"), in, hook);
|
||||
ASSERT_TRUE(has(w.body, "\"webhook\":\"https://example.com/h\""));
|
||||
ASSERT_THROW(c->plan_prediction(ReplicateClient::ModelRef::parse("o/n"), in, obj({{"nope", Json(1)}})), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_prepare, replicate_model_references_are_validated) {
|
||||
using R = ReplicateClient::ModelRef;
|
||||
ASSERT_THROW(R::parse("justaname"), ApiError);
|
||||
ASSERT_THROW(R::parse("a/b/c"), ApiError);
|
||||
ASSERT_THROW(R::parse("a/../b"), ApiError);
|
||||
ASSERT_THROW(R::parse("a/b:v/x"), ApiError);
|
||||
ASSERT_THROW(R::parse("/b"), ApiError);
|
||||
R ok = R::parse("o/n:v1");
|
||||
ASSERT_EQ(ok.owner, std::string("o"));
|
||||
ASSERT_EQ(ok.version, std::string("v1"));
|
||||
}
|
||||
|
||||
TEST(media_prepare, replicate_uses_the_v1_base_and_bearer_auth) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/v1/account", empty_ok());
|
||||
MockTransport* t = m.raw;
|
||||
auto c = ReplicateClient::with(m.take(), "tok");
|
||||
c->account();
|
||||
auto seen = t->seen();
|
||||
ASSERT_EQ(seen[0].target, std::string("/v1/account"));
|
||||
ASSERT_EQ(*seen[0].headers.get("Authorization"), std::string("Bearer tok"));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------ fal
|
||||
|
||||
TEST(media_prepare, fal_endpoint_ids_split_into_owner_app_path) {
|
||||
FalEndpoint e = FalEndpoint::parse("fal-ai/flux/dev");
|
||||
ASSERT_EQ(e.owner, std::string("fal-ai"));
|
||||
ASSERT_EQ(e.app, std::string("flux"));
|
||||
ASSERT_EQ(e.path, std::string("dev"));
|
||||
ASSERT_EQ(e.app_id(), std::string("fal-ai/flux"));
|
||||
FalEndpoint two = FalEndpoint::parse("fal-ai/veo3");
|
||||
ASSERT_EQ(two.path, std::string(""));
|
||||
FalEndpoint deep = FalEndpoint::parse("fal-ai/kling-video/v2.1/master/text-to-video");
|
||||
ASSERT_EQ(deep.path, std::string("v2.1/master/text-to-video"));
|
||||
ASSERT_THROW(FalEndpoint::parse("only"), ApiError);
|
||||
ASSERT_THROW(FalEndpoint::parse("a/../b"), ApiError);
|
||||
ASSERT_THROW(FalEndpoint::parse("a//b"), ApiError);
|
||||
ASSERT_THROW(FalEndpoint::parse("a/b/c/d/e/f"), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_prepare, fal_submit_carries_priority_header_and_webhook_query) {
|
||||
Mock m;
|
||||
auto q = FalQueue::with(m.take(), "id:secret");
|
||||
FalSubmitOptions o;
|
||||
o.priority = "high";
|
||||
o.webhook = "https://example.com/h";
|
||||
o.max_queue_length = 3;
|
||||
PreparedCall p = q->plan(FalEndpoint::parse("fal-ai/flux/dev"), obj({{"prompt", Json("x")}}), o);
|
||||
ASSERT_EQ(p.method, std::string("POST"));
|
||||
ASSERT_EQ(p.target, std::string("/fal-ai/flux/dev?fal_max_queue_length=3&fal_webhook=https%3A%2F%2Fexample.com%2Fh"));
|
||||
ASSERT_EQ(p.headers.at("X-Fal-Queue-Priority"), std::string("high"));
|
||||
ASSERT_EQ(p.body, std::string("{\"prompt\":\"x\"}"));
|
||||
PreparedCall two = q->plan(FalEndpoint::parse("fal-ai/veo3"), Json::object(), FalSubmitOptions{});
|
||||
ASSERT_EQ(two.target, std::string("/fal-ai/veo3"));
|
||||
FalSubmitOptions bad;
|
||||
bad.priority = "urgent";
|
||||
ASSERT_THROW(q->plan(FalEndpoint::parse("fal-ai/veo3"), Json::object(), bad), ApiError);
|
||||
}
|
||||
|
||||
TEST(media_prepare, fal_status_result_and_cancel_use_owner_app_only) {
|
||||
Mock m;
|
||||
auto q = FalQueue::with(m.take(), "k");
|
||||
ASSERT_THROW(q->status(FalEndpoint::parse("fal-ai/flux/dev"), "../x"), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
// FIXTURE: queue.json cancelQueueRequest example 202 (verbatim, https://fal.ai/docs/api-reference/inference-api/openapi/queue.json)
|
||||
serve(m.raw, Method::kPut, "/fal-ai/flux/requests/019a1b2c/cancel", reply("{\"status\": \"CANCELLATION_REQUESTED\"}", 202));
|
||||
serve(m.raw, Method::kGet, "/fal-ai/flux/requests/019a1b2c/status", empty_ok());
|
||||
Json r = q->cancel(FalEndpoint::parse("fal-ai/flux/dev"), "019a1b2c", true);
|
||||
ASSERT_EQ(r.find("status")->as_string(), std::string("CANCELLATION_REQUESTED"));
|
||||
q->status(FalEndpoint::parse("fal-ai/flux/dev"), "019a1b2c", true);
|
||||
auto seen = m.raw->seen();
|
||||
ASSERT_EQ(seen[0].method, std::string("PUT"));
|
||||
ASSERT_TRUE(has(seen[1].target, "/status?logs=1"));
|
||||
}
|
||||
|
||||
TEST(media_prepare, fal_sends_the_key_prefix_on_both_hosts) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/v1/meta", empty_ok());
|
||||
MockTransport* t = m.raw;
|
||||
auto p = FalPlatform::with(m.take(), "id:secret");
|
||||
p->meta();
|
||||
ASSERT_EQ(*t->seen()[0].headers.get("Authorization"), std::string("Key id:secret"));
|
||||
ASSERT_EQ(FalQueue::header_value(""), std::string(""));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------- Ideogram
|
||||
|
||||
TEST(media_prepare, ideogram_quote_is_a_free_dry_run_and_generate_is_paid) {
|
||||
Mock m;
|
||||
auto c = IdeogramClient::with(m.take(), "KEY");
|
||||
Json f = obj({{"prompt", Json("a cat")}, {"num_images", Json(1)}});
|
||||
PreparedCall q = c->plan("image.generate.ideogram-3", f, {}, true);
|
||||
ASSERT_EQ(q.target, std::string("/v2/image/generate/ideogram-3?dry_run=true"));
|
||||
ASSERT_TRUE(has(q.content_type, "multipart/form-data")); // ideogram-3 declares multipart only
|
||||
ASSERT_TRUE(q.effect == Effect::kPaid);
|
||||
PreparedCall j = c->plan("image.generate.ideogram-4", f, {}, false);
|
||||
ASSERT_EQ(j.content_type, std::string("application/json"));
|
||||
ASSERT_EQ(j.target, std::string("/v2/image/generate/ideogram-4"));
|
||||
ASSERT_THROW(c->plan("image.generate.ideogram-3", obj({{"nope", Json(1)}}), {}), ApiError);
|
||||
ASSERT_EQ(m.raw->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_prepare, ideogram_uses_the_api_key_header) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/v2/generations/abc", empty_ok());
|
||||
MockTransport* t = m.raw;
|
||||
auto c = IdeogramClient::with(m.take(), "KEY");
|
||||
c->generation("abc");
|
||||
ASSERT_EQ(*t->seen()[0].headers.get("Api-Key"), std::string("KEY"));
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------- Recraft
|
||||
|
||||
TEST(media_prepare, recraft_json_by_default_multipart_with_files) {
|
||||
Mock m;
|
||||
auto c = RecraftClient::with(m.take(), "KEY");
|
||||
PreparedCall j = c->plan("images.generations", obj({{"prompt", Json("x")}, {"n", Json(2)}}), {});
|
||||
ASSERT_EQ(j.target, std::string("/v1/images/generations"));
|
||||
ASSERT_EQ(j.body, std::string("{\"n\":2,\"prompt\":\"x\"}"));
|
||||
ASSERT_THROW(c->plan("images.generations", obj({{"prompt", Json("x")}}), {FilePart{"image", "a.png", "image/png", "x"}}),
|
||||
ApiError); // generations is JSON-only
|
||||
ASSERT_THROW(c->style("../x"), ApiError);
|
||||
ASSERT_THROW(c->delete_style("s1", false), ApiError); // gate
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------- Leonardo
|
||||
|
||||
TEST(media_prepare, leonardo_v2_create_wraps_model_and_parameters) {
|
||||
Mock m;
|
||||
auto c = LeonardoClient::with(m.take(), "KEY");
|
||||
Json b = LeonardoJobs::body("flux-dev", obj({{"prompt", Json("x")}}), false);
|
||||
PreparedCall p = c->plan(b);
|
||||
ASSERT_EQ(p.target, std::string("/v2/generations"));
|
||||
ASSERT_EQ(p.body, std::string("{\"model\":\"flux-dev\",\"parameters\":{\"prompt\":\"x\"},\"public\":false}"));
|
||||
ASSERT_THROW(c->generation("../x"), ApiError);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------- Runway
|
||||
|
||||
TEST(media_prepare, runway_sends_the_version_header_and_accepts_both_key_names) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/v1/organization", empty_ok());
|
||||
MockTransport* t = m.raw;
|
||||
auto c = RunwayClient::with(m.take(), "KEY");
|
||||
c->organization();
|
||||
ASSERT_EQ(*t->seen()[0].headers.get("X-Runway-Version"), std::string(RunwayClient::kApiVersion));
|
||||
ASSERT_EQ(*t->seen()[0].headers.get("Authorization"), std::string("Bearer KEY"));
|
||||
ASSERT_EQ(std::string(RunwayClient::kKeyEnv), std::string("RUNWAY_API_KEY"));
|
||||
ASSERT_EQ(std::string(RunwayClient::kKeyEnvSdk), std::string("RUNWAYML_API_SECRET"));
|
||||
}
|
||||
|
||||
TEST(media_prepare, runway_task_bodies_are_checked_against_the_sdk_field_union) {
|
||||
Mock m;
|
||||
auto c = RunwayClient::with(m.take(), "KEY");
|
||||
PreparedCall p = c->plan("text_to_video", obj({{"model", Json("gen4.5")}, {"promptText", Json("x")}, {"duration", Json(5)}}));
|
||||
ASSERT_EQ(p.target, std::string("/v1/text_to_video"));
|
||||
ASSERT_THROW(c->plan("text_to_video", obj({{"promt", Json("typo")}})), ApiError);
|
||||
ASSERT_THROW(c->task("a/b"), ApiError);
|
||||
ASSERT_THROW(c->cancel_or_delete_task("t1", false), ApiError);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------ Luma
|
||||
|
||||
TEST(media_prepare, luma_paths_and_the_dream_machine_base) {
|
||||
Mock m;
|
||||
serve(m.raw, Method::kGet, "/dream-machine/v1/credits", empty_ok());
|
||||
MockTransport* t = m.raw;
|
||||
auto c = LumaClient::with(m.take(), "KEY");
|
||||
PreparedCall p = c->plan("generations.create_video", obj({{"model", Json("ray-2")}, {"prompt", Json("x")}}));
|
||||
ASSERT_EQ(p.target, std::string("/generations/video"));
|
||||
c->credits();
|
||||
ASSERT_EQ(t->seen()[0].target, std::string("/dream-machine/v1/credits"));
|
||||
CallArgs a;
|
||||
a.path["id"] = "g1";
|
||||
ASSERT_EQ(c->prepare("generations.upscale", a).target, std::string("/generations/g1/upscale"));
|
||||
ASSERT_THROW(c->remove("g1", false), ApiError);
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------- MiniMax
|
||||
|
||||
TEST(media_prepare, minimax_v1_and_v2_rows) {
|
||||
Mock m;
|
||||
auto c = MinimaxClient::with(m.take(), "KEY");
|
||||
PreparedCall p = c->plan_video(obj({{"model", Json("MiniMax-Hailuo-2.3")}, {"prompt", Json("x")}, {"duration", Json(6)}}));
|
||||
ASSERT_EQ(p.target, std::string("/v1/video_generation"));
|
||||
CallArgs q;
|
||||
q.query["task_id"] = "t1";
|
||||
ASSERT_EQ(c->prepare("video.query", q).target, std::string("/v1/query/video_generation?task_id=t1"));
|
||||
CallArgs d;
|
||||
d.path["task_id"] = "t1";
|
||||
ASSERT_EQ(c->prepare("v2.video.delete", d).target, std::string("/v2/video_generation/t1"));
|
||||
ASSERT_THROW(c->delete_video_v2("t1", false), ApiError);
|
||||
Json ok = obj({{"base_resp", obj({{"status_code", Json(0)}, {"status_msg", Json("success")}})}});
|
||||
MinimaxClient::check_base_resp(ok); // no throw
|
||||
Json bad = obj({{"base_resp", obj({{"status_code", Json(1004)}, {"status_msg", Json("login fail")}})}});
|
||||
ASSERT_THROW(MinimaxClient::check_base_resp(bad), ApiError);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Higgsfield
|
||||
|
||||
TEST(media_prepare, higgsfield_typed_rows_for_known_paths_and_a_generic_row_for_the_rest) {
|
||||
ASSERT_EQ(HiggsfieldClient::row_for("higgsfield-ai/soul/standard"), std::string("higgsfield-ai.soul.standard"));
|
||||
ASSERT_EQ(HiggsfieldClient::row_for("vendor/new-model/v9"), std::string("model.submit"));
|
||||
Mock m;
|
||||
auto c = HiggsfieldClient::with(m.take(), "id", "secret");
|
||||
PreparedCall typed = c->plan("higgsfield-ai/soul/standard", obj({{"prompt", Json("x")}}));
|
||||
ASSERT_EQ(typed.target, std::string("/higgsfield-ai/soul/standard"));
|
||||
ASSERT_THROW(c->plan("higgsfield-ai/soul/standard", obj({{"nope", Json(1)}})), ApiError); // typed row validates fields
|
||||
PreparedCall generic = c->plan("vendor/new-model/v9", obj({{"anything", Json(1)}}));
|
||||
ASSERT_EQ(generic.target, std::string("/vendor/new-model/v9"));
|
||||
ASSERT_THROW(c->plan("vendor/../x", Json::object()), ApiError);
|
||||
ASSERT_EQ(HiggsfieldClient::header_value("id", "secret"), std::string("Key id:secret"));
|
||||
ASSERT_EQ(HiggsfieldClient::header_value("id", ""), std::string(""));
|
||||
}
|
||||
|
||||
TEST(media_prepare, higgsfield_submit_and_status_over_the_wire) {
|
||||
Mock m;
|
||||
// FIXTURE: docs.higgsfield.ai quickstart, submit reply (verbatim)
|
||||
const char* kSubmit =
|
||||
"{\"status\":\"queued\",\"request_id\":\"d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff\","
|
||||
"\"status_url\":\"https://api.higgsfield.ai/requests/d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff/status\","
|
||||
"\"cancel_url\":\"https://api.higgsfield.ai/requests/d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff/cancel\"}";
|
||||
serve(m.raw, Method::kPost, "/higgsfield-ai/soul/standard", reply(kSubmit));
|
||||
MockTransport* t = m.raw;
|
||||
auto c = HiggsfieldClient::with(m.take(), "id", "secret");
|
||||
Json out = c->submit("higgsfield-ai/soul/standard", obj({{"prompt", Json("Editorial portrait in soft daylight")}}), true);
|
||||
JobTicket ticket = HiggsfieldJobs::ticket_from(out, "higgsfield-ai/soul/standard", MediaKind::kImage);
|
||||
ASSERT_EQ(ticket.id, std::string("d7e6c0f3-6699-4f6c-bb45-2ad7fd9158ff"));
|
||||
auto seen = t->seen();
|
||||
ASSERT_EQ(*seen[0].headers.get("Authorization"), std::string("Key id:secret"));
|
||||
ASSERT_EQ(seen[0].body, std::string("{\"prompt\":\"Editorial portrait in soft daylight\"}"));
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------- Kling
|
||||
|
||||
TEST(media_prepare, kling_rows_and_envelope) {
|
||||
Mock m;
|
||||
auto c = std::make_shared<KlingClient>("AK", "SK", [] { return std::make_unique<MockTransport>(); });
|
||||
PreparedCall p = c->plan("text2video.create", obj({{"model_name", Json("kling-v1-6")}, {"prompt", Json("x")}}));
|
||||
ASSERT_EQ(p.target, std::string("/v1/videos/text2video"));
|
||||
ASSERT_TRUE(c->configured());
|
||||
KlingClient::check_envelope(obj({{"code", Json(0)}}));
|
||||
ASSERT_THROW(KlingClient::check_envelope(obj({{"code", Json(1001)}, {"message", Json("Authorization is empty")}})), ApiError);
|
||||
auto none = std::make_shared<KlingClient>("", "", [] { return std::make_unique<MockTransport>(); });
|
||||
ASSERT_FALSE(none->configured());
|
||||
}
|
||||
|
||||
// ------------------------------------------------------- confirmation gate
|
||||
|
||||
TEST(media_gate, non_read_calls_need_confirm_and_make_no_traffic_without_it) {
|
||||
Mock m;
|
||||
MockTransport* t = m.raw;
|
||||
auto bfl = BflClient::with(m.take(), "KEY");
|
||||
ASSERT_THROW(bfl->submit("flux-2-pro", obj({{"prompt", Json("x")}}), false), ApiError);
|
||||
ASSERT_THROW(bfl->delete_finetune("my-ft", false), ApiError);
|
||||
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
|
||||
Mock m2;
|
||||
MockTransport* t2 = m2.raw;
|
||||
auto rep = ReplicateClient::with(m2.take(), "KEY");
|
||||
ASSERT_THROW(rep->cancel_prediction("p1", false), ApiError);
|
||||
ASSERT_THROW(rep->file_delete("f1", false), ApiError);
|
||||
ASSERT_THROW(rep->file_upload("a.png", "image/png", "x", false), ApiError);
|
||||
ASSERT_THROW(rep->create_prediction(ReplicateClient::ModelRef::parse("o/n"), Json::object(), Json::object(), false), ApiError);
|
||||
ASSERT_EQ(t2->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(media_gate, missing_credentials_fail_before_any_traffic) {
|
||||
Mock m;
|
||||
MockTransport* t = m.raw;
|
||||
auto c = BflClient::with(m.take(), "");
|
||||
ASSERT_FALSE(c->configured());
|
||||
ASSERT_THROW(c->credits(), molo::llm::NotConfiguredError);
|
||||
ASSERT_THROW(c->submit("flux-2-pro", obj({{"prompt", Json("x")}}), true), molo::llm::NotConfiguredError);
|
||||
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
|
||||
ASSERT_FALSE(FalQueue::with(std::make_unique<MockTransport>(), "")->configured());
|
||||
ASSERT_FALSE(HiggsfieldClient::with(std::make_unique<MockTransport>(), "id", "")->configured());
|
||||
}
|
||||
|
||||
TEST(media_gate, headers_and_query_are_bounded) {
|
||||
Mock m;
|
||||
auto c = BflClient::with(m.take(), "KEY");
|
||||
CallArgs a;
|
||||
a.headers["Bad Name"] = "v";
|
||||
ASSERT_THROW(c->prepare("credits", a), ApiError);
|
||||
CallArgs b;
|
||||
b.headers["X-A"] = "line\r\nbreak";
|
||||
ASSERT_THROW(c->prepare("credits", b), ApiError);
|
||||
CallArgs d;
|
||||
d.query["id"] = std::string(2000, 'x');
|
||||
ASSERT_THROW(c->prepare("get_result", d), ApiError);
|
||||
CallArgs e;
|
||||
e.body = obj({{"x", Json(1)}});
|
||||
ASSERT_THROW(c->prepare("credits", e), ApiError); // a GET takes no body
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
// Webhook signature verification (Replicate, Standard Webhooks scheme) and the
|
||||
// Kling JWT. Both use vectors produced outside this code base.
|
||||
#include "support.h"
|
||||
#include "vendors/media/kling/kling.h"
|
||||
#include "vendors/media/replicate/replicate.h"
|
||||
|
||||
using namespace molo::media;
|
||||
using namespace media_test;
|
||||
|
||||
namespace {
|
||||
// Standard Webhooks published test vector (github.com/standard-webhooks): secret, id, timestamp, payload.
|
||||
const char* kSecret = "whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw";
|
||||
const char* kId = "msg_p5jXN8AQM9LWM0D4loKWxJek";
|
||||
const char* kTimestamp = "1614265330";
|
||||
const char* kPayload = "{\"test\": 2432232314}";
|
||||
const char* kSignature = "g0hM9SsE+OTPJTGt/tmIKtSyZlE3uFJELVlNIOLJ1OE=";
|
||||
} // namespace
|
||||
|
||||
TEST(media_signing, webhook_signature_matches_the_standard_webhooks_vector) {
|
||||
ASSERT_EQ(ReplicateWebhook::sign(kSecret, kId, kTimestamp, kPayload), std::string(kSignature));
|
||||
ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, std::string("v1,") + kSignature, kPayload, 1614265330 + 10));
|
||||
}
|
||||
|
||||
TEST(media_signing, webhook_rejects_tampering_staleness_and_malformed_input) {
|
||||
const std::string good = std::string("v1,") + kSignature;
|
||||
const long now = 1614265330 + 10;
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, "{\"test\": 2432232315}", now)); // body changed
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, "msg_other", kTimestamp, good, kPayload, now)); // id changed
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, "1614265331", good, kPayload, now)); // timestamp changed
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 + 301)); // too old
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 - 301)); // too new
|
||||
ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, good, kPayload, 1614265330 + 300)); // boundary
|
||||
ASSERT_FALSE(ReplicateWebhook::verify("whsec_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", kId, kTimestamp, good, kPayload, now)); // wrong secret
|
||||
ASSERT_FALSE(ReplicateWebhook::verify("whsec_!!notbase64!!", kId, kTimestamp, good, kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify("", kId, kTimestamp, good, kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, "", kTimestamp, good, kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, "12ab", good, kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "", kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, kSignature, kPayload, now)); // no v1, prefix
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "v2," + std::string(kSignature), kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, std::string(2000, 'x'), kPayload, now)); // header cap
|
||||
}
|
||||
|
||||
TEST(media_signing, webhook_accepts_any_of_several_signatures) {
|
||||
const long now = 1614265330;
|
||||
std::string rotated = "v1,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= v1," + std::string(kSignature);
|
||||
ASSERT_TRUE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, rotated, kPayload, now));
|
||||
ASSERT_FALSE(ReplicateWebhook::verify(kSecret, kId, kTimestamp, "v1,AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", kPayload, now));
|
||||
}
|
||||
|
||||
TEST(media_signing, kling_jwt_matches_an_independent_hs256_implementation) {
|
||||
// Computed with python3 hmac / base64 (see SPEC "Verified against spec"): header {"alg":"HS256","typ":"JWT"},
|
||||
// payload {"exp":1700001800,"iss":"AK-test","nbf":1699999995}, secret "SK-test".
|
||||
const char* expected =
|
||||
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MDAwMDE4MDAsImlzcyI6IkFLLXRlc3QiLCJuYmYiOjE2OTk5OTk5OTV9."
|
||||
"f3MPnq8kVzVYtGGChN2U_Rd1oYs3-a_FFYC4PCVGdSc";
|
||||
ASSERT_EQ(KlingJwt::make("AK-test", "SK-test", 1700000000), std::string(expected));
|
||||
ASSERT_TRUE(KlingJwt::make("AK-test", "SK-other", 1700000000) != std::string(expected));
|
||||
ASSERT_EQ(KlingJwt::kTtlSeconds, 1800L);
|
||||
ASSERT_TRUE(KlingJwt::kRenewAfterSeconds < KlingJwt::kTtlSeconds);
|
||||
}
|
||||
Reference in New Issue
Block a user