Files
moloapp/example/app/ai/HuggingfaceDemo.cpp
T

750 lines
35 KiB
C++

#include "../AiDemos.hpp"
#include <functional>
#include "../../fw/Router.hpp"
#include "../../fw/Templates.hpp"
#include "clients/url.h"
#include "security/crypto/Base64.h"
#include "vendors/huggingface/caps.h"
#include "vendors/huggingface/calls.h"
#include "vendors/huggingface/chat_model.h"
#include "vendors/huggingface/community.h"
#include "vendors/huggingface/endpoints.h"
#include "vendors/huggingface/hub.h"
#include "vendors/huggingface/platform.h"
#include "vendors/huggingface/providers.h"
#include "vendors/huggingface/servers.h"
namespace molo::example::app {
// Hugging Face over the real src/vendors/huggingface package. GET routes need
// no credential. Every route that calls Hugging Face is a POST: 422 on bad
// input, 503 huggingface_not_configured without HF_TOKEN. Routes that cost
// money (chat, tasks, creating endpoints) or change resources (discussions,
// collections, webhooks, endpoint lifecycle) are dry runs that show the exact
// request unless the body has "confirm":true.
namespace {
namespace hf = huggingface;
using Json = clients::body::Json;
constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // inference inputs travel as base64
const char kPage[] = R"HUGGINGFACE({% extends "base.html" %}
{% block content %}
<h1>Hugging Face</h1>
<p class="hint">Hub (models, datasets, spaces, repos, search, papers, discussions, collections,
webhooks), Inference Providers router chat, hf-inference tasks, Inference Endpoints management,
TGI and TEI servers. Paid and changing calls are dry runs until the body says
<code>"confirm":true</code>.</p>
<p id="hf-config" class="hint">Checking credential...</p>
<section class="hf">
<label>Route
<select id="hf-route"></select></label>
<label>Body (JSON object) <textarea id="hf-body" rows="8"></textarea></label>
<div class="hf-actions">
<button id="hf-send" type="button" class="primary">Send</button>
<button id="hf-confirm" type="button">Send with confirm</button>
</div>
<p id="hf-error" class="error" role="alert"></p>
<pre id="hf-out" class="hf-out"></pre>
</section>
<style>
.hf label{display:block;margin:.5rem 0}
.hf select,.hf textarea{width:100%;box-sizing:border-box}
.hf-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
.hf-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
</style>
<script src="/embedded/ai_huggingface.js"></script>
{% endblock %}
)HUGGINGFACE";
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
fw::Response json_out(const Json& j, int status = 200) {
fw::Response res = fw::JsonResponse(j.dump(), status);
res.headers.set("Cache-Control", "no-store");
return res;
}
Json parse_body(const fw::Request& req) {
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
if (req.body.empty()) return Json::object();
try {
Json j = Json::parse(req.body);
if (!j.is_object()) bad("body must be a JSON object");
return j;
} catch (const clients::body::JsonError&) {
bad("body is not valid JSON");
}
}
std::string text(const Json& b, const char* key, bool required = false) {
const Json* v = b.find(key);
if (!v || v->is_null()) {
if (required) bad(std::string(key) + " is required");
return "";
}
if (!v->is_string()) bad(std::string(key) + " must be a string");
if (required && v->as_string().empty()) bad(std::string(key) + " is required");
return v->as_string();
}
int integer(const Json& b, const char* key, int fallback, int lo, int hi) {
const Json* v = b.find(key);
if (!v || v->is_null()) return fallback;
if (!v->is_number()) bad(std::string(key) + " must be a number");
long n = v->as_int();
if (n < lo || n > hi) bad(std::string(key) + " must be " + std::to_string(lo) + ".." + std::to_string(hi));
return static_cast<int>(n);
}
bool flag(const Json& b, const char* key) {
const Json* v = b.find(key);
if (!v || v->is_null()) return false;
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
return v->as_bool();
}
std::vector<std::string> strings(const Json& b, const char* key) {
std::vector<std::string> out;
const Json* v = b.find(key);
if (!v || v->is_null()) return out;
if (!v->is_array()) bad(std::string(key) + " must be an array of strings");
for (const Json& s : v->as_array()) {
if (!s.is_string()) bad(std::string(key) + " must be an array of strings");
out.push_back(s.as_string());
}
return out;
}
hf::RepoType repo_type(const Json& b) {
std::string k = text(b, "kind");
if (k.empty()) return hf::RepoType::kModel;
hf::RepoType t;
if (!hf::parse_repo_type(k, t)) bad("kind must be models, datasets or spaces");
return t;
}
hf::RepoRef repo_ref(const Json& b, const char* key = "id") {
try {
return hf::RepoRef::parse(text(b, key, true));
} catch (const llm::ApiError& e) {
bad(e.message());
}
}
Json dry_run(const Json& request, const std::string& path) {
Json o = Json::object();
o.set("dry_run", Json(true));
o.set("would_send_to", Json(path));
o.set("request", request);
o.set("hint", Json("send confirm:true to run it"));
return o;
}
Json page_json(const hf::Page& p) {
Json o = Json::object();
o.set("items", p.items);
o.set("pages", Json(static_cast<int>(p.pages)));
o.set("truncated", Json(p.truncated));
o.set("truncated_by", Json(p.truncated_by));
o.set("next_cursor", Json(p.next_cursor));
return o;
}
struct Plan {
Json request; // what would be sent (shown in a dry run)
std::string path; // for the dry-run message
bool needs_confirm = false; // paid or changes a resource
std::function<Json()> call;
};
// Servers (TGI, TEI) get the token only when they are Hugging Face hosted.
std::string token_for(const std::string& url, const std::string& token) {
clients::Url u = clients::Url::parse(url);
const std::string& h = u.host();
auto ends = [&](const char* suffix) {
std::string s(suffix);
return h.size() >= s.size() && h.compare(h.size() - s.size(), s.size(), s) == 0;
};
return (ends(".huggingface.cloud") || ends(".hf.space")) ? token : std::string();
}
class HuggingfaceRoutes {
public:
HuggingfaceRoutes() : p_(hf::Platform::from_env()), token_(hf::Platform::token_from_env()) {}
const hf::Platform& platform() const { return p_; }
using Planner = std::function<Plan(const Json&)>;
fw::Response run(const fw::Request& req, const Planner& planner) const {
Json body = parse_body(req);
Plan plan;
try {
plan = planner(body);
} catch (const llm::ApiError& e) {
map(e);
}
if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.path));
if (!p_.configured()) throw fw::HttpError(503, "huggingface_not_configured", "no credential configured (set HF_TOKEN)");
try {
return json_out(plan.call());
} catch (const llm::ApiError& e) {
map(e);
}
}
// ---- Hub --------------------------------------------------------------------
Plan hub_list(const Json& b) const {
hf::RepoType type = repo_type(b);
hf::RepoListParams lp;
lp.search = text(b, "search");
lp.author = text(b, "author");
lp.sort = text(b, "sort");
lp.direction = text(b, "direction");
lp.pipeline_tag = text(b, "pipeline_tag");
lp.filter = strings(b, "filter");
lp.limit = integer(b, "limit", 20, 1, 100);
const int pages = integer(b, "pages", 1, 1, 5);
hf::RequestPlan::make(hf::Hub::list_call(type), hf::PathArgs(), lp.to_query());
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
Json req = Json::object();
req.set("kind", Json(hf::repo_type_plural(type)));
return Plan{req, std::string("GET /api/") + hf::repo_type_plural(type), false, [=] {
return page_json(hub->list_repos(type, lp, static_cast<std::size_t>(lp.limit) * pages,
static_cast<std::size_t>(pages)));
}};
}
Plan hub_repo(const Json& b) const {
const std::string action = text(b, "action", true);
hf::RepoType type = repo_type(b);
hf::RepoRef ref = repo_ref(b);
const std::string rev = text(b, "rev").empty() ? "main" : text(b, "rev");
const std::string path = text(b, "path");
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
Json req = Json::object();
req.set("action", Json(action));
req.set("id", Json(ref.id()));
auto make = [&](std::function<Json()> fn) { return Plan{req, "GET /api/" + std::string(hf::repo_type_plural(type)) + "/" + ref.id(), false, std::move(fn)}; };
if (action == "info") {
const std::string revision = text(b, "revision");
return make([=] { return hub->repo_info(type, ref, revision, strings(b, "expand")); });
}
if (action == "tree") {
const bool rec = flag(b, "recursive");
const int lim = integer(b, "limit", 0, 0, 1000);
return make([=] { return hub->tree(type, ref, rev, path, rec, lim); });
}
if (action == "refs") return make([=] { return hub->refs(type, ref, flag(b, "include_prs")); });
if (action == "commits") {
const int lim = integer(b, "limit", 20, 1, 100);
return make([=] { return hub->commits(type, ref, rev, lim); });
}
if (action == "compare") {
const std::string cmp = text(b, "compare", true);
return make([=] { return hub->compare(type, ref, cmp); });
}
if (action == "likers") return make([=] { return hub->likers(type, ref); });
if (action == "paths_info") {
std::vector<std::string> paths = strings(b, "paths");
return make([=] { return hub->paths_info(type, ref, rev, paths); });
}
if (action == "safetensors") {
const std::string p = text(b, "path", true);
return make([=] { return hub->safetensors_metadata(ref, p); });
}
if (action == "file") {
const std::string p = text(b, "path", true);
hf::RequestPlan::encode_file_path(p);
return make([=] {
hf::Download d = hub->download(type, ref, rev, p);
Json o = Json::object();
o.set("bytes", Json(static_cast<int>(d.bytes.size())));
o.set("content_type", Json(d.content_type));
o.set("commit", Json(d.commit));
o.set("etag", Json(d.etag));
const bool texty = d.content_type.find("text") != std::string::npos || d.content_type.find("json") != std::string::npos;
o.set("preview", Json(texty ? d.bytes.substr(0, 20000) : std::string()));
return o;
});
}
bad("action must be info, tree, refs, commits, compare, likers, paths_info, safetensors or file");
}
Plan hub_search(const Json& b) const {
const std::string action = text(b, "action", true);
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
Json req = Json::object();
req.set("action", Json(action));
auto make = [&](std::function<Json()> fn) { return Plan{req, "GET /api (" + action + ")", false, std::move(fn)}; };
const int lim = integer(b, "limit", 10, 1, 100);
if (action == "quicksearch") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->quicksearch(q, lim, t); }); }
if (action == "fulltext") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->full_text_search(q, lim, t); }); }
if (action == "trending") { std::string t = text(b, "type"); return make([=] { return hub->trending(t, lim); }); }
if (action == "tags") { hf::RepoType t = repo_type(b); return make([=] { return hub->tags_by_type(t); }); }
if (action == "tasks") return make([=] { return hub->tasks(); });
if (action == "hardware") return make([=] { return hub->spaces_hardware(); });
if (action == "featured") return make([=] { return hub->spaces_featured(); });
if (action == "semantic") { std::string q = text(b, "q", true); return make([=] { return hub->spaces_semantic_search(q); }); }
if (action == "papers_daily") return make([=] { return hub->daily_papers(lim, ""); });
if (action == "papers_search") { std::string q = text(b, "q", true); return make([=] { return hub->papers_search(q, lim); }); }
if (action == "paper") { std::string id = text(b, "id", true); return make([=] { return hub->paper(id); }); }
if (action == "partners") { std::string pr = text(b, "provider", true); return make([=] { return hub->partner_models(pr, text(b, "status")); }); }
bad("action must be quicksearch, fulltext, trending, tags, tasks, hardware, featured, semantic, papers_daily, papers_search, paper or partners");
}
Plan hub_whoami(const Json&) const {
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
return Plan{Json::object(), "GET /api/whoami-v2", false, [hub] { return hub->whoami(); }};
}
Plan hub_discussions(const Json& b) const {
const std::string action = text(b, "action", true);
hf::RepoType type = repo_type(b);
hf::RepoRef ref = repo_ref(b);
auto api = std::make_shared<hf::Discussions>(p_.hub);
Json req = Json::object();
req.set("action", Json(action));
req.set("id", Json(ref.id()));
const std::string base = std::string("/api/") + hf::repo_type_plural(type) + "/" + ref.id() + "/discussions";
const int num = integer(b, "num", 0, 0, 999999999);
if (action == "list") {
hf::DiscussionFilter f;
f.page = integer(b, "page", 0, 0, 100000);
f.kind = text(b, "type");
f.status = text(b, "status");
f.author = text(b, "author");
f.search = text(b, "search");
return Plan{req, "GET " + base, false, [=] { return api->list(type, ref, f); }};
}
if (action == "get") {
if (!num) bad("num is required");
return Plan{req, "GET " + base + "/{num}", false, [=] { return api->get(type, ref, num); }};
}
if (action == "create") {
std::string title = text(b, "title", true), desc = text(b, "description");
bool pr = flag(b, "pull_request");
req.set("title", Json(title));
return Plan{req, "POST " + base, true, [=] { return api->create(type, ref, title, desc, pr); }};
}
if (!num) bad("num is required");
req.set("num", Json(num));
if (action == "comment") {
std::string c = text(b, "comment", true);
return Plan{req, "POST " + base + "/{num}/comment", true, [=] { return api->comment(type, ref, num, c); }};
}
if (action == "status") {
std::string s = text(b, "status", true), c = text(b, "comment");
return Plan{req, "POST " + base + "/{num}/status", true, [=] { return api->change_status(type, ref, num, s, c); }};
}
if (action == "rename") {
std::string t = text(b, "title", true);
return Plan{req, "POST " + base + "/{num}/title", true, [=] { return api->rename(type, ref, num, t); }};
}
if (action == "remove")
return Plan{req, "DELETE " + base + "/{num}", true, [=] { return api->remove_discussion(type, ref, num); }};
bad("action must be list, get, create, comment, status, rename or remove");
}
Plan hub_collections(const Json& b) const {
const std::string action = text(b, "action", true);
auto api = std::make_shared<hf::Collections>(p_.hub);
Json req = Json::object();
req.set("action", Json(action));
if (action == "list") {
hf::CollectionListParams lp;
lp.owner = text(b, "owner");
lp.q = text(b, "q");
lp.sort = text(b, "sort");
lp.limit = integer(b, "limit", 20, 1, 100);
lp.to_query();
return Plan{req, "GET /api/collections", false, [=] { return page_json(api->list(lp, 100, 2)); }};
}
const std::string owner = text(b, "owner", true), slug = text(b, "slug");
req.set("owner", Json(owner));
if (action == "create") {
std::string title = text(b, "title", true), desc = text(b, "description");
bool priv = flag(b, "private");
req.set("title", Json(title));
return Plan{req, "POST /api/collections", true, [=] { return api->create(owner, title, desc, priv); }};
}
if (slug.empty()) bad("slug is required");
req.set("slug", Json(slug));
if (action == "get") return Plan{req, "GET /api/collections/{owner}/{slug}", false, [=] { return api->get(owner, slug); }};
if (action == "update") {
const Json* f = b.find("fields");
if (!f || !f->is_object()) bad("fields must be an object");
Json fields = *f;
return Plan{req, "PATCH /api/collections/{owner}/{slug}", true, [=] { return api->update(owner, slug, fields); }};
}
if (action == "add_item") {
hf::CollectionItem item;
item.type = text(b, "item_type", true);
item.id = text(b, "item_id", true);
item.to_json();
std::string note = text(b, "note");
return Plan{req, "POST /api/collections/{owner}/{slug}/items", true, [=] { return api->add_item(owner, slug, item, note); }};
}
if (action == "remove")
return Plan{req, "DELETE /api/collections/{owner}/{slug}", true, [=] { return api->remove_collection(owner, slug); }};
bad("action must be list, get, create, update, add_item or remove");
}
Plan hub_webhooks(const Json& b) const {
const std::string action = text(b, "action", true);
auto api = std::make_shared<hf::Webhooks>(p_.hub);
Json req = Json::object();
req.set("action", Json(action));
auto spec_of = [&]() {
hf::WebhookSpec s;
s.url = text(b, "url", true);
s.domains = strings(b, "domains");
s.secret = text(b, "secret");
const Json* w = b.find("watched");
if (!w || !w->is_array()) bad("watched must be an array of {type,name}");
for (const Json& x : w->as_array()) {
if (!x.is_object()) bad("watched entries must be objects");
s.watched.push_back({text(x, "type", true), text(x, "name", true)});
}
s.to_json();
return s;
};
if (action == "list") return Plan{req, "GET /api/settings/webhooks", false, [=] { return api->list(); }};
if (action == "create") {
hf::WebhookSpec s = spec_of();
req.set("url", Json(s.url));
return Plan{req, "POST /api/settings/webhooks", true, [=] { return api->create(s); }};
}
const std::string id = text(b, "id", true);
req.set("id", Json(id));
if (action == "get") return Plan{req, "GET /api/settings/webhooks/{id}", false, [=] { return api->get(id); }};
if (action == "update") {
hf::WebhookSpec s = spec_of();
return Plan{req, "POST /api/settings/webhooks/{id}", true, [=] { return api->update(id, s); }};
}
if (action == "enable") return Plan{req, "POST /api/settings/webhooks/{id}/enable", true, [=] { return api->enable(id); }};
if (action == "disable") return Plan{req, "POST /api/settings/webhooks/{id}/disable", true, [=] { return api->disable(id); }};
if (action == "replay") {
std::string log = text(b, "log_id", true);
return Plan{req, "POST /api/settings/webhooks/{id}/replay/{log_id}", true, [=] { return api->replay(id, log); }};
}
if (action == "remove") return Plan{req, "DELETE /api/settings/webhooks/{id}", true, [=] { return api->remove_webhook(id); }};
bad("action must be list, get, create, update, enable, disable, replay or remove");
}
// ---- Inference ---------------------------------------------------------------
llm::ChatRequest chat_request(const Json& b) const {
llm::ChatRequest r;
r.model = text(b, "model", true);
hf::RouterModel::parse(r.model);
r.system = text(b, "system");
r.messages.push_back(llm::Message::user(text(b, "prompt", true)));
if (b.find("max_tokens")) r.max_tokens = integer(b, "max_tokens", 256, 1, hf::HfCaps::kMaxTokensCeiling);
if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number();
r.json_mode = flag(b, "json");
r.reasoning_effort = text(b, "effort");
return r;
}
Plan chat(const Json& b) const {
llm::ChatRequest r = chat_request(b);
auto model = hf::HuggingfaceChatModel::over_router(p_, r.model);
Json body = model->to_body(r, false);
return Plan{body, "/v1/chat/completions", true, [model, r] {
llm::ChatResponse res = model->generate(r);
Json o = Json::object();
o.set("id", Json(res.id));
o.set("model", Json(res.model));
o.set("text", Json(res.text()));
o.set("finish_reason", Json(res.finish_reason));
Json u = Json::object();
u.set("input_tokens", Json(res.usage.input_tokens));
u.set("output_tokens", Json(res.usage.output_tokens));
u.set("total_tokens", Json(res.usage.total_tokens));
o.set("usage", u);
return o;
}};
}
Plan chat_stream(const Json& b) const {
llm::ChatRequest r = chat_request(b);
auto model = hf::HuggingfaceChatModel::over_router(p_, r.model);
Json body = model->to_body(r, true);
return Plan{body, "/v1/chat/completions (stream)", true, [model, r] {
std::string content;
std::size_t chunks = 0;
std::string finish;
model->stream(r, [&](const llm::ChatChunk& c) {
content += c.text_delta;
if (!c.finish_reason.empty()) finish = c.finish_reason;
return ++chunks < hf::HfCaps::kMaxStreamEvents;
});
Json o = Json::object();
o.set("text", Json(content));
o.set("finish_reason", Json(finish));
o.set("chunks", Json(static_cast<int>(chunks)));
return o;
}};
}
Plan task(const Json& b) const {
const std::string task = text(b, "task", true), model = text(b, "model", true);
const hf::TaskInfo* info = hf::TaskCatalog::find(task);
if (!info) bad("unknown task");
hf::RequestPlan::check_model(model);
Json params = b.find("parameters") ? *b.find("parameters") : Json();
auto api = std::make_shared<hf::InferenceTasks>(hf::InferenceTasks::from_platform(p_));
Json req = Json::object();
req.set("task", Json(task));
req.set("model", Json(model));
auto result_json = [](const hf::InferenceResult& r) {
Json o = Json::object();
o.set("content_type", Json(r.content_type));
if (r.is_json) {
o.set("result", r.json);
} else {
o.set("bytes", Json(static_cast<int>(r.bytes.size())));
o.set("data_base64", Json(security::crypto::Base64::encode(r.bytes.substr(0, 1024 * 1024))));
}
return o;
};
if (info->binary_input) {
std::string bytes;
if (!security::crypto::Base64::decode(text(b, "data_base64", true), bytes)) bad("data_base64 is not valid base64");
std::string mime = text(b, "mime", true);
req.set("bytes", Json(static_cast<int>(bytes.size())));
return Plan{req, "POST /hf-inference/models/{model}", true,
[=] { return result_json(api->run_binary(task, model, bytes, mime, params)); }};
}
const Json* in = b.find("inputs");
if (!in || in->is_null()) bad("inputs is required");
Json inputs = *in;
req.set("inputs", inputs);
return Plan{req, "POST /hf-inference/models/{model}", true,
[=] { return result_json(api->run_json(task, model, inputs, params)); }};
}
Plan endpoints(const Json& b) const {
const std::string action = text(b, "action", true);
const std::string ns = text(b, "namespace");
const std::string name = text(b, "name");
auto api = std::make_shared<hf::Endpoints>(hf::Endpoints::from_platform(p_));
Json req = Json::object();
req.set("action", Json(action));
if (!ns.empty()) req.set("namespace", Json(ns));
if (!name.empty()) req.set("name", Json(name));
auto need_ns = [&] { if (ns.empty()) bad("namespace is required"); };
auto need_name = [&] { need_ns(); if (name.empty()) bad("name is required"); hf::Endpoints::validate_name(name); };
if (action == "vendors") return Plan{req, "GET /v2/provider", false, [=] { return api->vendors(); }};
if (action == "list") { need_ns(); return Plan{req, "GET /v2/endpoint/{ns}", false, [=] { return api->list(ns); }}; }
if (action == "search") { need_ns(); std::string q = text(b, "q"); return Plan{req, "GET /v2/search/{ns}", false, [=] { return api->search(ns, q); }}; }
if (action == "states") { need_ns(); return Plan{req, "GET /v2/search/{ns}/states", false, [=] { return api->state_counts(ns); }}; }
if (action == "quotas") { need_ns(); return Plan{req, "GET /v2/provider/quotas/{ns}", false, [=] { return api->quotas(ns); }}; }
if (action == "audit") { need_ns(); return Plan{req, "GET /v2/audit-logs/{ns}", false, [=] { return api->audit_logs(ns); }}; }
if (action == "get") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}", false, [=] { return api->get(ns, name); }}; }
if (action == "replicas") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}/replica", false, [=] { return api->replicas(ns, name); }}; }
if (action == "logs") {
need_name();
int tail = integer(b, "tail", 100, 1, 1000);
return Plan{req, "GET /v2/endpoint/{ns}/{name}/logs", false, [=] {
Json o = Json::object();
o.set("logs", Json(api->logs(ns, name, tail)));
return o;
}};
}
if (action == "metrics") {
need_name();
hf::MetricKind k;
if (!hf::parse_metric_kind(text(b, "kind").empty() ? "all" : text(b, "kind"), k)) bad("unknown metric kind");
std::string start = text(b, "start", true), stop = text(b, "stop", true);
return Plan{req, "POST /v2/endpoint/{ns}/{name}/metrics", false, [=] { return api->metrics(k, ns, name, start, stop); }};
}
if (action == "create") {
need_ns();
const Json* s = b.find("spec");
if (!s || !s->is_object()) bad("spec must be an object");
Json spec = *s;
hf::Endpoints::validate_spec(spec, true);
req.set("spec", spec);
return Plan{req, "POST /v2/endpoint/{ns} (costs money while it runs)", true, [=] { return api->create(ns, spec); }};
}
if (action == "update") {
need_name();
const Json* s = b.find("spec");
if (!s || !s->is_object()) bad("spec must be an object");
Json spec = *s;
hf::Endpoints::validate_spec(spec, false);
return Plan{req, "PUT /v2/endpoint/{ns}/{name}", true, [=] { return api->update(ns, name, spec); }};
}
if (action == "pause") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/pause", true, [=] { return api->pause(ns, name); }}; }
if (action == "resume") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/resume", true, [=] { return api->resume(ns, name); }}; }
if (action == "scale_to_zero") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/scale-to-zero", true, [=] { return api->scale_to_zero(ns, name); }}; }
if (action == "remove") { need_name(); return Plan{req, "DELETE /v2/endpoint/{ns}/{name}", true, [=] { return api->remove_endpoint(ns, name); }}; }
bad("action must be vendors, list, search, states, quotas, audit, get, replicas, logs, metrics, create, update, pause, resume, scale_to_zero or remove");
}
hf::Access server_access(const Json& b) const {
const std::string url = text(b, "url", true);
try {
return hf::Platform::server(url, token_for(hf::Platform::check_server_url(url), token_));
} catch (const clients::UrlError& e) {
bad(e.what());
}
}
Plan tgi(const Json& b) const {
const std::string action = text(b, "action", true);
auto api = std::make_shared<hf::TgiServer>(server_access(b));
Json req = Json::object();
req.set("action", Json(action));
if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }};
if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }};
if (action == "models") return Plan{req, "GET /v1/models", false, [=] { return api->models(); }};
if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }};
Json params = b.find("parameters") ? *b.find("parameters") : Json();
if (action == "generate") { std::string in = text(b, "inputs", true); return Plan{req, "POST /generate", false, [=] { return api->generate(in, params); }}; }
if (action == "tokenize") { std::string in = text(b, "inputs", true); return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(in, params); }}; }
if (action == "chat") {
llm::ChatRequest r = chat_request(b);
hf::ChatWire::to_body(r, "tgi", false);
auto model = std::make_shared<hf::HuggingfaceChatModel>(api, r.model);
return Plan{req, "POST /v1/chat/completions", false, [=] {
Json o = Json::object();
o.set("text", Json(model->generate(r).text()));
return o;
}};
}
bad("action must be health, info, models, metrics, generate, tokenize or chat");
}
Plan tei(const Json& b) const {
const std::string action = text(b, "action", true);
auto api = std::make_shared<hf::TeiServer>(server_access(b));
Json req = Json::object();
req.set("action", Json(action));
if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }};
if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }};
if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }};
if (action == "rerank") {
std::string q = text(b, "query", true);
std::vector<std::string> texts = strings(b, "texts");
return Plan{req, "POST /rerank", false, [=] { return api->rerank(q, texts, flag(b, "return_text")); }};
}
const Json* in = b.find("inputs");
if (!in || in->is_null()) bad("inputs is required");
Json inputs = *in;
hf::TeiServer::check_inputs(inputs);
if (action == "embed") return Plan{req, "POST /embed", false, [=] { return api->embed(inputs); }};
if (action == "predict") return Plan{req, "POST /predict", false, [=] { return api->predict(inputs); }};
if (action == "tokenize") return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(inputs); }};
bad("action must be health, info, metrics, embed, predict, rerank or tokenize");
}
private:
[[noreturn]] static void map(const llm::ApiError& e) {
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
throw fw::HttpError(503, "huggingface_not_configured", e.message());
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
throw fw::HttpError(422, "validation_error", e.message());
if (e.type() == "LOCAL_BUDGET") throw fw::HttpError(429, "huggingface_local_budget", e.message());
throw fw::HttpError(502, "huggingface_error", e.type() + ": " + e.message());
}
hf::Platform p_;
std::string token_;
};
} // namespace
void register_ai_huggingface_demo(fw::Router& router, Services& services) {
auto routes = std::make_shared<HuggingfaceRoutes>();
fw::Environment* env = services.templates.get();
services.templates->add_template("ai_huggingface.html", kPage);
services.add_feature("ai-huggingface", "/ai/huggingface");
router.get("/ai/huggingface", "ai-huggingface-page", [env](fw::Request req) {
(void)req;
fw::TemplateCtx ctx;
ctx["title"] = std::string("Hugging Face");
return fw::Response(200, env->render("ai_huggingface.html", ctx), "text/html; charset=utf-8");
});
router.get("/ai/huggingface/status", "ai-huggingface-status", [routes](fw::Request req) {
(void)req;
Json o = Json::object();
o.set("configured", Json(routes->platform().configured()));
o.set("credential_env", Json(hf::Platform::kKeyEnv));
o.set("hub", Json(hf::Platform::kHubHost));
o.set("router", Json(hf::Platform::kRouterHost));
o.set("endpoints", Json(hf::Platform::kEndpointsHost));
o.set("calls", Json(static_cast<int>(hf::Calls::count())));
o.set("tasks", Json(static_cast<int>(hf::TaskCatalog::count())));
o.set("budget_calls", Json(hf::HfCaps::kBudgetMaxCalls));
o.set("budget_window_seconds", Json(hf::HfCaps::kBudgetWindowSeconds));
o.set("max_pages", Json(static_cast<int>(hf::HfCaps::kMaxPages)));
return json_out(o);
});
router.get("/ai/huggingface/calls", "ai-huggingface-calls", [](fw::Request req) {
(void)req;
Json arr = Json::array();
for (std::size_t i = 0; i < hf::Calls::count(); ++i) {
const hf::CallSpec& s = hf::Calls::at(i);
Json row = Json::object();
row.set("surface", Json(hf::surface_name(s.surface)));
row.set("http", Json(s.http_method));
row.set("path", Json(s.path));
row.set("spec", Json(s.spec));
row.set("auth", Json(s.auth));
row.set("flags", Json(s.flags));
arr.push(row);
}
Json o = Json::object();
o.set("calls", arr);
return json_out(o);
});
router.get("/ai/huggingface/tasks", "ai-huggingface-tasks", [](fw::Request req) {
(void)req;
Json arr = Json::array();
for (std::size_t i = 0; i < hf::TaskCatalog::count(); ++i) {
const hf::TaskInfo& t = hf::TaskCatalog::at(i);
Json row = Json::object();
row.set("task", Json(t.name));
row.set("binary_input", Json(t.binary_input));
row.set("pipeline_path", Json(t.pipeline_path));
row.set("note", Json(t.note));
arr.push(row);
}
Json o = Json::object();
o.set("tasks", arr);
return json_out(o);
});
using Member = HuggingfaceRoutes::Planner;
HuggingfaceRoutes* r = routes.get();
auto add = [&](const char* path, const char* name, Plan (HuggingfaceRoutes::*m)(const Json&) const) {
Member planner = [r, m](const Json& b) { return (r->*m)(b); };
router.post(path, name, [routes, planner](fw::Request req) { return routes->run(req, planner); });
};
add("/ai/huggingface/hub/list", "ai-huggingface-hub-list", &HuggingfaceRoutes::hub_list);
add("/ai/huggingface/hub/repo", "ai-huggingface-hub-repo", &HuggingfaceRoutes::hub_repo);
add("/ai/huggingface/hub/search", "ai-huggingface-hub-search", &HuggingfaceRoutes::hub_search);
add("/ai/huggingface/hub/whoami", "ai-huggingface-hub-whoami", &HuggingfaceRoutes::hub_whoami);
add("/ai/huggingface/hub/discussions", "ai-huggingface-hub-discussions", &HuggingfaceRoutes::hub_discussions);
add("/ai/huggingface/hub/collections", "ai-huggingface-hub-collections", &HuggingfaceRoutes::hub_collections);
add("/ai/huggingface/hub/webhooks", "ai-huggingface-hub-webhooks", &HuggingfaceRoutes::hub_webhooks);
add("/ai/huggingface/chat", "ai-huggingface-chat", &HuggingfaceRoutes::chat);
add("/ai/huggingface/chat/stream", "ai-huggingface-chat-stream", &HuggingfaceRoutes::chat_stream);
add("/ai/huggingface/task", "ai-huggingface-task", &HuggingfaceRoutes::task);
add("/ai/huggingface/endpoints", "ai-huggingface-endpoints", &HuggingfaceRoutes::endpoints);
add("/ai/huggingface/tgi", "ai-huggingface-tgi", &HuggingfaceRoutes::tgi);
add("/ai/huggingface/tei", "ai-huggingface-tei", &HuggingfaceRoutes::tei);
}
} // namespace molo::example::app