#include "../AiDemos.hpp" #include #include "../../fw/Router.hpp" #include "../../fw/Templates.hpp" #include "clients/url.h" #include "security/crypto/Base64.h" #include "vendors/huggingface/caps.h" #include "vendors/huggingface/calls.h" #include "vendors/huggingface/chat_model.h" #include "vendors/huggingface/community.h" #include "vendors/huggingface/endpoints.h" #include "vendors/huggingface/hub.h" #include "vendors/huggingface/platform.h" #include "vendors/huggingface/providers.h" #include "vendors/huggingface/servers.h" namespace molo::example::app { // Hugging Face over the real src/vendors/huggingface package. GET routes need // no credential. Every route that calls Hugging Face is a POST: 422 on bad // input, 503 huggingface_not_configured without HF_TOKEN. Routes that cost // money (chat, tasks, creating endpoints) or change resources (discussions, // collections, webhooks, endpoint lifecycle) are dry runs that show the exact // request unless the body has "confirm":true. namespace { namespace hf = huggingface; using Json = clients::body::Json; constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // inference inputs travel as base64 const char kPage[] = R"HUGGINGFACE({% extends "base.html" %} {% block content %}

Hugging Face

Hub (models, datasets, spaces, repos, search, papers, discussions, collections, webhooks), Inference Providers router chat, hf-inference tasks, Inference Endpoints management, TGI and TEI servers. Paid and changing calls are dry runs until the body says "confirm":true.

Checking credential...


{% endblock %} )HUGGINGFACE"; [[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } fw::Response json_out(const Json& j, int status = 200) { fw::Response res = fw::JsonResponse(j.dump(), status); res.headers.set("Cache-Control", "no-store"); return res; } Json parse_body(const fw::Request& req) { if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); if (req.body.empty()) return Json::object(); try { Json j = Json::parse(req.body); if (!j.is_object()) bad("body must be a JSON object"); return j; } catch (const clients::body::JsonError&) { bad("body is not valid JSON"); } } std::string text(const Json& b, const char* key, bool required = false) { const Json* v = b.find(key); if (!v || v->is_null()) { if (required) bad(std::string(key) + " is required"); return ""; } if (!v->is_string()) bad(std::string(key) + " must be a string"); if (required && v->as_string().empty()) bad(std::string(key) + " is required"); return v->as_string(); } int integer(const Json& b, const char* key, int fallback, int lo, int hi) { const Json* v = b.find(key); if (!v || v->is_null()) return fallback; if (!v->is_number()) bad(std::string(key) + " must be a number"); long n = v->as_int(); if (n < lo || n > hi) bad(std::string(key) + " must be " + std::to_string(lo) + ".." + std::to_string(hi)); return static_cast(n); } bool flag(const Json& b, const char* key) { const Json* v = b.find(key); if (!v || v->is_null()) return false; if (!v->is_bool()) bad(std::string(key) + " must be true or false"); return v->as_bool(); } std::vector strings(const Json& b, const char* key) { std::vector out; const Json* v = b.find(key); if (!v || v->is_null()) return out; if (!v->is_array()) bad(std::string(key) + " must be an array of strings"); for (const Json& s : v->as_array()) { if (!s.is_string()) bad(std::string(key) + " must be an array of strings"); out.push_back(s.as_string()); } return out; } hf::RepoType repo_type(const Json& b) { std::string k = text(b, "kind"); if (k.empty()) return hf::RepoType::kModel; hf::RepoType t; if (!hf::parse_repo_type(k, t)) bad("kind must be models, datasets or spaces"); return t; } hf::RepoRef repo_ref(const Json& b, const char* key = "id") { try { return hf::RepoRef::parse(text(b, key, true)); } catch (const llm::ApiError& e) { bad(e.message()); } } Json dry_run(const Json& request, const std::string& path) { Json o = Json::object(); o.set("dry_run", Json(true)); o.set("would_send_to", Json(path)); o.set("request", request); o.set("hint", Json("send confirm:true to run it")); return o; } Json page_json(const hf::Page& p) { Json o = Json::object(); o.set("items", p.items); o.set("pages", Json(static_cast(p.pages))); o.set("truncated", Json(p.truncated)); o.set("truncated_by", Json(p.truncated_by)); o.set("next_cursor", Json(p.next_cursor)); return o; } struct Plan { Json request; // what would be sent (shown in a dry run) std::string path; // for the dry-run message bool needs_confirm = false; // paid or changes a resource std::function call; }; // Servers (TGI, TEI) get the token only when they are Hugging Face hosted. std::string token_for(const std::string& url, const std::string& token) { clients::Url u = clients::Url::parse(url); const std::string& h = u.host(); auto ends = [&](const char* suffix) { std::string s(suffix); return h.size() >= s.size() && h.compare(h.size() - s.size(), s.size(), s) == 0; }; return (ends(".huggingface.cloud") || ends(".hf.space")) ? token : std::string(); } class HuggingfaceRoutes { public: HuggingfaceRoutes() : p_(hf::Platform::from_env()), token_(hf::Platform::token_from_env()) {} const hf::Platform& platform() const { return p_; } using Planner = std::function; fw::Response run(const fw::Request& req, const Planner& planner) const { Json body = parse_body(req); Plan plan; try { plan = planner(body); } catch (const llm::ApiError& e) { map(e); } if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.path)); if (!p_.configured()) throw fw::HttpError(503, "huggingface_not_configured", "no credential configured (set HF_TOKEN)"); try { return json_out(plan.call()); } catch (const llm::ApiError& e) { map(e); } } // ---- Hub -------------------------------------------------------------------- Plan hub_list(const Json& b) const { hf::RepoType type = repo_type(b); hf::RepoListParams lp; lp.search = text(b, "search"); lp.author = text(b, "author"); lp.sort = text(b, "sort"); lp.direction = text(b, "direction"); lp.pipeline_tag = text(b, "pipeline_tag"); lp.filter = strings(b, "filter"); lp.limit = integer(b, "limit", 20, 1, 100); const int pages = integer(b, "pages", 1, 1, 5); hf::RequestPlan::make(hf::Hub::list_call(type), hf::PathArgs(), lp.to_query()); auto hub = std::make_shared(hf::Hub::from_platform(p_)); Json req = Json::object(); req.set("kind", Json(hf::repo_type_plural(type))); return Plan{req, std::string("GET /api/") + hf::repo_type_plural(type), false, [=] { return page_json(hub->list_repos(type, lp, static_cast(lp.limit) * pages, static_cast(pages))); }}; } Plan hub_repo(const Json& b) const { const std::string action = text(b, "action", true); hf::RepoType type = repo_type(b); hf::RepoRef ref = repo_ref(b); const std::string rev = text(b, "rev").empty() ? "main" : text(b, "rev"); const std::string path = text(b, "path"); auto hub = std::make_shared(hf::Hub::from_platform(p_)); Json req = Json::object(); req.set("action", Json(action)); req.set("id", Json(ref.id())); auto make = [&](std::function fn) { return Plan{req, "GET /api/" + std::string(hf::repo_type_plural(type)) + "/" + ref.id(), false, std::move(fn)}; }; if (action == "info") { const std::string revision = text(b, "revision"); return make([=] { return hub->repo_info(type, ref, revision, strings(b, "expand")); }); } if (action == "tree") { const bool rec = flag(b, "recursive"); const int lim = integer(b, "limit", 0, 0, 1000); return make([=] { return hub->tree(type, ref, rev, path, rec, lim); }); } if (action == "refs") return make([=] { return hub->refs(type, ref, flag(b, "include_prs")); }); if (action == "commits") { const int lim = integer(b, "limit", 20, 1, 100); return make([=] { return hub->commits(type, ref, rev, lim); }); } if (action == "compare") { const std::string cmp = text(b, "compare", true); return make([=] { return hub->compare(type, ref, cmp); }); } if (action == "likers") return make([=] { return hub->likers(type, ref); }); if (action == "paths_info") { std::vector paths = strings(b, "paths"); return make([=] { return hub->paths_info(type, ref, rev, paths); }); } if (action == "safetensors") { const std::string p = text(b, "path", true); return make([=] { return hub->safetensors_metadata(ref, p); }); } if (action == "file") { const std::string p = text(b, "path", true); hf::RequestPlan::encode_file_path(p); return make([=] { hf::Download d = hub->download(type, ref, rev, p); Json o = Json::object(); o.set("bytes", Json(static_cast(d.bytes.size()))); o.set("content_type", Json(d.content_type)); o.set("commit", Json(d.commit)); o.set("etag", Json(d.etag)); const bool texty = d.content_type.find("text") != std::string::npos || d.content_type.find("json") != std::string::npos; o.set("preview", Json(texty ? d.bytes.substr(0, 20000) : std::string())); return o; }); } bad("action must be info, tree, refs, commits, compare, likers, paths_info, safetensors or file"); } Plan hub_search(const Json& b) const { const std::string action = text(b, "action", true); auto hub = std::make_shared(hf::Hub::from_platform(p_)); Json req = Json::object(); req.set("action", Json(action)); auto make = [&](std::function fn) { return Plan{req, "GET /api (" + action + ")", false, std::move(fn)}; }; const int lim = integer(b, "limit", 10, 1, 100); if (action == "quicksearch") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->quicksearch(q, lim, t); }); } if (action == "fulltext") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->full_text_search(q, lim, t); }); } if (action == "trending") { std::string t = text(b, "type"); return make([=] { return hub->trending(t, lim); }); } if (action == "tags") { hf::RepoType t = repo_type(b); return make([=] { return hub->tags_by_type(t); }); } if (action == "tasks") return make([=] { return hub->tasks(); }); if (action == "hardware") return make([=] { return hub->spaces_hardware(); }); if (action == "featured") return make([=] { return hub->spaces_featured(); }); if (action == "semantic") { std::string q = text(b, "q", true); return make([=] { return hub->spaces_semantic_search(q); }); } if (action == "papers_daily") return make([=] { return hub->daily_papers(lim, ""); }); if (action == "papers_search") { std::string q = text(b, "q", true); return make([=] { return hub->papers_search(q, lim); }); } if (action == "paper") { std::string id = text(b, "id", true); return make([=] { return hub->paper(id); }); } if (action == "partners") { std::string pr = text(b, "provider", true); return make([=] { return hub->partner_models(pr, text(b, "status")); }); } bad("action must be quicksearch, fulltext, trending, tags, tasks, hardware, featured, semantic, papers_daily, papers_search, paper or partners"); } Plan hub_whoami(const Json&) const { auto hub = std::make_shared(hf::Hub::from_platform(p_)); return Plan{Json::object(), "GET /api/whoami-v2", false, [hub] { return hub->whoami(); }}; } Plan hub_discussions(const Json& b) const { const std::string action = text(b, "action", true); hf::RepoType type = repo_type(b); hf::RepoRef ref = repo_ref(b); auto api = std::make_shared(p_.hub); Json req = Json::object(); req.set("action", Json(action)); req.set("id", Json(ref.id())); const std::string base = std::string("/api/") + hf::repo_type_plural(type) + "/" + ref.id() + "/discussions"; const int num = integer(b, "num", 0, 0, 999999999); if (action == "list") { hf::DiscussionFilter f; f.page = integer(b, "page", 0, 0, 100000); f.kind = text(b, "type"); f.status = text(b, "status"); f.author = text(b, "author"); f.search = text(b, "search"); return Plan{req, "GET " + base, false, [=] { return api->list(type, ref, f); }}; } if (action == "get") { if (!num) bad("num is required"); return Plan{req, "GET " + base + "/{num}", false, [=] { return api->get(type, ref, num); }}; } if (action == "create") { std::string title = text(b, "title", true), desc = text(b, "description"); bool pr = flag(b, "pull_request"); req.set("title", Json(title)); return Plan{req, "POST " + base, true, [=] { return api->create(type, ref, title, desc, pr); }}; } if (!num) bad("num is required"); req.set("num", Json(num)); if (action == "comment") { std::string c = text(b, "comment", true); return Plan{req, "POST " + base + "/{num}/comment", true, [=] { return api->comment(type, ref, num, c); }}; } if (action == "status") { std::string s = text(b, "status", true), c = text(b, "comment"); return Plan{req, "POST " + base + "/{num}/status", true, [=] { return api->change_status(type, ref, num, s, c); }}; } if (action == "rename") { std::string t = text(b, "title", true); return Plan{req, "POST " + base + "/{num}/title", true, [=] { return api->rename(type, ref, num, t); }}; } if (action == "remove") return Plan{req, "DELETE " + base + "/{num}", true, [=] { return api->remove_discussion(type, ref, num); }}; bad("action must be list, get, create, comment, status, rename or remove"); } Plan hub_collections(const Json& b) const { const std::string action = text(b, "action", true); auto api = std::make_shared(p_.hub); Json req = Json::object(); req.set("action", Json(action)); if (action == "list") { hf::CollectionListParams lp; lp.owner = text(b, "owner"); lp.q = text(b, "q"); lp.sort = text(b, "sort"); lp.limit = integer(b, "limit", 20, 1, 100); lp.to_query(); return Plan{req, "GET /api/collections", false, [=] { return page_json(api->list(lp, 100, 2)); }}; } const std::string owner = text(b, "owner", true), slug = text(b, "slug"); req.set("owner", Json(owner)); if (action == "create") { std::string title = text(b, "title", true), desc = text(b, "description"); bool priv = flag(b, "private"); req.set("title", Json(title)); return Plan{req, "POST /api/collections", true, [=] { return api->create(owner, title, desc, priv); }}; } if (slug.empty()) bad("slug is required"); req.set("slug", Json(slug)); if (action == "get") return Plan{req, "GET /api/collections/{owner}/{slug}", false, [=] { return api->get(owner, slug); }}; if (action == "update") { const Json* f = b.find("fields"); if (!f || !f->is_object()) bad("fields must be an object"); Json fields = *f; return Plan{req, "PATCH /api/collections/{owner}/{slug}", true, [=] { return api->update(owner, slug, fields); }}; } if (action == "add_item") { hf::CollectionItem item; item.type = text(b, "item_type", true); item.id = text(b, "item_id", true); item.to_json(); std::string note = text(b, "note"); return Plan{req, "POST /api/collections/{owner}/{slug}/items", true, [=] { return api->add_item(owner, slug, item, note); }}; } if (action == "remove") return Plan{req, "DELETE /api/collections/{owner}/{slug}", true, [=] { return api->remove_collection(owner, slug); }}; bad("action must be list, get, create, update, add_item or remove"); } Plan hub_webhooks(const Json& b) const { const std::string action = text(b, "action", true); auto api = std::make_shared(p_.hub); Json req = Json::object(); req.set("action", Json(action)); auto spec_of = [&]() { hf::WebhookSpec s; s.url = text(b, "url", true); s.domains = strings(b, "domains"); s.secret = text(b, "secret"); const Json* w = b.find("watched"); if (!w || !w->is_array()) bad("watched must be an array of {type,name}"); for (const Json& x : w->as_array()) { if (!x.is_object()) bad("watched entries must be objects"); s.watched.push_back({text(x, "type", true), text(x, "name", true)}); } s.to_json(); return s; }; if (action == "list") return Plan{req, "GET /api/settings/webhooks", false, [=] { return api->list(); }}; if (action == "create") { hf::WebhookSpec s = spec_of(); req.set("url", Json(s.url)); return Plan{req, "POST /api/settings/webhooks", true, [=] { return api->create(s); }}; } const std::string id = text(b, "id", true); req.set("id", Json(id)); if (action == "get") return Plan{req, "GET /api/settings/webhooks/{id}", false, [=] { return api->get(id); }}; if (action == "update") { hf::WebhookSpec s = spec_of(); return Plan{req, "POST /api/settings/webhooks/{id}", true, [=] { return api->update(id, s); }}; } if (action == "enable") return Plan{req, "POST /api/settings/webhooks/{id}/enable", true, [=] { return api->enable(id); }}; if (action == "disable") return Plan{req, "POST /api/settings/webhooks/{id}/disable", true, [=] { return api->disable(id); }}; if (action == "replay") { std::string log = text(b, "log_id", true); return Plan{req, "POST /api/settings/webhooks/{id}/replay/{log_id}", true, [=] { return api->replay(id, log); }}; } if (action == "remove") return Plan{req, "DELETE /api/settings/webhooks/{id}", true, [=] { return api->remove_webhook(id); }}; bad("action must be list, get, create, update, enable, disable, replay or remove"); } // ---- Inference --------------------------------------------------------------- llm::ChatRequest chat_request(const Json& b) const { llm::ChatRequest r; r.model = text(b, "model", true); hf::RouterModel::parse(r.model); r.system = text(b, "system"); r.messages.push_back(llm::Message::user(text(b, "prompt", true))); if (b.find("max_tokens")) r.max_tokens = integer(b, "max_tokens", 256, 1, hf::HfCaps::kMaxTokensCeiling); if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number(); r.json_mode = flag(b, "json"); r.reasoning_effort = text(b, "effort"); return r; } Plan chat(const Json& b) const { llm::ChatRequest r = chat_request(b); auto model = hf::HuggingfaceChatModel::over_router(p_, r.model); Json body = model->to_body(r, false); return Plan{body, "/v1/chat/completions", true, [model, r] { llm::ChatResponse res = model->generate(r); Json o = Json::object(); o.set("id", Json(res.id)); o.set("model", Json(res.model)); o.set("text", Json(res.text())); o.set("finish_reason", Json(res.finish_reason)); Json u = Json::object(); u.set("input_tokens", Json(res.usage.input_tokens)); u.set("output_tokens", Json(res.usage.output_tokens)); u.set("total_tokens", Json(res.usage.total_tokens)); o.set("usage", u); return o; }}; } Plan chat_stream(const Json& b) const { llm::ChatRequest r = chat_request(b); auto model = hf::HuggingfaceChatModel::over_router(p_, r.model); Json body = model->to_body(r, true); return Plan{body, "/v1/chat/completions (stream)", true, [model, r] { std::string content; std::size_t chunks = 0; std::string finish; model->stream(r, [&](const llm::ChatChunk& c) { content += c.text_delta; if (!c.finish_reason.empty()) finish = c.finish_reason; return ++chunks < hf::HfCaps::kMaxStreamEvents; }); Json o = Json::object(); o.set("text", Json(content)); o.set("finish_reason", Json(finish)); o.set("chunks", Json(static_cast(chunks))); return o; }}; } Plan task(const Json& b) const { const std::string task = text(b, "task", true), model = text(b, "model", true); const hf::TaskInfo* info = hf::TaskCatalog::find(task); if (!info) bad("unknown task"); hf::RequestPlan::check_model(model); Json params = b.find("parameters") ? *b.find("parameters") : Json(); auto api = std::make_shared(hf::InferenceTasks::from_platform(p_)); Json req = Json::object(); req.set("task", Json(task)); req.set("model", Json(model)); auto result_json = [](const hf::InferenceResult& r) { Json o = Json::object(); o.set("content_type", Json(r.content_type)); if (r.is_json) { o.set("result", r.json); } else { o.set("bytes", Json(static_cast(r.bytes.size()))); o.set("data_base64", Json(security::crypto::Base64::encode(r.bytes.substr(0, 1024 * 1024)))); } return o; }; if (info->binary_input) { std::string bytes; if (!security::crypto::Base64::decode(text(b, "data_base64", true), bytes)) bad("data_base64 is not valid base64"); std::string mime = text(b, "mime", true); req.set("bytes", Json(static_cast(bytes.size()))); return Plan{req, "POST /hf-inference/models/{model}", true, [=] { return result_json(api->run_binary(task, model, bytes, mime, params)); }}; } const Json* in = b.find("inputs"); if (!in || in->is_null()) bad("inputs is required"); Json inputs = *in; req.set("inputs", inputs); return Plan{req, "POST /hf-inference/models/{model}", true, [=] { return result_json(api->run_json(task, model, inputs, params)); }}; } Plan endpoints(const Json& b) const { const std::string action = text(b, "action", true); const std::string ns = text(b, "namespace"); const std::string name = text(b, "name"); auto api = std::make_shared(hf::Endpoints::from_platform(p_)); Json req = Json::object(); req.set("action", Json(action)); if (!ns.empty()) req.set("namespace", Json(ns)); if (!name.empty()) req.set("name", Json(name)); auto need_ns = [&] { if (ns.empty()) bad("namespace is required"); }; auto need_name = [&] { need_ns(); if (name.empty()) bad("name is required"); hf::Endpoints::validate_name(name); }; if (action == "vendors") return Plan{req, "GET /v2/provider", false, [=] { return api->vendors(); }}; if (action == "list") { need_ns(); return Plan{req, "GET /v2/endpoint/{ns}", false, [=] { return api->list(ns); }}; } if (action == "search") { need_ns(); std::string q = text(b, "q"); return Plan{req, "GET /v2/search/{ns}", false, [=] { return api->search(ns, q); }}; } if (action == "states") { need_ns(); return Plan{req, "GET /v2/search/{ns}/states", false, [=] { return api->state_counts(ns); }}; } if (action == "quotas") { need_ns(); return Plan{req, "GET /v2/provider/quotas/{ns}", false, [=] { return api->quotas(ns); }}; } if (action == "audit") { need_ns(); return Plan{req, "GET /v2/audit-logs/{ns}", false, [=] { return api->audit_logs(ns); }}; } if (action == "get") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}", false, [=] { return api->get(ns, name); }}; } if (action == "replicas") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}/replica", false, [=] { return api->replicas(ns, name); }}; } if (action == "logs") { need_name(); int tail = integer(b, "tail", 100, 1, 1000); return Plan{req, "GET /v2/endpoint/{ns}/{name}/logs", false, [=] { Json o = Json::object(); o.set("logs", Json(api->logs(ns, name, tail))); return o; }}; } if (action == "metrics") { need_name(); hf::MetricKind k; if (!hf::parse_metric_kind(text(b, "kind").empty() ? "all" : text(b, "kind"), k)) bad("unknown metric kind"); std::string start = text(b, "start", true), stop = text(b, "stop", true); return Plan{req, "POST /v2/endpoint/{ns}/{name}/metrics", false, [=] { return api->metrics(k, ns, name, start, stop); }}; } if (action == "create") { need_ns(); const Json* s = b.find("spec"); if (!s || !s->is_object()) bad("spec must be an object"); Json spec = *s; hf::Endpoints::validate_spec(spec, true); req.set("spec", spec); return Plan{req, "POST /v2/endpoint/{ns} (costs money while it runs)", true, [=] { return api->create(ns, spec); }}; } if (action == "update") { need_name(); const Json* s = b.find("spec"); if (!s || !s->is_object()) bad("spec must be an object"); Json spec = *s; hf::Endpoints::validate_spec(spec, false); return Plan{req, "PUT /v2/endpoint/{ns}/{name}", true, [=] { return api->update(ns, name, spec); }}; } if (action == "pause") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/pause", true, [=] { return api->pause(ns, name); }}; } if (action == "resume") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/resume", true, [=] { return api->resume(ns, name); }}; } if (action == "scale_to_zero") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/scale-to-zero", true, [=] { return api->scale_to_zero(ns, name); }}; } if (action == "remove") { need_name(); return Plan{req, "DELETE /v2/endpoint/{ns}/{name}", true, [=] { return api->remove_endpoint(ns, name); }}; } bad("action must be vendors, list, search, states, quotas, audit, get, replicas, logs, metrics, create, update, pause, resume, scale_to_zero or remove"); } hf::Access server_access(const Json& b) const { const std::string url = text(b, "url", true); try { return hf::Platform::server(url, token_for(hf::Platform::check_server_url(url), token_)); } catch (const clients::UrlError& e) { bad(e.what()); } } Plan tgi(const Json& b) const { const std::string action = text(b, "action", true); auto api = std::make_shared(server_access(b)); Json req = Json::object(); req.set("action", Json(action)); if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }}; if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }}; if (action == "models") return Plan{req, "GET /v1/models", false, [=] { return api->models(); }}; if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }}; Json params = b.find("parameters") ? *b.find("parameters") : Json(); if (action == "generate") { std::string in = text(b, "inputs", true); return Plan{req, "POST /generate", false, [=] { return api->generate(in, params); }}; } if (action == "tokenize") { std::string in = text(b, "inputs", true); return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(in, params); }}; } if (action == "chat") { llm::ChatRequest r = chat_request(b); hf::ChatWire::to_body(r, "tgi", false); auto model = std::make_shared(api, r.model); return Plan{req, "POST /v1/chat/completions", false, [=] { Json o = Json::object(); o.set("text", Json(model->generate(r).text())); return o; }}; } bad("action must be health, info, models, metrics, generate, tokenize or chat"); } Plan tei(const Json& b) const { const std::string action = text(b, "action", true); auto api = std::make_shared(server_access(b)); Json req = Json::object(); req.set("action", Json(action)); if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }}; if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }}; if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }}; if (action == "rerank") { std::string q = text(b, "query", true); std::vector texts = strings(b, "texts"); return Plan{req, "POST /rerank", false, [=] { return api->rerank(q, texts, flag(b, "return_text")); }}; } const Json* in = b.find("inputs"); if (!in || in->is_null()) bad("inputs is required"); Json inputs = *in; hf::TeiServer::check_inputs(inputs); if (action == "embed") return Plan{req, "POST /embed", false, [=] { return api->embed(inputs); }}; if (action == "predict") return Plan{req, "POST /predict", false, [=] { return api->predict(inputs); }}; if (action == "tokenize") return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(inputs); }}; bad("action must be health, info, metrics, embed, predict, rerank or tokenize"); } private: [[noreturn]] static void map(const llm::ApiError& e) { if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") throw fw::HttpError(503, "huggingface_not_configured", e.message()); if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) throw fw::HttpError(422, "validation_error", e.message()); if (e.type() == "LOCAL_BUDGET") throw fw::HttpError(429, "huggingface_local_budget", e.message()); throw fw::HttpError(502, "huggingface_error", e.type() + ": " + e.message()); } hf::Platform p_; std::string token_; }; } // namespace void register_ai_huggingface_demo(fw::Router& router, Services& services) { auto routes = std::make_shared(); fw::Environment* env = services.templates.get(); services.templates->add_template("ai_huggingface.html", kPage); services.add_feature("ai-huggingface", "/ai/huggingface"); router.get("/ai/huggingface", "ai-huggingface-page", [env](fw::Request req) { (void)req; fw::TemplateCtx ctx; ctx["title"] = std::string("Hugging Face"); return fw::Response(200, env->render("ai_huggingface.html", ctx), "text/html; charset=utf-8"); }); router.get("/ai/huggingface/status", "ai-huggingface-status", [routes](fw::Request req) { (void)req; Json o = Json::object(); o.set("configured", Json(routes->platform().configured())); o.set("credential_env", Json(hf::Platform::kKeyEnv)); o.set("hub", Json(hf::Platform::kHubHost)); o.set("router", Json(hf::Platform::kRouterHost)); o.set("endpoints", Json(hf::Platform::kEndpointsHost)); o.set("calls", Json(static_cast(hf::Calls::count()))); o.set("tasks", Json(static_cast(hf::TaskCatalog::count()))); o.set("budget_calls", Json(hf::HfCaps::kBudgetMaxCalls)); o.set("budget_window_seconds", Json(hf::HfCaps::kBudgetWindowSeconds)); o.set("max_pages", Json(static_cast(hf::HfCaps::kMaxPages))); return json_out(o); }); router.get("/ai/huggingface/calls", "ai-huggingface-calls", [](fw::Request req) { (void)req; Json arr = Json::array(); for (std::size_t i = 0; i < hf::Calls::count(); ++i) { const hf::CallSpec& s = hf::Calls::at(i); Json row = Json::object(); row.set("surface", Json(hf::surface_name(s.surface))); row.set("http", Json(s.http_method)); row.set("path", Json(s.path)); row.set("spec", Json(s.spec)); row.set("auth", Json(s.auth)); row.set("flags", Json(s.flags)); arr.push(row); } Json o = Json::object(); o.set("calls", arr); return json_out(o); }); router.get("/ai/huggingface/tasks", "ai-huggingface-tasks", [](fw::Request req) { (void)req; Json arr = Json::array(); for (std::size_t i = 0; i < hf::TaskCatalog::count(); ++i) { const hf::TaskInfo& t = hf::TaskCatalog::at(i); Json row = Json::object(); row.set("task", Json(t.name)); row.set("binary_input", Json(t.binary_input)); row.set("pipeline_path", Json(t.pipeline_path)); row.set("note", Json(t.note)); arr.push(row); } Json o = Json::object(); o.set("tasks", arr); return json_out(o); }); using Member = HuggingfaceRoutes::Planner; HuggingfaceRoutes* r = routes.get(); auto add = [&](const char* path, const char* name, Plan (HuggingfaceRoutes::*m)(const Json&) const) { Member planner = [r, m](const Json& b) { return (r->*m)(b); }; router.post(path, name, [routes, planner](fw::Request req) { return routes->run(req, planner); }); }; add("/ai/huggingface/hub/list", "ai-huggingface-hub-list", &HuggingfaceRoutes::hub_list); add("/ai/huggingface/hub/repo", "ai-huggingface-hub-repo", &HuggingfaceRoutes::hub_repo); add("/ai/huggingface/hub/search", "ai-huggingface-hub-search", &HuggingfaceRoutes::hub_search); add("/ai/huggingface/hub/whoami", "ai-huggingface-hub-whoami", &HuggingfaceRoutes::hub_whoami); add("/ai/huggingface/hub/discussions", "ai-huggingface-hub-discussions", &HuggingfaceRoutes::hub_discussions); add("/ai/huggingface/hub/collections", "ai-huggingface-hub-collections", &HuggingfaceRoutes::hub_collections); add("/ai/huggingface/hub/webhooks", "ai-huggingface-hub-webhooks", &HuggingfaceRoutes::hub_webhooks); add("/ai/huggingface/chat", "ai-huggingface-chat", &HuggingfaceRoutes::chat); add("/ai/huggingface/chat/stream", "ai-huggingface-chat-stream", &HuggingfaceRoutes::chat_stream); add("/ai/huggingface/task", "ai-huggingface-task", &HuggingfaceRoutes::task); add("/ai/huggingface/endpoints", "ai-huggingface-endpoints", &HuggingfaceRoutes::endpoints); add("/ai/huggingface/tgi", "ai-huggingface-tgi", &HuggingfaceRoutes::tgi); add("/ai/huggingface/tei", "ai-huggingface-tei", &HuggingfaceRoutes::tei); } } // namespace molo::example::app