diff --git a/example/app/ai/HuggingfaceDemo.cpp b/example/app/ai/HuggingfaceDemo.cpp index 2110b89..244abe4 100644 --- a/example/app/ai/HuggingfaceDemo.cpp +++ b/example/app/ai/HuggingfaceDemo.cpp @@ -1,13 +1,749 @@ #include "../AiDemos.hpp" +#include + #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "clients/url.h" +#include "security/crypto/Base64.h" +#include "vendors/huggingface/caps.h" +#include "vendors/huggingface/calls.h" +#include "vendors/huggingface/chat_model.h" +#include "vendors/huggingface/community.h" +#include "vendors/huggingface/endpoints.h" +#include "vendors/huggingface/hub.h" +#include "vendors/huggingface/platform.h" +#include "vendors/huggingface/providers.h" +#include "vendors/huggingface/servers.h" namespace molo::example::app { -// huggingface. Stub: the package agent fills this in. +// Hugging Face over the real src/vendors/huggingface package. GET routes need +// no credential. Every route that calls Hugging Face is a POST: 422 on bad +// input, 503 huggingface_not_configured without HF_TOKEN. Routes that cost +// money (chat, tasks, creating endpoints) or change resources (discussions, +// collections, webhooks, endpoint lifecycle) are dry runs that show the exact +// request unless the body has "confirm":true. + +namespace { + +namespace hf = huggingface; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // inference inputs travel as base64 + +const char kPage[] = R"HUGGINGFACE({% extends "base.html" %} +{% block content %} +

Hugging Face

+

Hub (models, datasets, spaces, repos, search, papers, discussions, collections, +webhooks), Inference Providers router chat, hf-inference tasks, Inference Endpoints management, +TGI and TEI servers. Paid and changing calls are dry runs until the body says +"confirm":true.

+

Checking credential...

+
+ + +
+ + +
+ +

+
+ + +{% endblock %} +)HUGGINGFACE"; + +[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); } + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string text(const Json& b, const char* key, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + if (required && v->as_string().empty()) bad(std::string(key) + " is required"); + return v->as_string(); +} + +int integer(const Json& b, const char* key, int fallback, int lo, int hi) { + const Json* v = b.find(key); + if (!v || v->is_null()) return fallback; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + long n = v->as_int(); + if (n < lo || n > hi) bad(std::string(key) + " must be " + std::to_string(lo) + ".." + std::to_string(hi)); + return static_cast(n); +} + +bool flag(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +std::vector strings(const Json& b, const char* key) { + std::vector out; + const Json* v = b.find(key); + if (!v || v->is_null()) return out; + if (!v->is_array()) bad(std::string(key) + " must be an array of strings"); + for (const Json& s : v->as_array()) { + if (!s.is_string()) bad(std::string(key) + " must be an array of strings"); + out.push_back(s.as_string()); + } + return out; +} + +hf::RepoType repo_type(const Json& b) { + std::string k = text(b, "kind"); + if (k.empty()) return hf::RepoType::kModel; + hf::RepoType t; + if (!hf::parse_repo_type(k, t)) bad("kind must be models, datasets or spaces"); + return t; +} + +hf::RepoRef repo_ref(const Json& b, const char* key = "id") { + try { + return hf::RepoRef::parse(text(b, key, true)); + } catch (const llm::ApiError& e) { + bad(e.message()); + } +} + +Json dry_run(const Json& request, const std::string& path) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set("would_send_to", Json(path)); + o.set("request", request); + o.set("hint", Json("send confirm:true to run it")); + return o; +} + +Json page_json(const hf::Page& p) { + Json o = Json::object(); + o.set("items", p.items); + o.set("pages", Json(static_cast(p.pages))); + o.set("truncated", Json(p.truncated)); + o.set("truncated_by", Json(p.truncated_by)); + o.set("next_cursor", Json(p.next_cursor)); + return o; +} + +struct Plan { + Json request; // what would be sent (shown in a dry run) + std::string path; // for the dry-run message + bool needs_confirm = false; // paid or changes a resource + std::function call; +}; + +// Servers (TGI, TEI) get the token only when they are Hugging Face hosted. +std::string token_for(const std::string& url, const std::string& token) { + clients::Url u = clients::Url::parse(url); + const std::string& h = u.host(); + auto ends = [&](const char* suffix) { + std::string s(suffix); + return h.size() >= s.size() && h.compare(h.size() - s.size(), s.size(), s) == 0; + }; + return (ends(".huggingface.cloud") || ends(".hf.space")) ? token : std::string(); +} + +class HuggingfaceRoutes { + public: + HuggingfaceRoutes() : p_(hf::Platform::from_env()), token_(hf::Platform::token_from_env()) {} + const hf::Platform& platform() const { return p_; } + + using Planner = std::function; + + fw::Response run(const fw::Request& req, const Planner& planner) const { + Json body = parse_body(req); + Plan plan; + try { + plan = planner(body); + } catch (const llm::ApiError& e) { + map(e); + } + if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.path)); + if (!p_.configured()) throw fw::HttpError(503, "huggingface_not_configured", "no credential configured (set HF_TOKEN)"); + try { + return json_out(plan.call()); + } catch (const llm::ApiError& e) { + map(e); + } + } + + // ---- Hub -------------------------------------------------------------------- + Plan hub_list(const Json& b) const { + hf::RepoType type = repo_type(b); + hf::RepoListParams lp; + lp.search = text(b, "search"); + lp.author = text(b, "author"); + lp.sort = text(b, "sort"); + lp.direction = text(b, "direction"); + lp.pipeline_tag = text(b, "pipeline_tag"); + lp.filter = strings(b, "filter"); + lp.limit = integer(b, "limit", 20, 1, 100); + const int pages = integer(b, "pages", 1, 1, 5); + hf::RequestPlan::make(hf::Hub::list_call(type), hf::PathArgs(), lp.to_query()); + auto hub = std::make_shared(hf::Hub::from_platform(p_)); + Json req = Json::object(); + req.set("kind", Json(hf::repo_type_plural(type))); + return Plan{req, std::string("GET /api/") + hf::repo_type_plural(type), false, [=] { + return page_json(hub->list_repos(type, lp, static_cast(lp.limit) * pages, + static_cast(pages))); + }}; + } + + Plan hub_repo(const Json& b) const { + const std::string action = text(b, "action", true); + hf::RepoType type = repo_type(b); + hf::RepoRef ref = repo_ref(b); + const std::string rev = text(b, "rev").empty() ? "main" : text(b, "rev"); + const std::string path = text(b, "path"); + auto hub = std::make_shared(hf::Hub::from_platform(p_)); + Json req = Json::object(); + req.set("action", Json(action)); + req.set("id", Json(ref.id())); + auto make = [&](std::function fn) { return Plan{req, "GET /api/" + std::string(hf::repo_type_plural(type)) + "/" + ref.id(), false, std::move(fn)}; }; + if (action == "info") { + const std::string revision = text(b, "revision"); + return make([=] { return hub->repo_info(type, ref, revision, strings(b, "expand")); }); + } + if (action == "tree") { + const bool rec = flag(b, "recursive"); + const int lim = integer(b, "limit", 0, 0, 1000); + return make([=] { return hub->tree(type, ref, rev, path, rec, lim); }); + } + if (action == "refs") return make([=] { return hub->refs(type, ref, flag(b, "include_prs")); }); + if (action == "commits") { + const int lim = integer(b, "limit", 20, 1, 100); + return make([=] { return hub->commits(type, ref, rev, lim); }); + } + if (action == "compare") { + const std::string cmp = text(b, "compare", true); + return make([=] { return hub->compare(type, ref, cmp); }); + } + if (action == "likers") return make([=] { return hub->likers(type, ref); }); + if (action == "paths_info") { + std::vector paths = strings(b, "paths"); + return make([=] { return hub->paths_info(type, ref, rev, paths); }); + } + if (action == "safetensors") { + const std::string p = text(b, "path", true); + return make([=] { return hub->safetensors_metadata(ref, p); }); + } + if (action == "file") { + const std::string p = text(b, "path", true); + hf::RequestPlan::encode_file_path(p); + return make([=] { + hf::Download d = hub->download(type, ref, rev, p); + Json o = Json::object(); + o.set("bytes", Json(static_cast(d.bytes.size()))); + o.set("content_type", Json(d.content_type)); + o.set("commit", Json(d.commit)); + o.set("etag", Json(d.etag)); + const bool texty = d.content_type.find("text") != std::string::npos || d.content_type.find("json") != std::string::npos; + o.set("preview", Json(texty ? d.bytes.substr(0, 20000) : std::string())); + return o; + }); + } + bad("action must be info, tree, refs, commits, compare, likers, paths_info, safetensors or file"); + } + + Plan hub_search(const Json& b) const { + const std::string action = text(b, "action", true); + auto hub = std::make_shared(hf::Hub::from_platform(p_)); + Json req = Json::object(); + req.set("action", Json(action)); + auto make = [&](std::function fn) { return Plan{req, "GET /api (" + action + ")", false, std::move(fn)}; }; + const int lim = integer(b, "limit", 10, 1, 100); + if (action == "quicksearch") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->quicksearch(q, lim, t); }); } + if (action == "fulltext") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->full_text_search(q, lim, t); }); } + if (action == "trending") { std::string t = text(b, "type"); return make([=] { return hub->trending(t, lim); }); } + if (action == "tags") { hf::RepoType t = repo_type(b); return make([=] { return hub->tags_by_type(t); }); } + if (action == "tasks") return make([=] { return hub->tasks(); }); + if (action == "hardware") return make([=] { return hub->spaces_hardware(); }); + if (action == "featured") return make([=] { return hub->spaces_featured(); }); + if (action == "semantic") { std::string q = text(b, "q", true); return make([=] { return hub->spaces_semantic_search(q); }); } + if (action == "papers_daily") return make([=] { return hub->daily_papers(lim, ""); }); + if (action == "papers_search") { std::string q = text(b, "q", true); return make([=] { return hub->papers_search(q, lim); }); } + if (action == "paper") { std::string id = text(b, "id", true); return make([=] { return hub->paper(id); }); } + if (action == "partners") { std::string pr = text(b, "provider", true); return make([=] { return hub->partner_models(pr, text(b, "status")); }); } + bad("action must be quicksearch, fulltext, trending, tags, tasks, hardware, featured, semantic, papers_daily, papers_search, paper or partners"); + } + + Plan hub_whoami(const Json&) const { + auto hub = std::make_shared(hf::Hub::from_platform(p_)); + return Plan{Json::object(), "GET /api/whoami-v2", false, [hub] { return hub->whoami(); }}; + } + + Plan hub_discussions(const Json& b) const { + const std::string action = text(b, "action", true); + hf::RepoType type = repo_type(b); + hf::RepoRef ref = repo_ref(b); + auto api = std::make_shared(p_.hub); + Json req = Json::object(); + req.set("action", Json(action)); + req.set("id", Json(ref.id())); + const std::string base = std::string("/api/") + hf::repo_type_plural(type) + "/" + ref.id() + "/discussions"; + const int num = integer(b, "num", 0, 0, 999999999); + if (action == "list") { + hf::DiscussionFilter f; + f.page = integer(b, "page", 0, 0, 100000); + f.kind = text(b, "type"); + f.status = text(b, "status"); + f.author = text(b, "author"); + f.search = text(b, "search"); + return Plan{req, "GET " + base, false, [=] { return api->list(type, ref, f); }}; + } + if (action == "get") { + if (!num) bad("num is required"); + return Plan{req, "GET " + base + "/{num}", false, [=] { return api->get(type, ref, num); }}; + } + if (action == "create") { + std::string title = text(b, "title", true), desc = text(b, "description"); + bool pr = flag(b, "pull_request"); + req.set("title", Json(title)); + return Plan{req, "POST " + base, true, [=] { return api->create(type, ref, title, desc, pr); }}; + } + if (!num) bad("num is required"); + req.set("num", Json(num)); + if (action == "comment") { + std::string c = text(b, "comment", true); + return Plan{req, "POST " + base + "/{num}/comment", true, [=] { return api->comment(type, ref, num, c); }}; + } + if (action == "status") { + std::string s = text(b, "status", true), c = text(b, "comment"); + return Plan{req, "POST " + base + "/{num}/status", true, [=] { return api->change_status(type, ref, num, s, c); }}; + } + if (action == "rename") { + std::string t = text(b, "title", true); + return Plan{req, "POST " + base + "/{num}/title", true, [=] { return api->rename(type, ref, num, t); }}; + } + if (action == "remove") + return Plan{req, "DELETE " + base + "/{num}", true, [=] { return api->remove_discussion(type, ref, num); }}; + bad("action must be list, get, create, comment, status, rename or remove"); + } + + Plan hub_collections(const Json& b) const { + const std::string action = text(b, "action", true); + auto api = std::make_shared(p_.hub); + Json req = Json::object(); + req.set("action", Json(action)); + if (action == "list") { + hf::CollectionListParams lp; + lp.owner = text(b, "owner"); + lp.q = text(b, "q"); + lp.sort = text(b, "sort"); + lp.limit = integer(b, "limit", 20, 1, 100); + lp.to_query(); + return Plan{req, "GET /api/collections", false, [=] { return page_json(api->list(lp, 100, 2)); }}; + } + const std::string owner = text(b, "owner", true), slug = text(b, "slug"); + req.set("owner", Json(owner)); + if (action == "create") { + std::string title = text(b, "title", true), desc = text(b, "description"); + bool priv = flag(b, "private"); + req.set("title", Json(title)); + return Plan{req, "POST /api/collections", true, [=] { return api->create(owner, title, desc, priv); }}; + } + if (slug.empty()) bad("slug is required"); + req.set("slug", Json(slug)); + if (action == "get") return Plan{req, "GET /api/collections/{owner}/{slug}", false, [=] { return api->get(owner, slug); }}; + if (action == "update") { + const Json* f = b.find("fields"); + if (!f || !f->is_object()) bad("fields must be an object"); + Json fields = *f; + return Plan{req, "PATCH /api/collections/{owner}/{slug}", true, [=] { return api->update(owner, slug, fields); }}; + } + if (action == "add_item") { + hf::CollectionItem item; + item.type = text(b, "item_type", true); + item.id = text(b, "item_id", true); + item.to_json(); + std::string note = text(b, "note"); + return Plan{req, "POST /api/collections/{owner}/{slug}/items", true, [=] { return api->add_item(owner, slug, item, note); }}; + } + if (action == "remove") + return Plan{req, "DELETE /api/collections/{owner}/{slug}", true, [=] { return api->remove_collection(owner, slug); }}; + bad("action must be list, get, create, update, add_item or remove"); + } + + Plan hub_webhooks(const Json& b) const { + const std::string action = text(b, "action", true); + auto api = std::make_shared(p_.hub); + Json req = Json::object(); + req.set("action", Json(action)); + auto spec_of = [&]() { + hf::WebhookSpec s; + s.url = text(b, "url", true); + s.domains = strings(b, "domains"); + s.secret = text(b, "secret"); + const Json* w = b.find("watched"); + if (!w || !w->is_array()) bad("watched must be an array of {type,name}"); + for (const Json& x : w->as_array()) { + if (!x.is_object()) bad("watched entries must be objects"); + s.watched.push_back({text(x, "type", true), text(x, "name", true)}); + } + s.to_json(); + return s; + }; + if (action == "list") return Plan{req, "GET /api/settings/webhooks", false, [=] { return api->list(); }}; + if (action == "create") { + hf::WebhookSpec s = spec_of(); + req.set("url", Json(s.url)); + return Plan{req, "POST /api/settings/webhooks", true, [=] { return api->create(s); }}; + } + const std::string id = text(b, "id", true); + req.set("id", Json(id)); + if (action == "get") return Plan{req, "GET /api/settings/webhooks/{id}", false, [=] { return api->get(id); }}; + if (action == "update") { + hf::WebhookSpec s = spec_of(); + return Plan{req, "POST /api/settings/webhooks/{id}", true, [=] { return api->update(id, s); }}; + } + if (action == "enable") return Plan{req, "POST /api/settings/webhooks/{id}/enable", true, [=] { return api->enable(id); }}; + if (action == "disable") return Plan{req, "POST /api/settings/webhooks/{id}/disable", true, [=] { return api->disable(id); }}; + if (action == "replay") { + std::string log = text(b, "log_id", true); + return Plan{req, "POST /api/settings/webhooks/{id}/replay/{log_id}", true, [=] { return api->replay(id, log); }}; + } + if (action == "remove") return Plan{req, "DELETE /api/settings/webhooks/{id}", true, [=] { return api->remove_webhook(id); }}; + bad("action must be list, get, create, update, enable, disable, replay or remove"); + } + + // ---- Inference --------------------------------------------------------------- + llm::ChatRequest chat_request(const Json& b) const { + llm::ChatRequest r; + r.model = text(b, "model", true); + hf::RouterModel::parse(r.model); + r.system = text(b, "system"); + r.messages.push_back(llm::Message::user(text(b, "prompt", true))); + if (b.find("max_tokens")) r.max_tokens = integer(b, "max_tokens", 256, 1, hf::HfCaps::kMaxTokensCeiling); + if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number(); + r.json_mode = flag(b, "json"); + r.reasoning_effort = text(b, "effort"); + return r; + } + + Plan chat(const Json& b) const { + llm::ChatRequest r = chat_request(b); + auto model = hf::HuggingfaceChatModel::over_router(p_, r.model); + Json body = model->to_body(r, false); + return Plan{body, "/v1/chat/completions", true, [model, r] { + llm::ChatResponse res = model->generate(r); + Json o = Json::object(); + o.set("id", Json(res.id)); + o.set("model", Json(res.model)); + o.set("text", Json(res.text())); + o.set("finish_reason", Json(res.finish_reason)); + Json u = Json::object(); + u.set("input_tokens", Json(res.usage.input_tokens)); + u.set("output_tokens", Json(res.usage.output_tokens)); + u.set("total_tokens", Json(res.usage.total_tokens)); + o.set("usage", u); + return o; + }}; + } + + Plan chat_stream(const Json& b) const { + llm::ChatRequest r = chat_request(b); + auto model = hf::HuggingfaceChatModel::over_router(p_, r.model); + Json body = model->to_body(r, true); + return Plan{body, "/v1/chat/completions (stream)", true, [model, r] { + std::string content; + std::size_t chunks = 0; + std::string finish; + model->stream(r, [&](const llm::ChatChunk& c) { + content += c.text_delta; + if (!c.finish_reason.empty()) finish = c.finish_reason; + return ++chunks < hf::HfCaps::kMaxStreamEvents; + }); + Json o = Json::object(); + o.set("text", Json(content)); + o.set("finish_reason", Json(finish)); + o.set("chunks", Json(static_cast(chunks))); + return o; + }}; + } + + Plan task(const Json& b) const { + const std::string task = text(b, "task", true), model = text(b, "model", true); + const hf::TaskInfo* info = hf::TaskCatalog::find(task); + if (!info) bad("unknown task"); + hf::RequestPlan::check_model(model); + Json params = b.find("parameters") ? *b.find("parameters") : Json(); + auto api = std::make_shared(hf::InferenceTasks::from_platform(p_)); + Json req = Json::object(); + req.set("task", Json(task)); + req.set("model", Json(model)); + auto result_json = [](const hf::InferenceResult& r) { + Json o = Json::object(); + o.set("content_type", Json(r.content_type)); + if (r.is_json) { + o.set("result", r.json); + } else { + o.set("bytes", Json(static_cast(r.bytes.size()))); + o.set("data_base64", Json(security::crypto::Base64::encode(r.bytes.substr(0, 1024 * 1024)))); + } + return o; + }; + if (info->binary_input) { + std::string bytes; + if (!security::crypto::Base64::decode(text(b, "data_base64", true), bytes)) bad("data_base64 is not valid base64"); + std::string mime = text(b, "mime", true); + req.set("bytes", Json(static_cast(bytes.size()))); + return Plan{req, "POST /hf-inference/models/{model}", true, + [=] { return result_json(api->run_binary(task, model, bytes, mime, params)); }}; + } + const Json* in = b.find("inputs"); + if (!in || in->is_null()) bad("inputs is required"); + Json inputs = *in; + req.set("inputs", inputs); + return Plan{req, "POST /hf-inference/models/{model}", true, + [=] { return result_json(api->run_json(task, model, inputs, params)); }}; + } + + Plan endpoints(const Json& b) const { + const std::string action = text(b, "action", true); + const std::string ns = text(b, "namespace"); + const std::string name = text(b, "name"); + auto api = std::make_shared(hf::Endpoints::from_platform(p_)); + Json req = Json::object(); + req.set("action", Json(action)); + if (!ns.empty()) req.set("namespace", Json(ns)); + if (!name.empty()) req.set("name", Json(name)); + auto need_ns = [&] { if (ns.empty()) bad("namespace is required"); }; + auto need_name = [&] { need_ns(); if (name.empty()) bad("name is required"); hf::Endpoints::validate_name(name); }; + if (action == "vendors") return Plan{req, "GET /v2/provider", false, [=] { return api->vendors(); }}; + if (action == "list") { need_ns(); return Plan{req, "GET /v2/endpoint/{ns}", false, [=] { return api->list(ns); }}; } + if (action == "search") { need_ns(); std::string q = text(b, "q"); return Plan{req, "GET /v2/search/{ns}", false, [=] { return api->search(ns, q); }}; } + if (action == "states") { need_ns(); return Plan{req, "GET /v2/search/{ns}/states", false, [=] { return api->state_counts(ns); }}; } + if (action == "quotas") { need_ns(); return Plan{req, "GET /v2/provider/quotas/{ns}", false, [=] { return api->quotas(ns); }}; } + if (action == "audit") { need_ns(); return Plan{req, "GET /v2/audit-logs/{ns}", false, [=] { return api->audit_logs(ns); }}; } + if (action == "get") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}", false, [=] { return api->get(ns, name); }}; } + if (action == "replicas") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}/replica", false, [=] { return api->replicas(ns, name); }}; } + if (action == "logs") { + need_name(); + int tail = integer(b, "tail", 100, 1, 1000); + return Plan{req, "GET /v2/endpoint/{ns}/{name}/logs", false, [=] { + Json o = Json::object(); + o.set("logs", Json(api->logs(ns, name, tail))); + return o; + }}; + } + if (action == "metrics") { + need_name(); + hf::MetricKind k; + if (!hf::parse_metric_kind(text(b, "kind").empty() ? "all" : text(b, "kind"), k)) bad("unknown metric kind"); + std::string start = text(b, "start", true), stop = text(b, "stop", true); + return Plan{req, "POST /v2/endpoint/{ns}/{name}/metrics", false, [=] { return api->metrics(k, ns, name, start, stop); }}; + } + if (action == "create") { + need_ns(); + const Json* s = b.find("spec"); + if (!s || !s->is_object()) bad("spec must be an object"); + Json spec = *s; + hf::Endpoints::validate_spec(spec, true); + req.set("spec", spec); + return Plan{req, "POST /v2/endpoint/{ns} (costs money while it runs)", true, [=] { return api->create(ns, spec); }}; + } + if (action == "update") { + need_name(); + const Json* s = b.find("spec"); + if (!s || !s->is_object()) bad("spec must be an object"); + Json spec = *s; + hf::Endpoints::validate_spec(spec, false); + return Plan{req, "PUT /v2/endpoint/{ns}/{name}", true, [=] { return api->update(ns, name, spec); }}; + } + if (action == "pause") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/pause", true, [=] { return api->pause(ns, name); }}; } + if (action == "resume") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/resume", true, [=] { return api->resume(ns, name); }}; } + if (action == "scale_to_zero") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/scale-to-zero", true, [=] { return api->scale_to_zero(ns, name); }}; } + if (action == "remove") { need_name(); return Plan{req, "DELETE /v2/endpoint/{ns}/{name}", true, [=] { return api->remove_endpoint(ns, name); }}; } + bad("action must be vendors, list, search, states, quotas, audit, get, replicas, logs, metrics, create, update, pause, resume, scale_to_zero or remove"); + } + + hf::Access server_access(const Json& b) const { + const std::string url = text(b, "url", true); + try { + return hf::Platform::server(url, token_for(hf::Platform::check_server_url(url), token_)); + } catch (const clients::UrlError& e) { + bad(e.what()); + } + } + + Plan tgi(const Json& b) const { + const std::string action = text(b, "action", true); + auto api = std::make_shared(server_access(b)); + Json req = Json::object(); + req.set("action", Json(action)); + if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }}; + if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }}; + if (action == "models") return Plan{req, "GET /v1/models", false, [=] { return api->models(); }}; + if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }}; + Json params = b.find("parameters") ? *b.find("parameters") : Json(); + if (action == "generate") { std::string in = text(b, "inputs", true); return Plan{req, "POST /generate", false, [=] { return api->generate(in, params); }}; } + if (action == "tokenize") { std::string in = text(b, "inputs", true); return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(in, params); }}; } + if (action == "chat") { + llm::ChatRequest r = chat_request(b); + hf::ChatWire::to_body(r, "tgi", false); + auto model = std::make_shared(api, r.model); + return Plan{req, "POST /v1/chat/completions", false, [=] { + Json o = Json::object(); + o.set("text", Json(model->generate(r).text())); + return o; + }}; + } + bad("action must be health, info, models, metrics, generate, tokenize or chat"); + } + + Plan tei(const Json& b) const { + const std::string action = text(b, "action", true); + auto api = std::make_shared(server_access(b)); + Json req = Json::object(); + req.set("action", Json(action)); + if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }}; + if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }}; + if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }}; + if (action == "rerank") { + std::string q = text(b, "query", true); + std::vector texts = strings(b, "texts"); + return Plan{req, "POST /rerank", false, [=] { return api->rerank(q, texts, flag(b, "return_text")); }}; + } + const Json* in = b.find("inputs"); + if (!in || in->is_null()) bad("inputs is required"); + Json inputs = *in; + hf::TeiServer::check_inputs(inputs); + if (action == "embed") return Plan{req, "POST /embed", false, [=] { return api->embed(inputs); }}; + if (action == "predict") return Plan{req, "POST /predict", false, [=] { return api->predict(inputs); }}; + if (action == "tokenize") return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(inputs); }}; + bad("action must be health, info, metrics, embed, predict, rerank or tokenize"); + } + + private: + [[noreturn]] static void map(const llm::ApiError& e) { + if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") + throw fw::HttpError(503, "huggingface_not_configured", e.message()); + if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty()) + throw fw::HttpError(422, "validation_error", e.message()); + if (e.type() == "LOCAL_BUDGET") throw fw::HttpError(429, "huggingface_local_budget", e.message()); + throw fw::HttpError(502, "huggingface_error", e.type() + ": " + e.message()); + } + + hf::Platform p_; + std::string token_; +}; + +} // namespace + void register_ai_huggingface_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + auto routes = std::make_shared(); + fw::Environment* env = services.templates.get(); + services.templates->add_template("ai_huggingface.html", kPage); + services.add_feature("ai-huggingface", "/ai/huggingface"); + + router.get("/ai/huggingface", "ai-huggingface-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Hugging Face"); + return fw::Response(200, env->render("ai_huggingface.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/ai/huggingface/status", "ai-huggingface-status", [routes](fw::Request req) { + (void)req; + Json o = Json::object(); + o.set("configured", Json(routes->platform().configured())); + o.set("credential_env", Json(hf::Platform::kKeyEnv)); + o.set("hub", Json(hf::Platform::kHubHost)); + o.set("router", Json(hf::Platform::kRouterHost)); + o.set("endpoints", Json(hf::Platform::kEndpointsHost)); + o.set("calls", Json(static_cast(hf::Calls::count()))); + o.set("tasks", Json(static_cast(hf::TaskCatalog::count()))); + o.set("budget_calls", Json(hf::HfCaps::kBudgetMaxCalls)); + o.set("budget_window_seconds", Json(hf::HfCaps::kBudgetWindowSeconds)); + o.set("max_pages", Json(static_cast(hf::HfCaps::kMaxPages))); + return json_out(o); + }); + + router.get("/ai/huggingface/calls", "ai-huggingface-calls", [](fw::Request req) { + (void)req; + Json arr = Json::array(); + for (std::size_t i = 0; i < hf::Calls::count(); ++i) { + const hf::CallSpec& s = hf::Calls::at(i); + Json row = Json::object(); + row.set("surface", Json(hf::surface_name(s.surface))); + row.set("http", Json(s.http_method)); + row.set("path", Json(s.path)); + row.set("spec", Json(s.spec)); + row.set("auth", Json(s.auth)); + row.set("flags", Json(s.flags)); + arr.push(row); + } + Json o = Json::object(); + o.set("calls", arr); + return json_out(o); + }); + + router.get("/ai/huggingface/tasks", "ai-huggingface-tasks", [](fw::Request req) { + (void)req; + Json arr = Json::array(); + for (std::size_t i = 0; i < hf::TaskCatalog::count(); ++i) { + const hf::TaskInfo& t = hf::TaskCatalog::at(i); + Json row = Json::object(); + row.set("task", Json(t.name)); + row.set("binary_input", Json(t.binary_input)); + row.set("pipeline_path", Json(t.pipeline_path)); + row.set("note", Json(t.note)); + arr.push(row); + } + Json o = Json::object(); + o.set("tasks", arr); + return json_out(o); + }); + + using Member = HuggingfaceRoutes::Planner; + HuggingfaceRoutes* r = routes.get(); + auto add = [&](const char* path, const char* name, Plan (HuggingfaceRoutes::*m)(const Json&) const) { + Member planner = [r, m](const Json& b) { return (r->*m)(b); }; + router.post(path, name, [routes, planner](fw::Request req) { return routes->run(req, planner); }); + }; + add("/ai/huggingface/hub/list", "ai-huggingface-hub-list", &HuggingfaceRoutes::hub_list); + add("/ai/huggingface/hub/repo", "ai-huggingface-hub-repo", &HuggingfaceRoutes::hub_repo); + add("/ai/huggingface/hub/search", "ai-huggingface-hub-search", &HuggingfaceRoutes::hub_search); + add("/ai/huggingface/hub/whoami", "ai-huggingface-hub-whoami", &HuggingfaceRoutes::hub_whoami); + add("/ai/huggingface/hub/discussions", "ai-huggingface-hub-discussions", &HuggingfaceRoutes::hub_discussions); + add("/ai/huggingface/hub/collections", "ai-huggingface-hub-collections", &HuggingfaceRoutes::hub_collections); + add("/ai/huggingface/hub/webhooks", "ai-huggingface-hub-webhooks", &HuggingfaceRoutes::hub_webhooks); + add("/ai/huggingface/chat", "ai-huggingface-chat", &HuggingfaceRoutes::chat); + add("/ai/huggingface/chat/stream", "ai-huggingface-chat-stream", &HuggingfaceRoutes::chat_stream); + add("/ai/huggingface/task", "ai-huggingface-task", &HuggingfaceRoutes::task); + add("/ai/huggingface/endpoints", "ai-huggingface-endpoints", &HuggingfaceRoutes::endpoints); + add("/ai/huggingface/tgi", "ai-huggingface-tgi", &HuggingfaceRoutes::tgi); + add("/ai/huggingface/tei", "ai-huggingface-tei", &HuggingfaceRoutes::tei); } } // namespace molo::example::app diff --git a/example/assets/ai_huggingface.js b/example/assets/ai_huggingface.js new file mode 100644 index 0000000..3a4d958 --- /dev/null +++ b/example/assets/ai_huggingface.js @@ -0,0 +1,59 @@ +// Hugging Face page: thin client over /ai/huggingface/* (all limits server side). +(function () { + "use strict"; + var $ = function (id) { return document.getElementById(id); }; + var out = $("hf-out"), err = $("hf-error"); + var templates = { + "hub/list": { kind: "models", search: "bert", sort: "downloads", limit: 10 }, + "hub/repo": { action: "info", kind: "models", id: "openai-community/gpt2" }, + "hub/search": { action: "quicksearch", q: "llama", limit: 5 }, + "hub/whoami": {}, + "hub/discussions": { action: "list", kind: "models", id: "openai-community/gpt2" }, + "hub/collections": { action: "list", owner: "nvidia", limit: 5 }, + "hub/webhooks": { action: "list" }, + "chat": { model: "openai/gpt-oss-20b:fastest", prompt: "Hello!", max_tokens: 64 }, + "chat/stream": { model: "openai/gpt-oss-20b:fastest", prompt: "Count to five.", max_tokens: 64 }, + "task": { task: "text-classification", model: "distilbert/distilbert-base-uncased-finetuned-sst-2-english", inputs: "I like you" }, + "endpoints": { action: "vendors" }, + "tgi": { url: "http://127.0.0.1:8080", action: "health" }, + "tei": { url: "http://127.0.0.1:8081", action: "health" } + }; + var sel = $("hf-route"); + Object.keys(templates).forEach(function (k) { + var o = document.createElement("option"); + o.value = k; o.textContent = k; + sel.appendChild(o); + }); + function fill() { $("hf-body").value = JSON.stringify(templates[sel.value], null, 1); } + sel.onchange = fill; + fill(); + + function send(confirm) { + var body; + try { body = JSON.parse($("hf-body").value || "{}"); } + catch (e) { err.textContent = "Body is not valid JSON"; return; } + if (confirm) body.confirm = true; + err.textContent = ""; + out.textContent = "..."; + fetch("/ai/huggingface/" + sel.value, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body) + }).then(function (res) { + return res.text().then(function (t) { + var j; + try { j = JSON.parse(t); } catch (e) { j = { message: t }; } + if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; } + else { out.textContent = JSON.stringify(j, null, 2); } + }); + }).catch(function (e) { err.textContent = String(e); out.textContent = ""; }); + } + $("hf-send").onclick = function () { send(false); }; + $("hf-confirm").onclick = function () { send(true); }; + + fetch("/ai/huggingface/status").then(function (r) { return r.json(); }).then(function (s) { + $("hf-config").textContent = s.configured + ? "HF_TOKEN found. Calls are real; paid and changing ones need confirm." + : "No HF_TOKEN: calls answer 503 until one is configured; dry runs still work."; + }); +})(); diff --git a/src/vendors/huggingface/SPEC.md b/src/vendors/huggingface/SPEC.md new file mode 100644 index 0000000..0f491df --- /dev/null +++ b/src/vendors/huggingface/SPEC.md @@ -0,0 +1,234 @@ +# Hugging Face - Specification + +Owned directory: `src/vendors/huggingface/`. Namespace `molo::huggingface`. +Depends on `molo::llm` (`ApiClient`, `Service`, `SseDecoder`, `ChatModel`, +`Config`) and `molo::clients`. Built by `make ai` (the vendor glob in +`src/ai.mk` picks the directory up). Demo: `example/app/ai/HuggingfaceDemo.cpp`, +routes under `/ai/huggingface`. Plans: `plans/17-llm-vendor-platforms.md`, +`plans/18-breadth.md`. + +## 1. Scope + +Hugging Face is five HTTP surfaces on four hosts. All are covered: + +| Surface | Host | Class | What | +|---|---|---|---| +| Hub | `https://huggingface.co` | `Hub`, `Discussions`, `Collections`, `Webhooks` | models, datasets, spaces (list, info, revision), repo tree / refs / commits / compare / paths-info / likers / safetensors metadata, file download (`/resolve`), search, trending, tags, tasks, papers, Spaces catalogs, Inference Provider mappings (`/api/partners/{provider}/models`), whoami, discussions, collections, webhooks | +| Router (Inference Providers) | `https://router.huggingface.co` | `RouterChat`, `InferenceTasks` | OpenAI-compatible chat (`/v1/chat/completions`, routing suffix `:fastest` `:cheapest` `:preferred` `:`), `/v1/models`, hf-inference task APIs (25 tasks, the classic Inference API), hf-inference chat | +| Inference Endpoints | `https://api.endpoints.huggingface.cloud` | `Endpoints` | the whole management API v2 / v3: endpoints (list, get, create, update, pause, resume, scale to zero, delete), logs, 15 metric series, replicas, vendors and quotas, search, CIDR lists, custom domains, private services, audit logs | +| TGI server | any URL | `TgiServer` | native (`/generate`, `/generate_stream`, `/tokenize`, `/info`, `/health`, `/metrics`) and OpenAI-compatible (`/v1/chat/completions`, `/v1/completions`, `/v1/models`, `/invocations`) | +| TEI server | any URL | `TeiServer` | `/embed`, `/embed_all`, `/embed_sparse`, `/predict`, `/rerank`, `/similarity`, `/tokenize`, `/decode`, `/v1/embeddings`, `/info`, `/health`, `/metrics` | + +Plus `HuggingfaceChatModel : llm::ChatModel` over the router (default), a TGI +server, or any `ChatEndpoint`. + +Not covered (explicit, see `tools/check_spec.py` EXCLUDED and section 8): +repository writes (commit, upload, LFS, branches, tags, settings), organizations +and users, billing, Jobs, buckets / containers / kernels, gated-access requests, +SCIM, OAuth, `datasets-server.huggingface.co`, the `huggingface.co/api/blog` +and posts comment APIs, the provider-specific router paths. + +## 2. Module map (one class per concern) + +| File | Concern | +|---|---| +| `caps.h` | `HfCaps`: static ceilings | +| `calls.h/.cpp` | `Call` enum and `kCalls`: the ONE table of 132 HTTP calls (surface, method, path, spec kind, reference, auth, flags, body fields, query parameters) | +| `request_plan.h/.cpp` | `RequestPlan`, `PathArgs`, `Query`, `RepoRef`, `RepoType`: renders and validates one call from the table | +| `platform.h/.cpp` | `Platform`, `Access`, `RequestBudget`: hosts, `HF_TOKEN`, anonymous clients, shared call budget, server URL check | +| `hf_service.h/.cpp` | `HfService : llm::Service`: surface check, token check, budget, JSON / raw / bytes / paged / SSE calls | +| `hub.h/.cpp` | `Hub` | +| `community.h/.cpp` | `Discussions`, `Collections`, `Webhooks` (+ `CollectionItem`, `WebhookSpec`) | +| `providers.h/.cpp` | `RouterChat`, `RouterModel`, `TaskCatalog`, `InferenceTasks` | +| `endpoints.h/.cpp` | `Endpoints`, `MetricKind` | +| `servers.h/.cpp` | `TgiServer`, `TeiServer` | +| `chat_wire.h/.cpp` | `ChatWire`, `ChatEndpoint`: neutral chat <-> OpenAI-compatible wire | +| `chat_model.h/.cpp` | `HuggingfaceChatModel` | +| `tools/check_spec.py` | table vs published documents | + +## 3. Static caps (caps.h) + +| Knob | Value | +|---|---| +| Path segment / repo-relative file path / segments | 256 / 1024 chars / 32 | +| Query parameters / one query value | 32 / 2048 chars | +| Model id / server URL | 256 / 2048 chars | +| Local call budget | 1000 calls per 300 s per process (observed server limit: 500 per 300 s, `RateLimit-Policy: "fixed window";"api";q=500;w=300`) | +| Page walk | 20 pages, 5000 items, page `limit` 1000 | +| `paths-info` paths | 256 | +| Download | 64 MiB (the `llm` response cap; larger fails loudly) | +| Inference input (binary) | 16 MiB | +| Batch inputs (TEI, embeddings) | 256 entries, 4 MiB text | +| Stream events | 200000 | +| Messages / tools / stop sequences | 4096 / 128 / 4 | +| Webhook domains | 8 | + +## 4. Contracts + +- **Token.** `HF_TOKEN` (or the legacy `HUGGING_FACE_HUB_TOKEN`) via + `llm::Config`. It travels as `Authorization: Bearer` only, never in a URL or an + error. A row with `auth: token` and no token throws `NotConfiguredError` + (503 `NOT_CONFIGURED`) before any traffic. Rows with `auth: none` or + `optional` (public Hub reads, `GET /v1/models`, `GET /v2/provider`, TGI / TEI) + run without a token. +- **Anonymous requests.** `llm::ApiClient` cannot send an unauthenticated + request, so without a token `Platform` builds the client with a marker header + `X-Molo-Anonymous: 1` instead of `Authorization`. Foundation gap, worked + around inside the package. +- **One table.** Every path, query parameter and body field comes from `kCalls`. + `RequestPlan::make` refuses a query parameter or body field the row does not + list, an unused or missing path argument, a body on a bodiless row, and rows + flagged `unsupported`. +- **Ids.** `namespace`, `repo`, `name`: `[A-Za-z0-9._-]`, 1..256, not `.` / `..`. + Repo ids must be `owner/name`; legacy ids without an owner (`gpt2`) are refused + because the Hub answers them with a redirect to the canonical id. A revision or + tag is one percent-encoded segment (`refs/pr/1` -> `refs%2Fpr%2F1`). File + paths are validated per segment and percent-encoded. Query values are + percent-encoded; a value containing `..` has its dots encoded because + `ApiClient` refuses `..` anywhere in the request path. +- **Lists.** Hub list calls follow `Link: <...>; rel="next"` cursors, accept only + the `cursor` of a Link to the same host, and return `Page` with `truncated`, + `truncated_by` (`items` or `pages`) and `next_cursor`. Never silent. +- **Writes and deletes.** Rows flagged `write` need a token. Deletes are + `remove_*` methods with no defaults, flagged `destructive` in the table + (tested: 7 DELETE rows, all flagged). The demo runs every paid or changing + action as a dry run unless the body has `"confirm":true`. +- **Chat wire.** `ChatWire` documents its mapping in `chat_wire.h`: tool + arguments that are not JSON arrive as `{"_invalid_json": raw}`; reasoning is + read from `reasoning_content` then `reasoning`; `response_schema` maps to a + strict `json_schema`; usage is requested in streams. +- **Errors.** Everything is an `ApiError`; the Hub error body + `{"error":"..."}` and the Endpoints body + `{"error":"401 Unauthorized","code":"UNAUTHORIZED"}` keep their message and + status. A local budget stop is `429 LOCAL_BUDGET`. +- **Tests.** `MOLO_ENV_FILES=off`; `MOLO_LIVE_NET=1` enables the live tests; + `MOLO_LIVE_PAID=1` additionally allows the one chat completion. + +## 5. Credentials + +| Env var | Page | +|---|---| +| `HF_TOKEN` | https://huggingface.co/settings/tokens (docs: https://huggingface.co/docs/hub/security-tokens). A fine-grained token needs "Make calls to Inference Providers" for the router, "Make calls to Inference Endpoints" and "Manage Inference Endpoints" for the endpoints API, and write scopes for discussions / collections / webhooks. Inference Providers bills the token owner (or the organization named by `RouterChat::set_bill_to`) | + +No key exists in the development environment: see sections 7 and 8. + +## 6. Sources + +Machine-readable documents, fetched 2026-10-25: + +- Hub OpenAPI 3.1 `https://huggingface.co/.well-known/openapi.json` (374 + operations, 307 paths; also `/.well-known/openapi.md`). It omits the + list / info routes of models, datasets and spaces (`GET /api/models`, + `/api/models/{ns}/{repo}`, ...), `/api/tasks` and `/api/spaces/{ns}/{repo}/runtime`; + those 11 Hub rows cite `huggingface_hub` `hf_api.py` (spec kind `src`); the + three hf-inference router rows cite `hf_inference.py` (also `src`). +- Inference Endpoints OpenAPI 2.0.0 `https://api.endpoints.huggingface.cloud/openapi.json` + (48 operations). The old `/api-doc/openapi.json` answers 404; the swagger page + at `/` points at `/openapi.json`. +- TGI `docs/openapi.json` (3.3.6-dev0, 12 operations) and TEI `docs/openapi.json` + (1.9.4, 12 operations) from the `huggingface/text-generation-inference` and + `huggingface/text-embeddings-inference` repositories, branch `main`. +- `huggingface_hub` `src/huggingface_hub/hf_api.py`, `constants.py`, + `inference/_providers/__init__.py` and `hf_inference.py`: endpoints and + parameters of the routes the OpenAPI omits, the router URL template + `https://router.huggingface.co/{provider}`, the 25-task hf-inference map, the + feature-extraction / sentence-similarity `/pipeline/{task}` path, binary-input + tasks, base64 form with parameters. +- `huggingface.js` `packages/tasks/src/tasks//spec/input.json` for the + chat-completion body fields (26 tasks fetched; `audio-to-audio` and + `tabular-classification` have no schema file). +- Unauthenticated probes, section 8. + +## 7. Verified against spec + +`python3 -I src/vendors/huggingface/tools/check_spec.py --probe`, 2026-10-25: + +``` +132 rows checked; spec operations hub=374 ie=48 tgi=12 tei=12; 0 problems +``` + +For `ie`, `tgi`, `tei`: every document operation, query parameter and body +field is in the table (equality). For `hub`: every row's method, path (expanded +over models / datasets / spaces where the document has one path per type), +query parameters and body fields exist in the document; the other 314 +operations are excluded by explicit rules in `EXCLUDED`, counted per reason in +the output (jobs 23, buckets / containers / kernels 33, organizations / users / +SCIM 77, ...). The chat row is checked against the huggingface.js +chat-completion schema (equality of fields). `--probe` GETs the `src` and +`none` rows without a credential: all answered 200 (public) or 401 (the Hub's +answer for an unknown or private repo id), none 404. + +Local tests: `make ai-test` runs 66 `huggingface_*` tests (table integrity, id +and path rules, query and body checks, caps, Link walks, budget window, token +header, anonymous header, error mapping, chat wire, SSE transcript, every +surface's request shapes). Replies in tests are `RECORDED` (copied from a probe, +trimmed) or `SHAPE` (written to the documented schema). Nothing is a scripted +claim about vendor behavior. Example route tests: `tests/example/ai/huggingface/`. + +## 8. Verified live + +Run on 2026-10-25 with no credential unless stated. All through the real HTTP +transport (`MOLO_LIVE_NET=1 make ai-test`: 4 live tests, 3 run without a key). + +| Request | Result | +|---|---| +| `GET huggingface.co/api/models?limit=2`, `/api/datasets?limit=2`, `/api/spaces?limit=2` | 200 JSON array; `Link: <...&cursor=...>; rel="next"`; `RateLimit-Policy: "fixed window";"api";q=500;w=300` | +| `GET /api/models/gpt2`, `/api/datasets/squad`, `/api/models/gpt2/refs` | 307 to the canonical `openai-community/gpt2` / `rajpurkar/squad` (the reason legacy ids are refused) | +| `GET /api/models/openai-community/gpt2`, `/refs`, `/tree/main`, `/discussions`; `POST /paths-info/main`; `GET /openai-community/gpt2/resolve/main/config.json` (665 bytes, `X-Repo-Commit` header) | 200 | +| `GET /api/spaces/gradio/hello_world`, `/api/spaces/hardware`, `/api/trending`, `/api/models-tags-by-type`, `/api/tasks`, `/api/daily_papers`, `/api/quicksearch`, `/api/collections?limit=2`, `/api/collections/nvidia/flora-6ad6a8986320a55467825cf5`, `/api/partners/groq/models` | 200 | +| `GET /api/whoami-v2`, `/api/settings/webhooks` without a token | 401 `{"error":"Invalid username or password."}` | +| `GET router.huggingface.co/v1/models` | 200, 134 models, per model a `providers` array (status, context_length, pricing, supports_tools, latency) | +| `POST router.huggingface.co/v1/chat/completions`, `/v1/embeddings`, `/hf-inference/models/gpt2`, any other path | 401 `text/html` without a token, including a made-up path. The router answers 401 before routing, so these probes do NOT prove a path exists | +| `GET router.huggingface.co/hf-inference/models/gpt2` | 401 JSON `{"error":"Invalid username or password."}` | +| `GET api.endpoints.huggingface.cloud/v2/provider` | 200 (public vendor / region / compute / price list) | +| `GET .../v2/endpoint/huggingface`, `/v2/provider/huggingface`, `/v2/search/huggingface`, `/v2/endpoint/huggingface/x/logs` | 401 `{"error":"401 Unauthorized","code":"UNAUTHORIZED"}` | +| Invalid token `hf_invalid_molo_probe` on `whoami` and `endpoints list` | 401 with the two envelopes above (live test) | +| `api-inference.huggingface.co` | host does not resolve (the classic host is gone; hf-inference lives behind the router) | +| `datasets-server.huggingface.co/is-valid?dataset=squad` | 404 `{"error":"The dataset has been renamed..."}` (out of scope, noted) | + +## 9. Unverified + +No token was available, so no authenticated success body was seen: + +- every `auth: token` call: whoami, space runtime, discussions create / comment / + status / title / delete, collections create / update / add item / delete, + webhooks (all), Endpoints (all except `GET /v2/provider`); +- the router chat completion, its routing suffixes, the `X-HF-Bill-To` header + (taken from huggingface_hub / docs, unprobed), streaming usage in the last + chunk, `reasoning` vs `reasoning_content` per provider; +- every hf-inference task call and reply shape: the `/hf-inference/models/{model}` + and `/pipeline/{task}` paths and the binary / base64 input forms come from + `huggingface_hub` source, not from a response; +- the response shapes our code reads (`choices`, `usage`, tool call fields) are + the OpenAI-compatible shapes of the chat-completion spec, not seen from the + router; +- TGI and TEI: no server was reachable; request shapes follow their OpenAPI + documents; `ready()` treats 503 as "starting" per TGI behavior and the + documents, unobserved; +- `Endpoints::validate_spec` checks only the required top-level keys, the name + rule from the schema description and the `type` enum; +- ZeroGPU / pro quotas and per-token rate limits other than the observed + anonymous `q=500;w=300`; +- text ranges in `/resolve` (no range support), xet / LFS downloads of files + over 64 MiB. + +## 10. Gaps + +- `llm::ApiClient` cannot send an unauthenticated request (workaround: marker + header), has no GET stream (the three Endpoints SSE routes `GET .../sse`, + `.../logs/sse` and `/v3/.../logs/sse` are in the table with flag + `unsupported` and refused), no per-call timeout, and drops response headers on + `request()` (paging uses `request_raw`). +- `llm::ApiClient::build` refuses `..` in any request path including the query; + query values with `..` are dot-encoded as a workaround. +- Provider-specific router paths (`router.huggingface.co/{provider}/...`, for + example text-to-image on fal-ai or Replicate) have no per-provider schema + published; only `hf-inference` tasks and the generic chat route are wrapped. +- Collection item update / delete (`.../items/{itemId}`) are not wrapped: the + document names the path with the `{slug}-{id}` split. +- Repo writes (commit, upload, branches, tags, LFS) are out of scope; `Hub` + reads and downloads only. Legacy ids without owner are refused. +- `kernels` repo type exists on the Hub and in the discussions document; not + wrapped. +- Services `Discussions`, `Collections`, `Webhooks` share `Surface::kHub`. +- Google services used: none. diff --git a/src/vendors/huggingface/calls.cpp b/src/vendors/huggingface/calls.cpp new file mode 100644 index 0000000..150069a --- /dev/null +++ b/src/vendors/huggingface/calls.cpp @@ -0,0 +1,197 @@ +#include "vendors/huggingface/calls.h" + +#include + +#include "llm/error.h" + +namespace molo::huggingface { + +const char* surface_name(Surface surface) { + switch (surface) { + case Surface::kHub: return "hub"; + case Surface::kRouter: return "router"; + case Surface::kEndpoints: return "endpoints"; + case Surface::kTgi: return "tgi"; + case Surface::kTei: return "tei"; + } + return "?"; +} + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order: +// http method, path, spec, ref, auth, flags, body fields, query params) +static const CallSpec kCalls[] = { + {Call::kWhoami, Surface::kHub, "GET", "/api/whoami-v2", "hub", "-", "token", "", "", ""}, + {Call::kModelsTagsByType, Surface::kHub, "GET", "/api/models-tags-by-type", "hub", "-", "optional", "", "", "type"}, + {Call::kDatasetsTagsByType, Surface::kHub, "GET", "/api/datasets-tags-by-type", "hub", "-", "optional", "", "", "type"}, + {Call::kTrending, Surface::kHub, "GET", "/api/trending", "hub", "-", "optional", "", "", "limit,type"}, + {Call::kQuicksearch, Surface::kHub, "GET", "/api/quicksearch", "hub", "-", "optional", "", "", "discussionCollectionName,discussionId,exclude,includeInvitees,lang,library,limit,namespace,orgsFilter,pipelines,q,repoName,repoType,reposFilter,spacesTags,type"}, + {Call::kFullTextSearch, Surface::kHub, "GET", "/api/search/full-text", "hub", "-", "optional", "", "", "filter,limit,q,skip,type"}, + {Call::kRepoLikers, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/likers", "hub", "-", "optional", "", "", "expand"}, + {Call::kRepoTree, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/tree/{rev}/{path}", "hub", "-", "optional", "optpath", "", "cursor,expand,limit,recursive"}, + {Call::kRepoTreeSize, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/treesize/{rev}/{path}", "hub", "-", "optional", "optpath", "", ""}, + {Call::kRepoRefs, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/refs", "hub", "-", "optional", "", "", "include_prs"}, + {Call::kRepoCommits, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/commits/{rev}", "hub", "-", "optional", "", "", "expand,limit,p"}, + {Call::kRepoCompare, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/compare/{compare}", "hub", "-", "optional", "", "", "raw"}, + {Call::kRepoPathsInfo, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/paths-info/{rev}", "hub", "-", "optional", "", "expand,paths", ""}, + {Call::kModelSafetensors, Surface::kHub, "GET", "/api/models/{namespace}/{repo}/safetensors-metadata/{path}", "hub", "-", "optional", "", "", ""}, + {Call::kFileResolve, Surface::kHub, "GET", "{typePrefix}/{namespace}/{repo}/resolve/{rev}/{path}", "hub", "-", "optional", "raw", "", ""}, + {Call::kDiscussionList, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/discussions", "hub", "-", "optional", "", "", "author,p,search,sort,status,type"}, + {Call::kDiscussionCreate, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions", "hub", "-", "token", "write", "description,pullRequest,title", ""}, + {Call::kDiscussionGet, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/discussions/{num}", "hub", "-", "optional", "", "", ""}, + {Call::kDiscussionDelete, Surface::kHub, "DELETE", "/api/{repoType}/{namespace}/{repo}/discussions/{num}", "hub", "-", "token", "write,destructive", "", ""}, + {Call::kDiscussionComment, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions/{num}/comment", "hub", "-", "token", "write", "comment", ""}, + {Call::kDiscussionStatus, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions/{num}/status", "hub", "-", "token", "write", "comment,status", ""}, + {Call::kDiscussionTitle, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions/{num}/title", "hub", "-", "token", "write", "title", ""}, + {Call::kCollectionsList, Surface::kHub, "GET", "/api/collections", "hub", "-", "optional", "", "", "cursor,expand,item,limit,owner,q,sort"}, + {Call::kCollectionGet, Surface::kHub, "GET", "/api/collections/{namespace}/{slug}", "hub", "-", "optional", "", "", ""}, + {Call::kCollectionCreate, Surface::kHub, "POST", "/api/collections", "hub", "-", "token", "write", "description,item,namespace,private,resourceGroupId,title", ""}, + {Call::kCollectionUpdate, Surface::kHub, "PATCH", "/api/collections/{namespace}/{slug}", "hub", "-", "token", "write", "description,gating,position,private,theme,title", ""}, + {Call::kCollectionDelete, Surface::kHub, "DELETE", "/api/collections/{namespace}/{slug}", "hub", "-", "token", "write,destructive", "", ""}, + {Call::kCollectionAddItem, Surface::kHub, "POST", "/api/collections/{namespace}/{slug}/items", "hub", "-", "token", "write", "item,note", ""}, + {Call::kWebhooksList, Surface::kHub, "GET", "/api/settings/webhooks", "hub", "-", "token", "", "", ""}, + {Call::kWebhookCreate, Surface::kHub, "POST", "/api/settings/webhooks", "hub", "-", "token", "write", "domains,job,jobSourceId,secret,secrets,url,watched", ""}, + {Call::kWebhookGet, Surface::kHub, "GET", "/api/settings/webhooks/{webhookId}", "hub", "-", "token", "", "", ""}, + {Call::kWebhookUpdate, Surface::kHub, "POST", "/api/settings/webhooks/{webhookId}", "hub", "-", "token", "write", "domains,job,jobSourceId,secret,secrets,url,watched", ""}, + {Call::kWebhookDelete, Surface::kHub, "DELETE", "/api/settings/webhooks/{webhookId}", "hub", "-", "token", "write,destructive", "", ""}, + {Call::kWebhookToggle, Surface::kHub, "POST", "/api/settings/webhooks/{webhookId}/{action}", "hub", "-", "token", "write", "", ""}, + {Call::kWebhookReplay, Surface::kHub, "POST", "/api/settings/webhooks/{webhookId}/replay/{logId}", "hub", "-", "token", "write", "", ""}, + {Call::kDailyPapers, Surface::kHub, "GET", "/api/daily_papers", "hub", "-", "optional", "", "", "date,limit,month,p,sort,submitter,week"}, + {Call::kPapersList, Surface::kHub, "GET", "/api/papers", "hub", "-", "optional", "", "", "cursor,limit"}, + {Call::kPapersSearch, Surface::kHub, "GET", "/api/papers/search", "hub", "-", "optional", "", "", "limit,q"}, + {Call::kPaperGet, Surface::kHub, "GET", "/api/papers/{paperId}", "hub", "-", "optional", "", "", "field"}, + {Call::kSpacesHardware, Surface::kHub, "GET", "/api/spaces/hardware", "hub", "-", "optional", "", "", ""}, + {Call::kSpacesTemplates, Surface::kHub, "GET", "/api/spaces/templates", "hub", "-", "optional", "", "", ""}, + {Call::kSpacesSemanticSearch, Surface::kHub, "GET", "/api/spaces/semantic-search", "hub", "-", "optional", "", "", "agents,category,filter,includeNonRunning,q,sdk"}, + {Call::kSpacesFeatured, Surface::kHub, "GET", "/api/spaces/featured", "hub", "-", "optional", "", "", "date"}, + {Call::kPartnerModels, Surface::kHub, "GET", "/api/partners/{provider}/models", "hub", "-", "optional", "", "", "status"}, + {Call::kModelsList, Surface::kHub, "GET", "/api/models", "src", "{self.endpoint}/api/models\"", "optional", "", "", "filter,author,apps,gated,inference,inference_provider,pipeline_tag,num_parameters,search,sort,limit,full,config,cardData,expand,cursor,direction"}, + {Call::kDatasetsList, Surface::kHub, "GET", "/api/datasets", "src", "{self.endpoint}/api/datasets\"", "optional", "", "", "filter,author,gated,search,sort,limit,full,expand,cursor,direction"}, + {Call::kSpacesList, Surface::kHub, "GET", "/api/spaces", "src", "{self.endpoint}/api/spaces\"", "optional", "", "", "filter,author,search,sort,limit,full,expand,cursor,direction,datasets,models,linked"}, + {Call::kModelInfo, Surface::kHub, "GET", "/api/models/{namespace}/{repo}", "src", "/api/models/{repo_id}\"", "optional", "", "", "securityStatus,blobs,expand"}, + {Call::kModelRevision, Surface::kHub, "GET", "/api/models/{namespace}/{repo}/revision/{rev}", "src", "/api/models/{repo_id}/revision/", "optional", "", "", "securityStatus,blobs,expand"}, + {Call::kDatasetInfo, Surface::kHub, "GET", "/api/datasets/{namespace}/{repo}", "src", "/api/datasets/{repo_id}\"", "optional", "", "", "blobs,expand"}, + {Call::kDatasetRevision, Surface::kHub, "GET", "/api/datasets/{namespace}/{repo}/revision/{rev}", "src", "/api/datasets/{repo_id}/revision/", "optional", "", "", "blobs,expand"}, + {Call::kSpaceInfo, Surface::kHub, "GET", "/api/spaces/{namespace}/{repo}", "src", "/api/spaces/{repo_id}\"", "optional", "", "", "expand"}, + {Call::kSpaceRevision, Surface::kHub, "GET", "/api/spaces/{namespace}/{repo}/revision/{rev}", "src", "/api/spaces/{repo_id}/revision/", "optional", "", "", "expand"}, + {Call::kSpaceRuntime, Surface::kHub, "GET", "/api/spaces/{namespace}/{repo}/runtime", "src", "/api/spaces/{repo_id}/runtime", "token", "", "", ""}, + {Call::kTasks, Surface::kHub, "GET", "/api/tasks", "src", "{constants.ENDPOINT}/api/tasks", "optional", "", "", ""}, + {Call::kRouterChat, Surface::kRouter, "POST", "/v1/chat/completions", "task", "chat-completion", "token", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,reasoning_effort,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""}, + {Call::kRouterModels, Surface::kRouter, "GET", "/v1/models", "none", "probe", "none", "", "", ""}, + {Call::kHfInferenceTask, Surface::kRouter, "POST", "/hf-inference/models/{model}", "src", "models/{mapped_model}", "token", "", "inputs,parameters,stream", ""}, + {Call::kHfInferencePipeline, Surface::kRouter, "POST", "/hf-inference/models/{model}/pipeline/{task}", "src", "pipeline/{self.task}", "token", "", "inputs,parameters,normalize,prompt_name,truncate,truncation_direction", ""}, + {Call::kHfInferenceChat, Surface::kRouter, "POST", "/hf-inference/models/{model}/v1/chat/completions", "src", "\"/v1/chat/completions\"", "token", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,reasoning_effort,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""}, + {Call::kIeGetAuditLogs, Surface::kEndpoints, "GET", "/v2/audit-logs/{namespace}", "ie", "get_audit_logs", "token", "", "", "cursor,endpoint,limit,username"}, + {Call::kIeListEndpoint, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}", "ie", "list_endpoint", "token", "", "", "cursor,direction,engine,instanceSize,instanceType,limit,search,sortBy,state,tags,task,type"}, + {Call::kIeCreateEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}", "ie", "create_endpoint", "token", "write", "cacheHttpResponses,compute,customRouter,experimentalFeatures,model,name,network,notifications,privateService,provider,route,tags,type", ""}, + {Call::kIeGetEndpoint, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}", "ie", "get_endpoint", "token", "", "", ""}, + {Call::kIeUpdateEndpoint, Surface::kEndpoints, "PUT", "/v2/endpoint/{namespace}/{name}", "ie", "update_endpoint", "token", "write", "compute,customRouter,experimentalFeatures,model,network,notifications,provider,route,tags,type", ""}, + {Call::kIeDeleteEndpoint, Surface::kEndpoints, "DELETE", "/v2/endpoint/{namespace}/{name}", "ie", "delete_endpoint", "token", "write,destructive", "", ""}, + {Call::kIeDebugAgent, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/agent/debug", "ie", "debug_agent", "token", "write", "", ""}, + {Call::kIeGetEndpointLogs, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/logs", "ie", "get_endpoint_logs", "token", "raw", "", "follow,line_max_length,replica,tail"}, + {Call::kIeGetEndpointLogsSse, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/logs/sse", "ie", "get_endpoint_logs_sse", "token", "unsupported", "", "line_max_length,replica,tail"}, + {Call::kIeGetEndpointMetrics, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics", "ie", "get_endpoint_metrics", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetHardwareCpu, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/hardware-cpu", "ie", "get_hardware_cpu", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetHardwareGpu, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/hardware-gpu", "ie", "get_hardware_gpu", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetInferenceItl, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/itl", "ie", "get_inference_itl", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetInferenceKvCache, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/kv-cache", "ie", "get_inference_kv_cache", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetInferencePrefixCache, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/prefix-cache", "ie", "get_inference_prefix_cache", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetInferenceRequests, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/requests", "ie", "get_inference_requests", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetInferenceTtft, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/ttft", "ie", "get_inference_ttft", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetPendingRequest, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/pending-request", "ie", "get_pending_request", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetReplicaIds, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/replica-ids", "ie", "get_replica_ids", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetReplicaRunning, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/replica-running", "ie", "get_replica_running", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetReplicaStatus, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/replica-status", "ie", "get_replica_status", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetResponseElapsed, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/response-elapsed", "ie", "get_response_elapsed", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetResponseStatusCode, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/response-status-code", "ie", "get_response_status_code", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetStatusCodeGrouped, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/response-status-code-grouped", "ie", "get_status_code_grouped", "token", "", "replicaId,start,stop", ""}, + {Call::kIeGetEndpointOpenMetrics, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/open-metrics", "ie", "get_endpoint_open_metrics", "token", "raw", "", ""}, + {Call::kIePauseEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/pause", "ie", "pause_endpoint", "token", "write,destructive", "", ""}, + {Call::kIeGetEndpointReplica, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/replica", "ie", "get_endpoint_replica", "token", "", "", ""}, + {Call::kIeResumeEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/resume", "ie", "resume_endpoint", "token", "write", "", ""}, + {Call::kIeScaleToZeroEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/scale-to-zero", "ie", "scale_to_zero_endpoint", "token", "write,destructive", "", ""}, + {Call::kIeGetEndpointSse, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/sse", "ie", "get_endpoint_sse", "token", "unsupported", "", ""}, + {Call::kIeListCidrsList, Surface::kEndpoints, "GET", "/v2/network/cidr-list/{namespace}", "ie", "list_cidrs_list", "token", "", "", ""}, + {Call::kIeCreateCidrList, Surface::kEndpoints, "POST", "/v2/network/cidr-list/{namespace}", "ie", "create_cidr_list", "token", "write", "cidrList,id,name", ""}, + {Call::kIeUpdateCidrList, Surface::kEndpoints, "POST", "/v2/network/cidr-list/{namespace}/{cidr_id}", "ie", "update_cidr_list", "token", "write", "cidrList,id,name", ""}, + {Call::kIeDeleteCidrList, Surface::kEndpoints, "DELETE", "/v2/network/cidr-list/{namespace}/{cidr_id}", "ie", "delete_cidr_list", "token", "write,destructive", "", ""}, + {Call::kIeListDomains, Surface::kEndpoints, "GET", "/v2/network/domain/{namespace}", "ie", "list_domains", "token", "", "", ""}, + {Call::kIeCreateDomain, Surface::kEndpoints, "POST", "/v2/network/domain/{namespace}", "ie", "create_domain", "token", "write", "domain", ""}, + {Call::kIeDeleteDomain, Surface::kEndpoints, "DELETE", "/v2/network/domain/{namespace}/{domain}", "ie", "delete_domain", "token", "write,destructive", "", ""}, + {Call::kIeListPrivateServices, Surface::kEndpoints, "GET", "/v2/network/private-service/{namespace}", "ie", "list_private_services", "token", "", "", "account-id,shared"}, + {Call::kIeDeletePrivateService, Surface::kEndpoints, "DELETE", "/v2/network/private-service/{namespace}/{service-id}", "ie", "delete_private_service", "token", "write,destructive", "", ""}, + {Call::kIeListVendors, Surface::kEndpoints, "GET", "/v2/provider", "ie", "list_vendors", "none", "", "", ""}, + {Call::kIeListVendorsQuotasNamespace, Surface::kEndpoints, "GET", "/v2/provider/quotas/{namespace}", "ie", "list_vendors_quotas_namespace", "token", "", "", ""}, + {Call::kIeListVendorsNamespace, Surface::kEndpoints, "GET", "/v2/provider/{namespace}", "ie", "list_vendors_namespace", "token", "", "", ""}, + {Call::kIeSearchEndpoint, Surface::kEndpoints, "GET", "/v2/search/{namespace}", "ie", "search_endpoint", "token", "", "", "limit,search,tags"}, + {Call::kIeCountEndpointStates, Surface::kEndpoints, "GET", "/v2/search/{namespace}/states", "ie", "count_endpoint_states", "token", "", "", ""}, + {Call::kIeListEndpointTags, Surface::kEndpoints, "GET", "/v2/search/{namespace}/tags", "ie", "list_endpoint_tags", "token", "", "", "limit,prefix"}, + {Call::kIeGetEndpointLogsV3, Surface::kEndpoints, "GET", "/v3/endpoint/{namespace}/{name}/logs", "ie", "get_endpoint_logs_v3", "token", "", "", "after,before,level,limit,line_max_length,order,replica,search,since,until"}, + {Call::kIeDownloadEndpointLogsV3, Surface::kEndpoints, "GET", "/v3/endpoint/{namespace}/{name}/logs/download", "ie", "download_endpoint_logs_v3", "token", "raw", "", "level,line_max_length,replica,search,since,until"}, + {Call::kIeGetEndpointLogsSseV3, Surface::kEndpoints, "GET", "/v3/endpoint/{namespace}/{name}/logs/sse", "ie", "get_endpoint_logs_sse_v3", "token", "unsupported", "", "after,level,line_max_length,replica,search,since,tail"}, + {Call::kTgiCompatGenerate, Surface::kTgi, "POST", "/", "tgi", "compat_generate", "optional", "", "inputs,parameters,stream", ""}, + {Call::kTgiGetChatTokenize, Surface::kTgi, "POST", "/chat_tokenize", "tgi", "get_chat_tokenize", "optional", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""}, + {Call::kTgiGenerate, Surface::kTgi, "POST", "/generate", "tgi", "generate", "optional", "", "inputs,parameters", ""}, + {Call::kTgiGenerateStream, Surface::kTgi, "POST", "/generate_stream", "tgi", "generate_stream", "optional", "stream", "inputs,parameters", ""}, + {Call::kTgiHealth, Surface::kTgi, "GET", "/health", "tgi", "health", "optional", "", "", ""}, + {Call::kTgiGetModelInfo, Surface::kTgi, "GET", "/info", "tgi", "get_model_info", "optional", "", "", ""}, + {Call::kTgiSagemakerCompatibility, Surface::kTgi, "POST", "/invocations", "tgi", "sagemaker_compatibility", "optional", "", "frequency_penalty,inputs,logit_bias,logprobs,max_tokens,messages,model,n,parameters,presence_penalty,prompt,repetition_penalty,response_format,seed,stop,stream,stream_options,suffix,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""}, + {Call::kTgiMetrics, Surface::kTgi, "GET", "/metrics", "tgi", "metrics", "optional", "raw", "", ""}, + {Call::kTgiTokenize, Surface::kTgi, "POST", "/tokenize", "tgi", "tokenize", "optional", "", "inputs,parameters", ""}, + {Call::kTgiChatCompletions, Surface::kTgi, "POST", "/v1/chat/completions", "tgi", "chat_completions", "optional", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""}, + {Call::kTgiCompletions, Surface::kTgi, "POST", "/v1/completions", "tgi", "completions", "optional", "", "frequency_penalty,max_tokens,model,prompt,repetition_penalty,seed,stop,stream,suffix,temperature,top_p", ""}, + {Call::kTgiOpenaiGetModelInfo, Surface::kTgi, "GET", "/v1/models", "tgi", "openai_get_model_info", "optional", "", "", ""}, + {Call::kTeiDecode, Surface::kTei, "POST", "/decode", "tei", "decode", "optional", "", "ids,skip_special_tokens", ""}, + {Call::kTeiEmbed, Surface::kTei, "POST", "/embed", "tei", "embed", "optional", "", "dimensions,inputs,normalize,prompt_name,truncate,truncation_direction", ""}, + {Call::kTeiEmbedAll, Surface::kTei, "POST", "/embed_all", "tei", "embed_all", "optional", "", "inputs,prompt_name,truncate,truncation_direction", ""}, + {Call::kTeiEmbedSparse, Surface::kTei, "POST", "/embed_sparse", "tei", "embed_sparse", "optional", "", "inputs,prompt_name,truncate,truncation_direction", ""}, + {Call::kTeiHealth, Surface::kTei, "GET", "/health", "tei", "health", "optional", "", "", ""}, + {Call::kTeiGetModelInfo, Surface::kTei, "GET", "/info", "tei", "get_model_info", "optional", "", "", ""}, + {Call::kTeiMetrics, Surface::kTei, "GET", "/metrics", "tei", "metrics", "optional", "raw", "", ""}, + {Call::kTeiPredict, Surface::kTei, "POST", "/predict", "tei", "predict", "optional", "", "inputs,raw_scores,truncate,truncation_direction", ""}, + {Call::kTeiRerank, Surface::kTei, "POST", "/rerank", "tei", "rerank", "optional", "", "query,raw_scores,return_text,texts,truncate,truncation_direction", ""}, + {Call::kTeiSimilarity, Surface::kTei, "POST", "/similarity", "tei", "similarity", "optional", "", "inputs,parameters", ""}, + {Call::kTeiTokenize, Surface::kTei, "POST", "/tokenize", "tei", "tokenize", "optional", "", "add_special_tokens,inputs,prompt_name", ""}, + {Call::kTeiOpenaiEmbed, Surface::kTei, "POST", "/v1/embeddings", "tei", "openai_embed", "optional", "", "dimensions,encoding_format,input,model,user", ""}, +}; +// CALLS-END + +bool CallSpec::has_flag(const char* flag) const { return Calls::in_list(flags, flag); } + +bool Calls::in_list(const char* list, const std::string& name) { + const std::size_t n = name.size(); + const char* p = list; + while (*p) { + const char* end = std::strchr(p, ','); + std::size_t len = end ? static_cast(end - p) : std::strlen(p); + if (len == n && std::strncmp(p, name.c_str(), n) == 0) return true; + if (!end) break; + p = end + 1; + } + return false; +} + +std::size_t Calls::count() { return sizeof(kCalls) / sizeof(kCalls[0]); } + +const CallSpec& Calls::at(std::size_t index) { + if (index >= count()) throw llm::ApiError::invalid("huggingface", "call index out of range"); + return kCalls[index]; +} + +const CallSpec& Calls::spec_of(Call call) { + const std::size_t i = static_cast(call); + if (i >= count() || kCalls[i].call != call) + throw llm::ApiError::invalid("huggingface", "call table out of order"); + return kCalls[i]; +} + +clients::Method Calls::method_of(Call call) { + const char* m = spec_of(call).http_method; + if (!std::strcmp(m, "GET")) return clients::Method::kGet; + if (!std::strcmp(m, "POST")) return clients::Method::kPost; + if (!std::strcmp(m, "PUT")) return clients::Method::kPut; + if (!std::strcmp(m, "PATCH")) return clients::Method::kPatch; + if (!std::strcmp(m, "DELETE")) return clients::Method::kDelete; + throw llm::ApiError::invalid("huggingface", "unknown http method in the call table"); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/calls.h b/src/vendors/huggingface/calls.h new file mode 100644 index 0000000..b8a086b --- /dev/null +++ b/src/vendors/huggingface/calls.h @@ -0,0 +1,189 @@ +#pragma once + +// The ONE table of HTTP calls this module makes (every surface). Code builds +// every path, query and body check from it, never from a literal. +// tools/check_spec.py verifies each row against the published documents: +// spec "hub" -> https://huggingface.co/.well-known/openapi.json (method, path, query, body fields) +// spec "ie" -> https://api.endpoints.huggingface.cloud/openapi.json +// spec "tgi" -> text-generation-inference docs/openapi.json +// spec "tei" -> text-embeddings-inference docs/openapi.json +// spec "src" -> pinned huggingface_hub source (the Hub OpenAPI omits these): `ref` must occur in it +// spec "task" -> huggingface.js task input schema named by `ref` +// spec "none" -> not in any published document; confirmed by an unauthenticated probe (see SPEC) +// Fields: auth is "none" (public), "optional" (public, a token widens it) or +// "token" (needs HF_TOKEN). flags: write, destructive, raw (non JSON reply), +// stream, optpath (a trailing {path} may be empty), unsupported (listed for +// coverage, refused at call time with the reason in SPEC.md). + +#include +#include + +#include "clients/request.h" + +namespace molo::huggingface { + +enum class Surface { kHub, kRouter, kEndpoints, kTgi, kTei }; + +const char* surface_name(Surface surface); + +enum class Call { + kWhoami, + kModelsTagsByType, + kDatasetsTagsByType, + kTrending, + kQuicksearch, + kFullTextSearch, + kRepoLikers, + kRepoTree, + kRepoTreeSize, + kRepoRefs, + kRepoCommits, + kRepoCompare, + kRepoPathsInfo, + kModelSafetensors, + kFileResolve, + kDiscussionList, + kDiscussionCreate, + kDiscussionGet, + kDiscussionDelete, + kDiscussionComment, + kDiscussionStatus, + kDiscussionTitle, + kCollectionsList, + kCollectionGet, + kCollectionCreate, + kCollectionUpdate, + kCollectionDelete, + kCollectionAddItem, + kWebhooksList, + kWebhookCreate, + kWebhookGet, + kWebhookUpdate, + kWebhookDelete, + kWebhookToggle, + kWebhookReplay, + kDailyPapers, + kPapersList, + kPapersSearch, + kPaperGet, + kSpacesHardware, + kSpacesTemplates, + kSpacesSemanticSearch, + kSpacesFeatured, + kPartnerModels, + kModelsList, + kDatasetsList, + kSpacesList, + kModelInfo, + kModelRevision, + kDatasetInfo, + kDatasetRevision, + kSpaceInfo, + kSpaceRevision, + kSpaceRuntime, + kTasks, + kRouterChat, + kRouterModels, + kHfInferenceTask, + kHfInferencePipeline, + kHfInferenceChat, + kIeGetAuditLogs, + kIeListEndpoint, + kIeCreateEndpoint, + kIeGetEndpoint, + kIeUpdateEndpoint, + kIeDeleteEndpoint, + kIeDebugAgent, + kIeGetEndpointLogs, + kIeGetEndpointLogsSse, + kIeGetEndpointMetrics, + kIeGetHardwareCpu, + kIeGetHardwareGpu, + kIeGetInferenceItl, + kIeGetInferenceKvCache, + kIeGetInferencePrefixCache, + kIeGetInferenceRequests, + kIeGetInferenceTtft, + kIeGetPendingRequest, + kIeGetReplicaIds, + kIeGetReplicaRunning, + kIeGetReplicaStatus, + kIeGetResponseElapsed, + kIeGetResponseStatusCode, + kIeGetStatusCodeGrouped, + kIeGetEndpointOpenMetrics, + kIePauseEndpoint, + kIeGetEndpointReplica, + kIeResumeEndpoint, + kIeScaleToZeroEndpoint, + kIeGetEndpointSse, + kIeListCidrsList, + kIeCreateCidrList, + kIeUpdateCidrList, + kIeDeleteCidrList, + kIeListDomains, + kIeCreateDomain, + kIeDeleteDomain, + kIeListPrivateServices, + kIeDeletePrivateService, + kIeListVendors, + kIeListVendorsQuotasNamespace, + kIeListVendorsNamespace, + kIeSearchEndpoint, + kIeCountEndpointStates, + kIeListEndpointTags, + kIeGetEndpointLogsV3, + kIeDownloadEndpointLogsV3, + kIeGetEndpointLogsSseV3, + kTgiCompatGenerate, + kTgiGetChatTokenize, + kTgiGenerate, + kTgiGenerateStream, + kTgiHealth, + kTgiGetModelInfo, + kTgiSagemakerCompatibility, + kTgiMetrics, + kTgiTokenize, + kTgiChatCompletions, + kTgiCompletions, + kTgiOpenaiGetModelInfo, + kTeiDecode, + kTeiEmbed, + kTeiEmbedAll, + kTeiEmbedSparse, + kTeiHealth, + kTeiGetModelInfo, + kTeiMetrics, + kTeiPredict, + kTeiRerank, + kTeiSimilarity, + kTeiTokenize, + kTeiOpenaiEmbed, +}; + +struct CallSpec { + Call call; + Surface surface; + const char* http_method; // GET POST PUT PATCH DELETE + const char* path; // template: {name} placeholders + const char* spec; // none, hub, ie, tgi, tei, src, task + const char* ref; // operationId, source needle, or task name + const char* auth; // none, optional, token + const char* flags; // comma list + const char* body_fields; // comma list: accepted top-level body fields (empty: no body) + const char* query_params; // comma list: accepted query parameters + + bool has_flag(const char* flag) const; + bool has_body() const { return body_fields[0] != '\0'; } +}; + +struct Calls { + static const CallSpec& spec_of(Call call); + static clients::Method method_of(Call call); + static std::size_t count(); + static const CallSpec& at(std::size_t index); + // True when `name` is in the comma list. + static bool in_list(const char* list, const std::string& name); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/caps.h b/src/vendors/huggingface/caps.h new file mode 100644 index 0000000..f46c0fa --- /dev/null +++ b/src/vendors/huggingface/caps.h @@ -0,0 +1,46 @@ +#pragma once + +// Static ceilings for the Hugging Face module. Never autotuned. The numbers +// marked "observed" come from response headers seen on 2026-10-25 (see +// SPEC.md ## Verified live); the rest are our own hard caps. + +#include + +namespace molo::huggingface { + +struct HfCaps { + // Path and query rendering. + static constexpr std::size_t kMaxSegmentChars = 256; // one path segment (id, rev, tag) + static constexpr std::size_t kMaxFilePathChars = 1024; // a path inside a repo + static constexpr std::size_t kMaxPathSegments = 32; + static constexpr std::size_t kMaxQueryParams = 32; + static constexpr std::size_t kMaxQueryValueChars = 2048; + static constexpr std::size_t kMaxModelChars = 256; + static constexpr std::size_t kMaxServerUrlChars = 2048; + + // Request budget. observed: RateLimit-Policy "fixed window";"api";q=500;w=300 + // on huggingface.co. We allow twice that per process so a single process + // never trips the server limit by accident, and stop locally beyond it. + static constexpr int kBudgetWindowSeconds = 300; + static constexpr int kBudgetMaxCalls = 1000; + + // Page walks (Link: rel="next" cursors). Truncation is reported, never silent. + static constexpr std::size_t kMaxPages = 20; + static constexpr std::size_t kMaxItems = 5000; + static constexpr int kMaxPageLimit = 1000; + static constexpr std::size_t kMaxPathsInfo = 256; + + // Payloads. + static constexpr std::size_t kMaxDownloadBytes = 64u * 1024u * 1024u; // matches llm::Caps response cap + static constexpr std::size_t kMaxInferenceInputBytes = 16u * 1024u * 1024u; + static constexpr std::size_t kMaxBatchInputs = 256; // TEI embed / rerank batches + static constexpr std::size_t kMaxStreamEvents = 200000; + static constexpr std::size_t kMaxStopSequences = 16; + static constexpr std::size_t kMaxMessages = 4096; + static constexpr std::size_t kMaxTools = 128; + static constexpr int kMaxTokensCeiling = 1048576; + static constexpr std::size_t kMaxWebhookDomains = 8; + static constexpr std::size_t kMaxTextChars = 4u * 1024u * 1024u; +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/chat_model.cpp b/src/vendors/huggingface/chat_model.cpp new file mode 100644 index 0000000..e9fdd7c --- /dev/null +++ b/src/vendors/huggingface/chat_model.cpp @@ -0,0 +1,45 @@ +#include "vendors/huggingface/chat_model.h" + +#include "vendors/huggingface/providers.h" +#include "llm/error.h" + +namespace molo::huggingface { + +HuggingfaceChatModel::HuggingfaceChatModel(std::shared_ptr endpoint, std::string default_model) + : endpoint_(std::move(endpoint)), default_model_(std::move(default_model)) { + if (!endpoint_) throw llm::ApiError::invalid("huggingface", "chat model needs an endpoint"); +} + +std::shared_ptr HuggingfaceChatModel::from_env(const std::string& default_model) { + return over_router(Platform::from_env(), default_model); +} + +std::shared_ptr HuggingfaceChatModel::over_router(const Platform& platform, + const std::string& default_model) { + return std::make_shared(std::make_shared(RouterChat::from_platform(platform)), + default_model); +} + +clients::body::Json HuggingfaceChatModel::to_body(const llm::ChatRequest& request, bool stream) const { + return ChatWire::to_body(request, default_model_, stream); +} + +llm::ChatResponse HuggingfaceChatModel::generate(const llm::ChatRequest& request) { + return ChatWire::to_response(endpoint_->create(to_body(request, false))); +} + +void HuggingfaceChatModel::stream(const llm::ChatRequest& request, + const std::function& on_chunk) { + ChatWire::StreamState state; + bool stopped = false; + endpoint_->stream(to_body(request, true), [&](const llm::SseEvent& e) { + if (stopped) return false; + if (!ChatWire::on_event(e.data, state, on_chunk)) { + stopped = true; + return false; + } + return true; + }); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/chat_model.h b/src/vendors/huggingface/chat_model.h new file mode 100644 index 0000000..2c14712 --- /dev/null +++ b/src/vendors/huggingface/chat_model.h @@ -0,0 +1,40 @@ +#pragma once + +// HuggingfaceChatModel: the llm::ChatModel seam over any Hugging Face chat +// endpoint that speaks the OpenAI-compatible chat wire: the Inference +// Providers router (default), hf-inference, or a TGI server. The mapping rules +// are in chat_wire.h. The default model must be given by the caller: the +// router has no default (model ids change; GET /v1/models lists them). + +#include +#include + +#include "vendors/huggingface/chat_wire.h" +#include "vendors/huggingface/platform.h" +#include "llm/chat_types.h" + +namespace molo::huggingface { + +class HuggingfaceChatModel : public llm::ChatModel { + public: + HuggingfaceChatModel(std::shared_ptr endpoint, std::string default_model); + // Router with HF_TOKEN from the environment. + static std::shared_ptr from_env(const std::string& default_model); + // Router over an explicit platform (tests). + static std::shared_ptr over_router(const Platform& platform, + const std::string& default_model); + + std::string vendor() const override { return "huggingface"; } + llm::ChatResponse generate(const llm::ChatRequest& request) override; + void stream(const llm::ChatRequest& request, + const std::function& on_chunk) override; + + // The wire body a neutral request maps to (tests, dry runs). + clients::body::Json to_body(const llm::ChatRequest& request, bool stream) const; + + private: + std::shared_ptr endpoint_; + std::string default_model_; +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/chat_wire.cpp b/src/vendors/huggingface/chat_wire.cpp new file mode 100644 index 0000000..44cd0ba --- /dev/null +++ b/src/vendors/huggingface/chat_wire.cpp @@ -0,0 +1,328 @@ +#include "vendors/huggingface/chat_wire.h" + +#include "vendors/huggingface/caps.h" +#include "llm/error.h" +#include "security/crypto/Base64.h" + +namespace molo::huggingface { + +using clients::body::Json; + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +constexpr std::size_t kMaxStop = 4; + +Json image_url_part(const llm::ImageData& img) { + std::string url = img.url; + if (url.empty()) { + if (img.bytes.empty()) bad("image has neither url nor bytes"); + if (img.mime.empty()) bad("image bytes need a mime type"); + url = "data:" + img.mime + ";base64," + security::crypto::Base64::encode(img.bytes); + } + Json u = Json::object(); + u.set("url", Json(url)); + Json p = Json::object(); + p.set("type", Json("image_url")); + p.set("image_url", u); + return p; +} + +Json text_part(const std::string& t) { + Json p = Json::object(); + p.set("type", Json("text")); + p.set("text", Json(t)); + return p; +} + +Json tool_call_json(const llm::ToolCall& c) { + Json fn = Json::object(); + fn.set("name", Json(c.name)); + fn.set("arguments", Json(c.arguments.is_null() ? std::string("{}") : c.arguments.dump())); + Json o = Json::object(); + o.set("id", Json(c.id)); + o.set("type", Json("function")); + o.set("function", fn); + return o; +} + +void append_message(Json& messages, const llm::Message& m) { + switch (m.role) { + case llm::Role::kSystem: { + Json o = Json::object(); + o.set("role", Json("system")); + o.set("content", Json(m.text())); + if (!m.name.empty()) o.set("name", Json(m.name)); + messages.push(o); + return; + } + case llm::Role::kUser: { + Json o = Json::object(); + o.set("role", Json("user")); + bool has_image = false; + for (const llm::ContentPart& p : m.parts) has_image = has_image || p.image.has_value(); + if (!has_image) { + o.set("content", Json(m.text())); + } else { + Json parts = Json::array(); + for (const llm::ContentPart& p : m.parts) { + if (!p.text.empty()) parts.push(text_part(p.text)); + if (p.image) parts.push(image_url_part(*p.image)); + } + o.set("content", parts); + } + if (!m.name.empty()) o.set("name", Json(m.name)); + messages.push(o); + return; + } + case llm::Role::kAssistant: { + Json o = Json::object(); + o.set("role", Json("assistant")); + Json calls = Json::array(); + for (const llm::ContentPart& p : m.parts) + if (p.tool_call) calls.push(tool_call_json(*p.tool_call)); + const std::string text = m.text(); + if (!text.empty() || calls.as_array().empty()) o.set("content", Json(text)); + if (!calls.as_array().empty()) o.set("tool_calls", calls); + messages.push(o); + return; + } + case llm::Role::kTool: { + bool any = false; + for (const llm::ContentPart& p : m.parts) { + if (!p.tool_result) continue; + Json o = Json::object(); + o.set("role", Json("tool")); + o.set("tool_call_id", Json(p.tool_result->call_id)); + o.set("content", Json(p.tool_result->content)); + messages.push(o); + any = true; + } + if (!any) bad("a tool message needs a tool result part"); + return; + } + } +} + +llm::Usage usage_of(const Json* u) { + llm::Usage out; + if (!u || !u->is_object()) return out; + auto num = [&](const Json& o, const char* k) { + const Json* v = o.find(k); + return v && v->is_number() ? static_cast(v->as_int()) : 0; + }; + out.input_tokens = num(*u, "prompt_tokens"); + out.output_tokens = num(*u, "completion_tokens"); + out.total_tokens = num(*u, "total_tokens"); + if (const Json* d = u->find("completion_tokens_details"); d && d->is_object()) + out.reasoning_tokens = num(*d, "reasoning_tokens"); + if (const Json* d = u->find("prompt_tokens_details"); d && d->is_object()) + out.cached_tokens = num(*d, "cached_tokens"); + return out; +} + +std::string str(const Json& o, const char* k) { + const Json* v = o.is_object() ? o.find(k) : nullptr; + return v && v->is_string() ? v->as_string() : std::string(); +} + +std::string reasoning_of(const Json& holder) { + std::string r = str(holder, "reasoning_content"); + if (r.empty()) r = str(holder, "reasoning"); + return r; +} + +Json parse_arguments(const std::string& raw) { + if (raw.empty()) return Json::object(); + try { + Json j = Json::parse(raw); + if (j.is_object()) return j; + } catch (const std::exception&) { + } + Json o = Json::object(); + o.set("_invalid_json", Json(raw)); + return o; +} + +} // namespace + +Json ChatWire::to_body(const llm::ChatRequest& r, const std::string& default_model, bool stream) { + const std::string model = r.model.empty() ? default_model : r.model; + if (model.empty() || model.size() > HfCaps::kMaxModelChars) bad("model is empty or too long"); + if (r.messages.empty()) bad("messages must not be empty"); + if (r.messages.size() > HfCaps::kMaxMessages) bad("too many messages"); + if (r.tools.size() > HfCaps::kMaxTools) bad("too many tools"); + if (r.stop.size() > kMaxStop) bad("at most 4 stop sequences"); + if (r.json_mode && !r.response_schema.is_null()) bad("json_mode and response_schema are exclusive"); + if (r.max_tokens && (*r.max_tokens < 1 || *r.max_tokens > HfCaps::kMaxTokensCeiling)) bad("max_tokens out of range"); + if (r.temperature && (*r.temperature < 0.0 || *r.temperature > 2.0)) bad("temperature must be 0..2"); + if (r.top_p && (*r.top_p <= 0.0 || *r.top_p > 1.0)) bad("top_p must be in (0, 1]"); + + Json body = Json::object(); + body.set("model", Json(model)); + Json messages = Json::array(); + if (!r.system.empty()) { + Json s = Json::object(); + s.set("role", Json("system")); + s.set("content", Json(r.system)); + messages.push(s); + } + for (const llm::Message& m : r.messages) append_message(messages, m); + body.set("messages", messages); + if (!r.tools.empty()) { + Json tools = Json::array(); + for (const llm::ToolSpec& t : r.tools) { + if (t.name.empty() || t.name.size() > 128) bad("tool name is empty or too long"); + Json fn = Json::object(); + fn.set("name", Json(t.name)); + if (!t.description.empty()) fn.set("description", Json(t.description)); + fn.set("parameters", t.parameters); + Json o = Json::object(); + o.set("type", Json("function")); + o.set("function", fn); + tools.push(o); + } + body.set("tools", tools); + } + if (!r.tool_choice.empty()) { + if (r.tool_choice == "auto" || r.tool_choice == "none" || r.tool_choice == "required") { + body.set("tool_choice", Json(r.tool_choice)); + } else { + Json n = Json::object(); + n.set("name", Json(r.tool_choice)); + Json c = Json::object(); + c.set("type", Json("function")); + c.set("function", n); + body.set("tool_choice", c); + } + } + if (r.max_tokens) body.set("max_tokens", Json(*r.max_tokens)); + if (r.temperature) body.set("temperature", Json(*r.temperature)); + if (r.top_p) body.set("top_p", Json(*r.top_p)); + if (!r.stop.empty()) { + Json s = Json::array(); + for (const std::string& x : r.stop) s.push(Json(x)); + body.set("stop", s); + } + if (r.json_mode) { + Json f = Json::object(); + f.set("type", Json("json_object")); + body.set("response_format", f); + } + if (!r.response_schema.is_null()) { + Json cfg = Json::object(); + cfg.set("name", Json("response")); + cfg.set("schema", r.response_schema); + cfg.set("strict", Json(true)); + Json f = Json::object(); + f.set("type", Json("json_schema")); + f.set("json_schema", cfg); + body.set("response_format", f); + } + if (!r.reasoning_effort.empty()) body.set("reasoning_effort", Json(r.reasoning_effort)); + if (r.extra.is_object()) + for (const auto& [k, v] : r.extra.as_object()) body.set(k, v); + if (stream) { + body.set("stream", Json(true)); + Json so = Json::object(); + so.set("include_usage", Json(true)); + body.set("stream_options", so); + } + return body; +} + +llm::ChatResponse ChatWire::to_response(const Json& doc) { + llm::ChatResponse out; + out.raw = doc; + out.id = str(doc, "id"); + out.model = str(doc, "model"); + out.message.role = llm::Role::kAssistant; + const Json* choices = doc.is_object() ? doc.find("choices") : nullptr; + if (!choices || !choices->is_array() || choices->as_array().empty()) + throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "reply has no choices"); + const Json& choice = choices->as_array().front(); + const Json* msg = choice.find("message"); + out.finish_reason = str(choice, "finish_reason"); + if (msg && msg->is_object()) { + std::string reasoning = reasoning_of(*msg); + if (!reasoning.empty()) { + llm::ContentPart r; + r.reasoning = reasoning; + out.message.parts.push_back(r); + } + std::string content = str(*msg, "content"); + const Json* calls = msg->find("tool_calls"); + const bool has_calls = calls && calls->is_array() && !calls->as_array().empty(); + if (!content.empty() || !has_calls) out.message.parts.push_back(llm::ContentPart::of_text(content)); + if (has_calls) { + for (const Json& c : calls->as_array()) { + llm::ToolCall tc; + tc.id = str(c, "id"); + const Json* fn = c.find("function"); + if (fn && fn->is_object()) { + tc.name = str(*fn, "name"); + const Json* args = fn->find("arguments"); + if (args && args->is_string()) tc.arguments = parse_arguments(args->as_string()); + else if (args && args->is_object()) tc.arguments = *args; // TGI may send an object + } + out.message.parts.push_back(llm::ContentPart::of_call(tc)); + } + } + } + out.usage = usage_of(doc.find("usage")); + return out; +} + +bool ChatWire::on_event(const Json& event, StreamState& state, + const std::function& sink) { + if (const Json* err = event.find("error")) { + std::string msg = err->is_string() ? err->as_string() : str(*err, "message"); + throw llm::ApiError("huggingface", 502, "STREAM_ERROR", msg.empty() ? "error event in stream" : msg); + } + const Json* choices = event.find("choices"); + if (choices && choices->is_array() && !choices->as_array().empty()) { + const Json& choice = choices->as_array().front(); + if (const Json* delta = choice.find("delta"); delta && delta->is_object()) { + llm::ChatChunk c; + c.text_delta = str(*delta, "content"); + c.reasoning_delta = reasoning_of(*delta); + if (!c.text_delta.empty() || !c.reasoning_delta.empty()) + if (!sink(c)) return false; + const Json* calls = delta->find("tool_calls"); + if (calls && calls->is_array()) { + for (const Json& f : calls->as_array()) { + int index = 0; + if (const Json* i = f.find("index"); i && i->is_number()) index = static_cast(i->as_int()); + std::string id = str(f, "id"); + if (!id.empty()) state.ids[index] = id; + const Json* fn = f.find("function"); + std::string name = fn ? str(*fn, "name") : ""; + if (!name.empty()) state.names[index] = name; + llm::ChatChunk t; + t.tool_call_id = state.ids[index]; + t.tool_call_name = state.names[index]; + if (fn) { + const Json* a = fn->find("arguments"); + if (a && a->is_string()) t.tool_args_delta = a->as_string(); + } + if (!sink(t)) return false; + } + } + } + std::string fin = str(choice, "finish_reason"); + if (!fin.empty()) { + llm::ChatChunk c; + c.finish_reason = fin; + if (!sink(c)) return false; + } + } + if (const Json* u = event.find("usage"); u && u->is_object()) { + llm::ChatChunk c; + c.usage = usage_of(u); + if (!sink(c)) return false; + } + return true; +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/chat_wire.h b/src/vendors/huggingface/chat_wire.h new file mode 100644 index 0000000..30e5064 --- /dev/null +++ b/src/vendors/huggingface/chat_wire.h @@ -0,0 +1,51 @@ +#pragma once + +// OpenAI-compatible chat wire used by every Hugging Face chat surface: the +// Inference Providers router, hf-inference and a TGI server share the shape +// (huggingface.js tasks/chat-completion/spec/input.json). ChatWire converts +// the vendor-neutral llm::ChatRequest into that body and the replies back. +// +// Decisions: +// - tool call arguments that are not valid JSON arrive as {"_invalid_json": raw} +// - reasoning text is read from "reasoning_content", else "reasoning" (providers differ) +// - response_schema maps to response_format json_schema (strict); json_mode to json_object; +// both together are refused +// - at most 4 stop sequences (the task spec's documented limit) +// - ChatRequest.extra is merged into the body; keys the call table does not list are refused + +#include +#include +#include + +#include "clients/body/json.h" +#include "vendors/huggingface/calls.h" +#include "llm/chat_types.h" +#include "llm/sse.h" + +namespace molo::huggingface { + +// Where a chat body goes: the router, hf-inference, or a TGI server. +class ChatEndpoint { + public: + virtual ~ChatEndpoint() = default; + virtual clients::body::Json create(const clients::body::Json& body) = 0; + virtual void stream(const clients::body::Json& body, const llm::SseDecoder::Sink& sink) = 0; +}; + +struct ChatWire { + static clients::body::Json to_body(const llm::ChatRequest& request, const std::string& default_model, + bool stream); + static llm::ChatResponse to_response(const clients::body::Json& doc); + + // Tracks tool call fragments across stream events (index -> id, name). + struct StreamState { + std::map ids; + std::map names; + }; + // Converts one SSE event into chunks; the sink returns false to stop. + // Returns false when the sink asked to stop. Throws ApiError for an error event. + static bool on_event(const clients::body::Json& event, StreamState& state, + const std::function& sink); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/community.cpp b/src/vendors/huggingface/community.cpp new file mode 100644 index 0000000..4498b71 --- /dev/null +++ b/src/vendors/huggingface/community.cpp @@ -0,0 +1,243 @@ +#include "vendors/huggingface/community.h" + +#include + +#include "vendors/huggingface/caps.h" +#include "llm/error.h" + +namespace molo::huggingface { + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +void add_if(Query& q, const char* name, const std::string& value) { + if (!value.empty()) q.add(name, value); +} + +PathArgs discussion_args(RepoType type, const RepoRef& repo, long num) { + if (num < 1 || num > 999999999) bad("discussion number must be 1..999999999"); + PathArgs a; + a.repo(repo).repo_type(type).set("num", std::to_string(num)); + return a; +} + +PathArgs repo_args(RepoType type, const RepoRef& repo) { + PathArgs a; + a.repo(repo).repo_type(type); + return a; +} + +void text_in(const std::string& what, const std::string& v, std::size_t min, std::size_t max) { + if (v.size() < min || v.size() > max) + bad(what + " must be " + std::to_string(min) + ".." + std::to_string(max) + " characters"); +} + +PathArgs collection_args(const std::string& owner, const std::string& slug) { + PathArgs a; + a.set("namespace", owner).set("slug", slug); + return a; +} + +const std::set& item_types() { + static const std::set s = {"paper", "collection", "space", "model", "dataset", "bucket"}; + return s; +} + +} // namespace + +Json Discussions::list(RepoType type, const RepoRef& repo, const DiscussionFilter& f) { + if (f.page < 0 || f.page > 100000) bad("page must be 0..100000"); + Query q; + if (f.page > 0) q.add("p", static_cast(f.page)); + add_if(q, "type", f.kind); + add_if(q, "status", f.status); + add_if(q, "author", f.author); + add_if(q, "search", f.search); + add_if(q, "sort", f.sort); + return call(Call::kDiscussionList, repo_args(type, repo), q); +} + +Json Discussions::get(RepoType type, const RepoRef& repo, long num) { + return call(Call::kDiscussionGet, discussion_args(type, repo, num)); +} + +Json Discussions::create(RepoType type, const RepoRef& repo, const std::string& title, + const std::string& description, bool pull_request) { + text_in("title", title, 3, 200); + text_in("description", description, 0, 65536); + Json body = Json::object(); + body.set("title", Json(title)); + body.set("description", Json(description)); + if (pull_request) body.set("pullRequest", Json(true)); + return call(Call::kDiscussionCreate, repo_args(type, repo), Query(), &body); +} + +Json Discussions::comment(RepoType type, const RepoRef& repo, long num, const std::string& text) { + text_in("comment", text, 1, 65536); + Json body = Json::object(); + body.set("comment", Json(text)); + return call(Call::kDiscussionComment, discussion_args(type, repo, num), Query(), &body); +} + +Json Discussions::change_status(RepoType type, const RepoRef& repo, long num, const std::string& status, + const std::string& comment) { + if (status != "open" && status != "closed") bad("status must be open or closed"); + text_in("comment", comment, 0, 65536); + Json body = Json::object(); + body.set("status", Json(status)); + if (!comment.empty()) body.set("comment", Json(comment)); + return call(Call::kDiscussionStatus, discussion_args(type, repo, num), Query(), &body); +} + +Json Discussions::rename(RepoType type, const RepoRef& repo, long num, const std::string& title) { + text_in("title", title, 3, 200); + Json body = Json::object(); + body.set("title", Json(title)); + return call(Call::kDiscussionTitle, discussion_args(type, repo, num), Query(), &body); +} + +Json Discussions::remove_discussion(RepoType type, const RepoRef& repo, long num) { + return call(Call::kDiscussionDelete, discussion_args(type, repo, num)); +} + +Json CollectionItem::to_json() const { + if (!item_types().count(type)) bad("collection item type must be paper, collection, space, model, dataset or bucket"); + text_in("collection item id", id, 1, HfCaps::kMaxSegmentChars); + Json o = Json::object(); + o.set("type", Json(type)); + o.set("id", Json(id)); + return o; +} + +Query CollectionListParams::to_query() const { + if (limit < 0 || limit > HfCaps::kMaxPageLimit) bad("limit out of range"); + Query query; + add_if(query, "owner", owner); + add_if(query, "q", q); + add_if(query, "sort", sort); + if (limit > 0) query.add("limit", static_cast(limit)); + if (!item.id.empty()) { + item.to_json(); + query.add("item", item.type + "/" + item.id); + } + return query; +} + +Page Collections::list(const CollectionListParams& params, std::size_t max_items, std::size_t max_pages) { + return walk(Call::kCollectionsList, PathArgs(), params.to_query(), max_items, max_pages); +} + +Json Collections::get(const std::string& owner, const std::string& slug) { + return call(Call::kCollectionGet, collection_args(owner, slug)); +} + +Json Collections::create(const std::string& owner, const std::string& title, const std::string& description, + bool is_private) { + if (!RequestPlan::valid_name(owner)) bad("bad owner"); + text_in("title", title, 1, 60); + text_in("description", description, 0, 150); + Json body = Json::object(); + body.set("title", Json(title)); + body.set("namespace", Json(owner)); + if (!description.empty()) body.set("description", Json(description)); + if (is_private) body.set("private", Json(true)); + return call(Call::kCollectionCreate, PathArgs(), Query(), &body); +} + +Json Collections::update(const std::string& owner, const std::string& slug, const Json& fields) { + if (!fields.is_object() || fields.as_object().empty()) bad("fields must be a non-empty object"); + return call(Call::kCollectionUpdate, collection_args(owner, slug), Query(), &fields); +} + +Json Collections::add_item(const std::string& owner, const std::string& slug, const CollectionItem& item, + const std::string& note) { + text_in("note", note, 0, 500); + Json body = Json::object(); + body.set("item", item.to_json()); + if (!note.empty()) body.set("note", Json(note)); + return call(Call::kCollectionAddItem, collection_args(owner, slug), Query(), &body); +} + +Json Collections::remove_collection(const std::string& owner, const std::string& slug) { + return call(Call::kCollectionDelete, collection_args(owner, slug)); +} + +Json WebhookSpec::to_json() const { + static const std::set kinds = {"dataset", "model", "space", "bucket", "kernel", + "container", "user", "org"}; + if (url.size() < 8 || url.size() > HfCaps::kMaxServerUrlChars || url.rfind("http", 0) != 0) + bad("webhook url must be an http(s) URL"); + if (watched.empty() || watched.size() > 64) bad("watched must hold 1..64 entries"); + if (domains.empty() || domains.size() > HfCaps::kMaxWebhookDomains) bad("domains must hold 1.. entries"); + Json w = Json::array(); + for (const Watched& x : watched) { + if (!kinds.count(x.type)) bad("watched type must be dataset, model, space, bucket, kernel, container, user or org"); + if (x.name.empty() || x.name.size() > 256) bad("watched name is empty or too long"); + Json e = Json::object(); + e.set("type", Json(x.type)); + e.set("name", Json(x.name)); + w.push(e); + } + Json d = Json::array(); + for (const std::string& dom : domains) { + if (dom != "repo" && dom != "discussion") bad("domain must be repo or discussion"); + d.push(Json(dom)); + } + Json o = Json::object(); + o.set("url", Json(url)); + o.set("watched", w); + o.set("domains", d); + if (!secret.empty()) { + for (unsigned char c : secret) + if (c < 0x20 || c > 0x7f) bad("secret must be printable ASCII"); + o.set("secret", Json(secret)); + } + return o; +} + +Json Webhooks::list() { return call(Call::kWebhooksList); } + +Json Webhooks::get(const std::string& id) { + PathArgs a; + a.set("webhookId", id); + return call(Call::kWebhookGet, a); +} + +Json Webhooks::create(const WebhookSpec& spec) { + Json body = spec.to_json(); + return call(Call::kWebhookCreate, PathArgs(), Query(), &body); +} + +Json Webhooks::update(const std::string& id, const WebhookSpec& spec) { + Json body = spec.to_json(); + PathArgs a; + a.set("webhookId", id); + return call(Call::kWebhookUpdate, a, Query(), &body); +} + +Json Webhooks::enable(const std::string& id) { + PathArgs a; + a.set("webhookId", id).set("action", "enable"); + return call(Call::kWebhookToggle, a); +} + +Json Webhooks::disable(const std::string& id) { + PathArgs a; + a.set("webhookId", id).set("action", "disable"); + return call(Call::kWebhookToggle, a); +} + +Json Webhooks::replay(const std::string& id, const std::string& log_id) { + PathArgs a; + a.set("webhookId", id).set("logId", log_id); + return call(Call::kWebhookReplay, a); +} + +Json Webhooks::remove_webhook(const std::string& id) { + PathArgs a; + a.set("webhookId", id); + return call(Call::kWebhookDelete, a); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/community.h b/src/vendors/huggingface/community.h new file mode 100644 index 0000000..41fec38 --- /dev/null +++ b/src/vendors/huggingface/community.h @@ -0,0 +1,94 @@ +#pragma once + +// Discussions (and pull request threads), Collections and Webhooks on the Hub. +// Reads are public where the Hub makes them public; every write needs a token +// and is an explicit method. Deletes have no defaults and are named remove_*. + +#include +#include + +#include "vendors/huggingface/hf_service.h" + +namespace molo::huggingface { + +struct DiscussionFilter { + int page = 0; // p + std::string kind; // all, discussion, pull_request + std::string status; // all, open, closed + std::string author; + std::string search; + std::string sort; // recently-created, trending, reactions +}; + +class Discussions : public HfService { + public: + explicit Discussions(Access access) : HfService(std::move(access), Surface::kHub) {} + Json list(RepoType type, const RepoRef& repo, const DiscussionFilter& filter = {}); + Json get(RepoType type, const RepoRef& repo, long num); + + // Writes (token required). + Json create(RepoType type, const RepoRef& repo, const std::string& title, const std::string& description, + bool pull_request); + Json comment(RepoType type, const RepoRef& repo, long num, const std::string& text); + Json change_status(RepoType type, const RepoRef& repo, long num, const std::string& status, + const std::string& comment = ""); + Json rename(RepoType type, const RepoRef& repo, long num, const std::string& title); + Json remove_discussion(RepoType type, const RepoRef& repo, long num); +}; + +struct CollectionItem { + std::string type; // paper, collection, space, model, dataset, bucket + std::string id; + Json to_json() const; // validates +}; + +struct CollectionListParams { + std::string owner; + std::string q; + std::string sort; + int limit = 0; + CollectionItem item; // optional filter: collections containing this item (id empty: none) + Query to_query() const; +}; + +class Collections : public HfService { + public: + explicit Collections(Access access) : HfService(std::move(access), Surface::kHub) {} + Page list(const CollectionListParams& params, std::size_t max_items = 100, std::size_t max_pages = 5); + // slug is "name-<24 hex id>" exactly as the Hub prints it after the owner. + Json get(const std::string& owner, const std::string& slug); + + Json create(const std::string& owner, const std::string& title, const std::string& description, + bool is_private); + Json update(const std::string& owner, const std::string& slug, const Json& fields); + Json add_item(const std::string& owner, const std::string& slug, const CollectionItem& item, + const std::string& note = ""); + Json remove_collection(const std::string& owner, const std::string& slug); +}; + +struct WebhookSpec { + struct Watched { + std::string type; // dataset, model, space, bucket, kernel, container, user, org + std::string name; + }; + std::string url; + std::vector watched; + std::vector domains; // repo, discussion + std::string secret; // printable ASCII, optional + Json to_json() const; // validates +}; + +class Webhooks : public HfService { + public: + explicit Webhooks(Access access) : HfService(std::move(access), Surface::kHub) {} + Json list(); + Json get(const std::string& id); + Json create(const WebhookSpec& spec); + Json update(const std::string& id, const WebhookSpec& spec); + Json enable(const std::string& id); + Json disable(const std::string& id); + Json replay(const std::string& id, const std::string& log_id); + Json remove_webhook(const std::string& id); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/endpoints.cpp b/src/vendors/huggingface/endpoints.cpp new file mode 100644 index 0000000..63d0a6c --- /dev/null +++ b/src/vendors/huggingface/endpoints.cpp @@ -0,0 +1,235 @@ +#include "vendors/huggingface/endpoints.h" + +#include + +#include "llm/error.h" + +namespace molo::huggingface { + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +struct MetricRow { + MetricKind kind; + const char* name; + Call call; +}; + +const MetricRow kMetrics[] = { + {MetricKind::kAll, "all", Call::kIeGetEndpointMetrics}, + {MetricKind::kHardwareCpu, "hardware-cpu", Call::kIeGetHardwareCpu}, + {MetricKind::kHardwareGpu, "hardware-gpu", Call::kIeGetHardwareGpu}, + {MetricKind::kItl, "itl", Call::kIeGetInferenceItl}, + {MetricKind::kKvCache, "kv-cache", Call::kIeGetInferenceKvCache}, + {MetricKind::kPrefixCache, "prefix-cache", Call::kIeGetInferencePrefixCache}, + {MetricKind::kRequests, "requests", Call::kIeGetInferenceRequests}, + {MetricKind::kTtft, "ttft", Call::kIeGetInferenceTtft}, + {MetricKind::kPendingRequest, "pending-request", Call::kIeGetPendingRequest}, + {MetricKind::kReplicaIds, "replica-ids", Call::kIeGetReplicaIds}, + {MetricKind::kReplicaRunning, "replica-running", Call::kIeGetReplicaRunning}, + {MetricKind::kReplicaStatus, "replica-status", Call::kIeGetReplicaStatus}, + {MetricKind::kResponseElapsed, "response-elapsed", Call::kIeGetResponseElapsed}, + {MetricKind::kResponseStatusCode, "response-status-code", Call::kIeGetResponseStatusCode}, + {MetricKind::kResponseStatusCodeGrouped, "response-status-code-grouped", Call::kIeGetStatusCodeGrouped}, +}; + +PathArgs ns_args(const std::string& ns) { + PathArgs a; + a.set("namespace", ns); + return a; +} + +PathArgs ep_args(const std::string& ns, const std::string& name) { + PathArgs a = ns_args(ns); + a.set("name", name); + return a; +} + +void check_limit(int limit, int max) { + if (limit < 0 || limit > max) bad("limit must be 0.." + std::to_string(max)); +} + +bool rfc3339_like(const std::string& s) { + if (s.size() < 20 || s.size() > 40) return false; + return std::isdigit(static_cast(s[0])) && s[4] == '-' && s[7] == '-' && s[10] == 'T'; +} + +} // namespace + +const char* metric_kind_name(MetricKind kind) { + for (const MetricRow& r : kMetrics) + if (r.kind == kind) return r.name; + return "all"; +} + +bool parse_metric_kind(const std::string& text, MetricKind& out) { + for (const MetricRow& r : kMetrics) + if (text == r.name) { + out = r.kind; + return true; + } + return false; +} + +void Endpoints::validate_name(const std::string& name) { + if (name.empty() || name.size() > 32) bad("endpoint name must be 1..32 characters"); + for (char c : name) + if (!(std::islower(static_cast(c)) || std::isdigit(static_cast(c)) || c == '-')) + bad("endpoint name: lowercase letters, digits and - only"); +} + +void Endpoints::validate_spec(const Json& spec, bool creating) { + if (!spec.is_object()) bad("endpoint spec must be a JSON object"); + if (creating) { + for (const char* k : {"name", "type", "provider", "compute", "model"}) + if (!spec.find(k)) bad(std::string("endpoint spec needs ") + k); + const Json* n = spec.find("name"); + if (!n->is_string()) bad("name must be a string"); + validate_name(n->as_string()); + } + if (const Json* t = spec.find("type")) { + if (!t->is_string() || (t->as_string() != "public" && t->as_string() != "authenticated" && + t->as_string() != "private")) + bad("type must be public, authenticated or private"); + } +} + +Json Endpoints::list(const std::string& ns, const EndpointListParams& p) { + check_limit(p.limit, 100); + Query q; + if (!p.search.empty()) q.add("search", p.search); + if (!p.tags.empty()) q.add("tags", p.tags); + if (!p.type.empty()) q.add("type", p.type); + if (!p.task.empty()) q.add("task", p.task); + if (!p.state.empty()) q.add("state", p.state); + if (!p.cursor.empty()) q.add("cursor", p.cursor); + if (p.limit > 0) q.add("limit", static_cast(p.limit)); + return call(Call::kIeListEndpoint, ns_args(ns), q); +} + +Json Endpoints::get(const std::string& ns, const std::string& name) { + return call(Call::kIeGetEndpoint, ep_args(ns, name)); +} + +Json Endpoints::replicas(const std::string& ns, const std::string& name) { + return call(Call::kIeGetEndpointReplica, ep_args(ns, name)); +} + +std::string Endpoints::logs(const std::string& ns, const std::string& name, int tail, const std::string& replica) { + check_limit(tail, 1000); + Query q; + if (tail > 0) q.add("tail", static_cast(tail)); + if (!replica.empty()) q.add("replica", replica); + return call_raw(Call::kIeGetEndpointLogs, ep_args(ns, name), q).body; +} + +Json Endpoints::logs_v3(const std::string& ns, const std::string& name, int limit) { + check_limit(limit, 1000); + Query q; + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kIeGetEndpointLogsV3, ep_args(ns, name), q); +} + +Json Endpoints::metrics(MetricKind kind, const std::string& ns, const std::string& name, + const std::string& start, const std::string& stop, const std::string& replica_id) { + if (!rfc3339_like(start) || !rfc3339_like(stop)) bad("start and stop must be RFC 3339 (2021-01-01T00:00:00Z)"); + Json body = Json::object(); + body.set("start", Json(start)); + body.set("stop", Json(stop)); + if (!replica_id.empty()) body.set("replicaId", Json(replica_id)); + for (const MetricRow& r : kMetrics) + if (r.kind == kind) return call(r.call, ep_args(ns, name), Query(), &body); + bad("unknown metric kind"); +} + +std::string Endpoints::open_metrics(const std::string& ns, const std::string& name) { + return call_raw(Call::kIeGetEndpointOpenMetrics, ep_args(ns, name)).body; +} + +Json Endpoints::search(const std::string& ns, const std::string& text, int limit) { + check_limit(limit, 100); + Query q; + if (!text.empty()) q.add("search", text); + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kIeSearchEndpoint, ns_args(ns), q); +} + +Json Endpoints::state_counts(const std::string& ns) { return call(Call::kIeCountEndpointStates, ns_args(ns)); } + +Json Endpoints::tags(const std::string& ns, const std::string& prefix, int limit) { + check_limit(limit, 100); + Query q; + if (!prefix.empty()) q.add("prefix", prefix); + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kIeListEndpointTags, ns_args(ns), q); +} + +Json Endpoints::vendors() { return call(Call::kIeListVendors); } +Json Endpoints::vendors_for(const std::string& ns) { return call(Call::kIeListVendorsNamespace, ns_args(ns)); } +Json Endpoints::quotas(const std::string& ns) { return call(Call::kIeListVendorsQuotasNamespace, ns_args(ns)); } + +Json Endpoints::audit_logs(const std::string& ns, int limit) { + check_limit(limit, 100); + Query q; + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kIeGetAuditLogs, ns_args(ns), q); +} + +Json Endpoints::cidr_lists(const std::string& ns) { return call(Call::kIeListCidrsList, ns_args(ns)); } +Json Endpoints::domains(const std::string& ns) { return call(Call::kIeListDomains, ns_args(ns)); } +Json Endpoints::private_services(const std::string& ns) { return call(Call::kIeListPrivateServices, ns_args(ns)); } + +Json Endpoints::create(const std::string& ns, const Json& spec) { + validate_spec(spec, true); + return call(Call::kIeCreateEndpoint, ns_args(ns), Query(), &spec); +} + +Json Endpoints::update(const std::string& ns, const std::string& name, const Json& spec) { + validate_spec(spec, false); + return call(Call::kIeUpdateEndpoint, ep_args(ns, name), Query(), &spec); +} + +Json Endpoints::pause(const std::string& ns, const std::string& name) { + return call(Call::kIePauseEndpoint, ep_args(ns, name)); +} +Json Endpoints::resume(const std::string& ns, const std::string& name) { + return call(Call::kIeResumeEndpoint, ep_args(ns, name)); +} +Json Endpoints::scale_to_zero(const std::string& ns, const std::string& name) { + return call(Call::kIeScaleToZeroEndpoint, ep_args(ns, name)); +} +Json Endpoints::remove_endpoint(const std::string& ns, const std::string& name) { + return call(Call::kIeDeleteEndpoint, ep_args(ns, name)); +} + +Json Endpoints::create_cidr_list(const std::string& ns, const Json& spec) { + return call(Call::kIeCreateCidrList, ns_args(ns), Query(), &spec); +} + +Json Endpoints::remove_cidr_list(const std::string& ns, const std::string& id) { + PathArgs a = ns_args(ns); + a.set("cidr_id", id); + return call(Call::kIeDeleteCidrList, a); +} + +Json Endpoints::create_domain(const std::string& ns, const std::string& domain) { + if (domain.empty() || domain.size() > 253) bad("domain is empty or too long"); + Json body = Json::object(); + body.set("domain", Json(domain)); + return call(Call::kIeCreateDomain, ns_args(ns), Query(), &body); +} + +Json Endpoints::remove_domain(const std::string& ns, const std::string& domain) { + PathArgs a = ns_args(ns); + a.set("domain", domain); + return call(Call::kIeDeleteDomain, a); +} + +Json Endpoints::remove_private_service(const std::string& ns, const std::string& id) { + PathArgs a = ns_args(ns); + a.set("service-id", id); + return call(Call::kIeDeletePrivateService, a); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/endpoints.h b/src/vendors/huggingface/endpoints.h new file mode 100644 index 0000000..bcc6d5b --- /dev/null +++ b/src/vendors/huggingface/endpoints.h @@ -0,0 +1,79 @@ +#pragma once + +// Inference Endpoints management API (https://api.endpoints.huggingface.cloud, +// OpenAPI 2.0.0): endpoints (list, get, create, update, pause, resume, scale +// to zero, delete), logs, metrics, replicas, providers and quotas, search, +// network (CIDR lists, domains, private services), audit logs. +// Creating an endpoint costs money; deleting, pausing and scaling to zero +// disrupt service. All of those are explicit methods and the demo only runs +// them with "confirm":true. The two GET SSE streams are listed in the call +// table as unsupported (ApiClient.stream is POST only). + +#include + +#include "vendors/huggingface/hf_service.h" + +namespace molo::huggingface { + +enum class MetricKind { + kAll, kHardwareCpu, kHardwareGpu, kItl, kKvCache, kPrefixCache, kRequests, kTtft, + kPendingRequest, kReplicaIds, kReplicaRunning, kReplicaStatus, kResponseElapsed, + kResponseStatusCode, kResponseStatusCodeGrouped, +}; + +const char* metric_kind_name(MetricKind kind); +bool parse_metric_kind(const std::string& text, MetricKind& out); + +struct EndpointListParams { + std::string search; + std::string tags; + std::string type; // public, authenticated, private + std::string task; + std::string state; + std::string cursor; + int limit = 0; // 1..100 +}; + +class Endpoints : public HfService { + public: + explicit Endpoints(Access access) : HfService(std::move(access), Surface::kEndpoints) {} + static Endpoints from_platform(const Platform& p) { return Endpoints(p.endpoints); } + + // Reads. + Json list(const std::string& ns, const EndpointListParams& params = {}); + Json get(const std::string& ns, const std::string& name); + Json replicas(const std::string& ns, const std::string& name); + std::string logs(const std::string& ns, const std::string& name, int tail = 100, const std::string& replica = ""); + Json logs_v3(const std::string& ns, const std::string& name, int limit = 100); + Json metrics(MetricKind kind, const std::string& ns, const std::string& name, const std::string& start, + const std::string& stop, const std::string& replica_id = ""); + std::string open_metrics(const std::string& ns, const std::string& name); + Json search(const std::string& ns, const std::string& text, int limit = 20); + Json state_counts(const std::string& ns); + Json tags(const std::string& ns, const std::string& prefix = "", int limit = 50); + Json vendors(); // public, no token + Json vendors_for(const std::string& ns); + Json quotas(const std::string& ns); + Json audit_logs(const std::string& ns, int limit = 50); + Json cidr_lists(const std::string& ns); + Json domains(const std::string& ns); + Json private_services(const std::string& ns); + + // Writes. create() costs money while the endpoint runs. + Json create(const std::string& ns, const Json& spec); + Json update(const std::string& ns, const std::string& name, const Json& spec); + Json pause(const std::string& ns, const std::string& name); + Json resume(const std::string& ns, const std::string& name); + Json scale_to_zero(const std::string& ns, const std::string& name); + Json remove_endpoint(const std::string& ns, const std::string& name); + Json create_cidr_list(const std::string& ns, const Json& spec); + Json remove_cidr_list(const std::string& ns, const std::string& id); + Json create_domain(const std::string& ns, const std::string& domain); + Json remove_domain(const std::string& ns, const std::string& domain); + Json remove_private_service(const std::string& ns, const std::string& id); + + static void validate_name(const std::string& name); // lowercase alnum and '-', 1..32 + static void validate_spec(const Json& spec, bool creating); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/hf_service.cpp b/src/vendors/huggingface/hf_service.cpp new file mode 100644 index 0000000..e544add --- /dev/null +++ b/src/vendors/huggingface/hf_service.cpp @@ -0,0 +1,136 @@ +#include "vendors/huggingface/hf_service.h" + +#include + +#include "clients/url.h" +#include "vendors/huggingface/caps.h" +#include "llm/caps.h" +#include "llm/error.h" + +namespace molo::huggingface { + +HfService::HfService(Access access, Surface surface) + : llm::Service(access.client), access_(std::move(access)), surface_(surface) {} + +const CallSpec& HfService::checked(Call call, bool) const { + const CallSpec& spec = Calls::spec_of(call); + if (spec.surface != surface_) + throw llm::ApiError::invalid("huggingface", std::string("call ") + spec.path + " belongs to the " + + surface_name(spec.surface) + " surface"); + if (!std::strcmp(spec.auth, "token") && !access_.has_token) + throw llm::NotConfiguredError("huggingface", Platform::kKeyEnv); + if (access_.budget) access_.budget->spend(); + return spec; +} + +Json HfService::call(Call c, const PathArgs& args, const Query& query, const Json* body, + const llm::Headers& extra) { + checked(c); + Plan plan = RequestPlan::make(c, args, query, body); + return client().request(plan.method, plan.target, body, extra); +} + +llm::RawResponse HfService::call_raw(Call c, const PathArgs& args, const Query& query, const Json* body, + const llm::Headers& extra) { + checked(c); + Plan plan = RequestPlan::make(c, args, query, body); + std::string text; + if (body) text = body->dump(); + return client().request_raw(plan.method, plan.target, body ? &text : nullptr, + body ? "application/json" : "", extra); +} + +llm::RawResponse HfService::call_bytes(Call c, const PathArgs& args, const std::string& body, + const std::string& content_type, const llm::Headers& extra) { + const CallSpec& spec = checked(c); + if (!spec.has_body()) throw llm::ApiError::invalid("huggingface", "this call takes no body"); + if (body.size() > HfCaps::kMaxInferenceInputBytes) + throw llm::ApiError::invalid("huggingface", "input is over the byte cap"); + Plan plan = RequestPlan::make(c, args, Query()); + return client().request_raw(plan.method, plan.target, &body, content_type, extra); +} + +std::string next_cursor_of(const std::string& link, const std::string& expected_host) { + // ; rel="next" + std::size_t pos = 0; + while (pos < link.size()) { + std::size_t lt = link.find('<', pos); + if (lt == std::string::npos) return ""; + std::size_t gt = link.find('>', lt); + if (gt == std::string::npos) return ""; + std::size_t next_entry = link.find('<', gt); + std::string params = link.substr(gt + 1, (next_entry == std::string::npos ? link.size() : next_entry) - gt - 1); + if (params.find("rel=\"next\"") != std::string::npos) { + clients::Url url = clients::Url::parse(link.substr(lt + 1, gt - lt - 1)); + if (url.host() != expected_host) + throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "Link header points at another host"); + auto cur = url.query_value("cursor"); + return cur ? *cur : ""; + } + if (next_entry == std::string::npos) break; + pos = next_entry; + } + return ""; +} + +Page HfService::walk(Call c, const PathArgs& args, const Query& query, std::size_t max_items, + std::size_t max_pages) { + const CallSpec& spec = Calls::spec_of(c); + if (std::strcmp(spec.http_method, "GET") != 0) + throw llm::ApiError::invalid("huggingface", "only GET lists can be walked"); + if (max_items == 0 || max_items > HfCaps::kMaxItems) max_items = HfCaps::kMaxItems; + if (max_pages == 0 || max_pages > HfCaps::kMaxPages) max_pages = HfCaps::kMaxPages; + clients::Url base = clients::Url::parse(client().config().base_url); + Page page; + Query q = query; + while (true) { + checked(c); + Plan plan = RequestPlan::make(c, args, q); + llm::RawResponse raw = client().request_raw(plan.method, plan.target); + Json doc; + try { + doc = Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", e.what()); + } + if (!doc.is_array()) throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "list reply is not an array"); + ++page.pages; + std::string cursor = next_cursor_of(raw.headers.get_or("link", ""), base.host()); + for (const Json& item : doc.as_array()) { + if (page.items.as_array().size() >= max_items) { + page.truncated = true; + page.truncated_by = "items"; + page.next_cursor = ""; + return page; + } + page.items.push(item); + } + if (cursor.empty()) return page; + if (page.pages >= max_pages) { + page.truncated = true; + page.truncated_by = "pages"; + page.next_cursor = cursor; + return page; + } + if (!Calls::in_list(spec.query_params, "cursor")) + throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "server paged a call that has no cursor"); + q.replace("cursor", cursor); + } +} + +void HfService::stream(Call c, const PathArgs& args, const Json& body, const llm::SseDecoder::Sink& sink, + const llm::Headers& extra) { + const CallSpec& spec = checked(c); + if (std::strcmp(spec.http_method, "POST") != 0) + throw llm::ApiError::invalid("huggingface", "streaming is POST only (llm gap)"); + Plan plan = RequestPlan::make(c, args, Query(), &body); + std::size_t events = 0; + client().stream(plan.target, body, + [&](const llm::SseEvent& e) { + if (++events > HfCaps::kMaxStreamEvents) return false; + return sink(e); + }, + extra); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/hf_service.h b/src/vendors/huggingface/hf_service.h new file mode 100644 index 0000000..21baccc --- /dev/null +++ b/src/vendors/huggingface/hf_service.h @@ -0,0 +1,61 @@ +#pragma once + +// HfService: base of every service in this package. One method per kind of +// call (JSON, raw bytes, JSON body to raw reply, page walk, SSE), all driven by +// the call table through RequestPlan. Enforces: the call belongs to this +// service's surface, a token exists when the row demands one, the shared +// request budget, destructive/write rows only through the explicit methods +// that pass `allow_write`. + +#include +#include + +#include "clients/body/json.h" +#include "vendors/huggingface/calls.h" +#include "vendors/huggingface/platform.h" +#include "vendors/huggingface/request_plan.h" +#include "llm/api_client.h" + +namespace molo::huggingface { + +using Json = clients::body::Json; + +// One walked list: items plus the loud truncation report. +struct Page { + Json items = Json::array(); + bool truncated = false; // stopped by a cap while the server had more + std::size_t pages = 0; + std::string next_cursor; // set when truncated: resume point + std::string truncated_by; // "items", "pages" or "" +}; + +class HfService : public llm::Service { + public: + HfService(Access access, Surface surface); + Surface surface() const { return surface_; } + bool has_token() const { return access_.has_token; } + + protected: + Json call(Call call, const PathArgs& args = {}, const Query& query = {}, const Json* body = nullptr, + const llm::Headers& extra = {}); + llm::RawResponse call_raw(Call call, const PathArgs& args = {}, const Query& query = {}, + const Json* body = nullptr, const llm::Headers& extra = {}); + // Raw request body (binary inference inputs). + llm::RawResponse call_bytes(Call call, const PathArgs& args, const std::string& body, + const std::string& content_type, const llm::Headers& extra = {}); + // GET list that follows Link: rel="next" cursors up to the caps. + Page walk(Call call, const PathArgs& args, const Query& query, std::size_t max_items, + std::size_t max_pages); + void stream(Call call, const PathArgs& args, const Json& body, const llm::SseDecoder::Sink& sink, + const llm::Headers& extra = {}); + + const CallSpec& checked(Call call, bool body_expected = false) const; + + Access access_; + Surface surface_; +}; + +// The cursor in a Link header's rel="next" target, or empty. +std::string next_cursor_of(const std::string& link_header, const std::string& expected_host); + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/hub.cpp b/src/vendors/huggingface/hub.cpp new file mode 100644 index 0000000..fb3936e --- /dev/null +++ b/src/vendors/huggingface/hub.cpp @@ -0,0 +1,244 @@ +#include "vendors/huggingface/hub.h" + +#include "vendors/huggingface/caps.h" +#include "llm/error.h" + +namespace molo::huggingface { + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +void check_limit(int limit) { + if (limit < 0 || limit > HfCaps::kMaxPageLimit) bad("limit must be 0.." + std::to_string(HfCaps::kMaxPageLimit)); +} + +void add_if(Query& q, const char* name, const std::string& value) { + if (!value.empty()) q.add(name, value); +} + +Call info_call(RepoType t, bool revision) { + switch (t) { + case RepoType::kModel: return revision ? Call::kModelRevision : Call::kModelInfo; + case RepoType::kDataset: return revision ? Call::kDatasetRevision : Call::kDatasetInfo; + case RepoType::kSpace: return revision ? Call::kSpaceRevision : Call::kSpaceInfo; + } + return Call::kModelInfo; +} + +PathArgs repo_args(RepoType type, const RepoRef& ref) { + PathArgs a; + a.repo(ref).repo_type(type); + return a; +} + +} // namespace + +Query RepoListParams::to_query() const { + check_limit(limit); + Query q; + add_if(q, "search", search); + add_if(q, "author", author); + add_if(q, "sort", sort); + add_if(q, "direction", direction); + add_if(q, "pipeline_tag", pipeline_tag); + add_if(q, "inference_provider", inference_provider); + for (const std::string& f : filter) q.add("filter", f); + for (const std::string& e : expand) q.add("expand", e); + if (limit > 0) q.add("limit", static_cast(limit)); + if (full) q.flag("full", true); + return q; +} + +Call Hub::list_call(RepoType t) { + switch (t) { + case RepoType::kModel: return Call::kModelsList; + case RepoType::kDataset: return Call::kDatasetsList; + case RepoType::kSpace: return Call::kSpacesList; + } + return Call::kModelsList; +} + +Json Hub::whoami() { return call(Call::kWhoami); } + +Page Hub::list_repos(RepoType type, const RepoListParams& params, std::size_t max_items, + std::size_t max_pages) { + return walk(list_call(type), PathArgs(), params.to_query(), max_items, max_pages); +} + +Json Hub::repo_info(RepoType type, const RepoRef& ref, const std::string& revision, + const std::vector& expand) { + PathArgs a; + a.set("namespace", ref.owner()).set("repo", ref.name()); + if (!revision.empty()) a.set("rev", revision); + Query q; + for (const std::string& e : expand) q.add("expand", e); + return call(info_call(type, !revision.empty()), a, q); +} + +Json Hub::inference_provider_mapping(const RepoRef& model) { + return repo_info(RepoType::kModel, model, "", {"inferenceProviderMapping"}); +} + +Json Hub::tree(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path, + bool recursive, int limit) { + check_limit(limit); + PathArgs a = repo_args(type, ref); + a.set("rev", rev).set("path", path); + Query q; + if (recursive) q.flag("recursive", true); + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kRepoTree, a, q); +} + +Page Hub::tree_all(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path, + bool recursive, std::size_t max_items, std::size_t max_pages) { + PathArgs a = repo_args(type, ref); + a.set("rev", rev).set("path", path); + Query q; + if (recursive) q.flag("recursive", true); + return walk(Call::kRepoTree, a, q, max_items, max_pages); +} + +Json Hub::refs(RepoType type, const RepoRef& ref, bool include_prs) { + Query q; + if (include_prs) q.flag("include_prs", true); + return call(Call::kRepoRefs, repo_args(type, ref), q); +} + +Json Hub::commits(RepoType type, const RepoRef& ref, const std::string& rev, int limit) { + check_limit(limit); + PathArgs a = repo_args(type, ref); + a.set("rev", rev); + Query q; + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kRepoCommits, a, q); +} + +Json Hub::compare(RepoType type, const RepoRef& ref, const std::string& cmp) { + PathArgs a = repo_args(type, ref); + a.set("compare", cmp); + return call(Call::kRepoCompare, a); +} + +Json Hub::paths_info(RepoType type, const RepoRef& ref, const std::string& rev, + const std::vector& paths) { + if (paths.empty() || paths.size() > HfCaps::kMaxPathsInfo) + bad("paths must hold 1.." + std::to_string(HfCaps::kMaxPathsInfo) + " entries"); + Json arr = Json::array(); + for (const std::string& p : paths) { + RequestPlan::encode_file_path(p); // validates + arr.push(Json(p)); + } + Json body = Json::object(); + body.set("paths", arr); + PathArgs a = repo_args(type, ref); + a.set("rev", rev); + return call(Call::kRepoPathsInfo, a, Query(), &body); +} + +Json Hub::likers(RepoType type, const RepoRef& ref) { return call(Call::kRepoLikers, repo_args(type, ref)); } + +Json Hub::safetensors_metadata(const RepoRef& model, const std::string& path) { + PathArgs a; + a.repo(model).set("path", path); + return call(Call::kModelSafetensors, a); +} + +Download Hub::download(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path) { + PathArgs a = repo_args(type, ref); + a.set("rev", rev).set("path", path); + llm::RawResponse raw = call_raw(Call::kFileResolve, a); + Download d; + d.bytes = std::move(raw.body); + d.content_type = raw.content_type; + d.commit = raw.headers.get_or("x-repo-commit", ""); + d.etag = raw.headers.get_or("etag", ""); + return d; +} + +Json Hub::quicksearch(const std::string& q, int limit, const std::string& type) { + check_limit(limit); + if (q.empty()) bad("q must not be empty"); + Query query; + query.add("q", q); + if (limit > 0) query.add("limit", static_cast(limit)); + add_if(query, "type", type); + return call(Call::kQuicksearch, PathArgs(), query); +} + +Json Hub::full_text_search(const std::string& q, int limit, const std::string& type) { + check_limit(limit); + if (q.empty()) bad("q must not be empty"); + Query query; + query.add("q", q); + if (limit > 0) query.add("limit", static_cast(limit)); + add_if(query, "type", type); + return call(Call::kFullTextSearch, PathArgs(), query); +} + +Json Hub::trending(const std::string& type, int limit) { + check_limit(limit); + Query q; + add_if(q, "type", type); + if (limit > 0) q.add("limit", static_cast(limit)); + return call(Call::kTrending, PathArgs(), q); +} + +Json Hub::tags_by_type(RepoType type, const std::string& tag_type) { + Query q; + add_if(q, "type", tag_type); + if (type == RepoType::kModel) return call(Call::kModelsTagsByType, PathArgs(), q); + if (type == RepoType::kDataset) return call(Call::kDatasetsTagsByType, PathArgs(), q); + bad("tags exist for models and datasets only"); +} + +Json Hub::tasks() { return call(Call::kTasks); } +Json Hub::spaces_hardware() { return call(Call::kSpacesHardware); } +Json Hub::spaces_featured() { return call(Call::kSpacesFeatured); } + +Json Hub::spaces_semantic_search(const std::string& q) { + if (q.empty()) bad("q must not be empty"); + Query query; + query.add("q", q); + return call(Call::kSpacesSemanticSearch, PathArgs(), query); +} + +Json Hub::space_runtime(const RepoRef& space) { + PathArgs a; + a.repo(space); + return call(Call::kSpaceRuntime, a); +} + +Json Hub::daily_papers(int limit, const std::string& date) { + check_limit(limit); + Query q; + if (limit > 0) q.add("limit", static_cast(limit)); + add_if(q, "date", date); + return call(Call::kDailyPapers, PathArgs(), q); +} + +Json Hub::papers_search(const std::string& q, int limit) { + check_limit(limit); + if (q.empty()) bad("q must not be empty"); + Query query; + query.add("q", q); + if (limit > 0) query.add("limit", static_cast(limit)); + return call(Call::kPapersSearch, PathArgs(), query); +} + +Json Hub::paper(const std::string& paper_id) { + PathArgs a; + a.set("paperId", paper_id); + return call(Call::kPaperGet, a); +} + +Json Hub::partner_models(const std::string& provider, const std::string& status) { + PathArgs a; + a.set("provider", provider); + Query q; + add_if(q, "status", status); + return call(Call::kPartnerModels, a, q); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/hub.h b/src/vendors/huggingface/hub.h new file mode 100644 index 0000000..a78951f --- /dev/null +++ b/src/vendors/huggingface/hub.h @@ -0,0 +1,93 @@ +#pragma once + +// Hub reads over https://huggingface.co/api: models, datasets, spaces, repo +// trees / refs / commits / files, search, trending, tags, papers, Spaces +// catalogs, Inference Provider mappings, whoami, and file download. +// Public reads work without a token (Platform builds an anonymous client). +// Writes to repositories (commit, upload, delete, settings, branches, LFS) are +// outside this module; see SPEC.md ## Scope. + +#include +#include + +#include "vendors/huggingface/hf_service.h" + +namespace molo::huggingface { + +// Query for GET /api/{models,datasets,spaces}. Fields a repo type does not +// accept are refused by the call table, loudly. +struct RepoListParams { + std::string search; + std::string author; + std::string sort; // downloads, likes, trendingScore, createdAt, lastModified, ... + std::string direction; // "-1" descending + std::string pipeline_tag; // models only + std::string inference_provider; // models only (provider name or "all") + std::vector filter; + std::vector expand; // repeated expand= + int limit = 0; // page size, 0: server default; max HfCaps::kMaxPageLimit + bool full = false; + + Query to_query() const; +}; + +struct Download { + std::string bytes; + std::string content_type; + std::string commit; // X-Repo-Commit when sent + std::string etag; +}; + +class Hub : public HfService { + public: + explicit Hub(Access access) : HfService(std::move(access), Surface::kHub) {} + static Hub from_platform(const Platform& p) { return Hub(p.hub); } + + // The call table row that lists this repo type. + static Call list_call(RepoType type); + + Json whoami(); // needs a token + + // Lists (page walks, truncation reported). + Page list_repos(RepoType type, const RepoListParams& params, std::size_t max_items = 100, + std::size_t max_pages = 5); + Json repo_info(RepoType type, const RepoRef& ref, const std::string& revision = "", + const std::vector& expand = {}); + Json inference_provider_mapping(const RepoRef& model); // model info expanded to the mapping + + // Repo contents. + Json tree(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path = "", + bool recursive = false, int limit = 0); + Page tree_all(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path, + bool recursive, std::size_t max_items = 1000, std::size_t max_pages = 10); + Json refs(RepoType type, const RepoRef& ref, bool include_prs = false); + Json commits(RepoType type, const RepoRef& ref, const std::string& rev, int limit = 0); + Json compare(RepoType type, const RepoRef& ref, const std::string& compare); + Json paths_info(RepoType type, const RepoRef& ref, const std::string& rev, + const std::vector& paths); + Json likers(RepoType type, const RepoRef& ref); + Json safetensors_metadata(const RepoRef& model, const std::string& path); + // File bytes through /{repo}/resolve/{rev}/{path}; over the byte cap fails loudly. + Download download(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path); + + // Search and catalogs. + Json quicksearch(const std::string& q, int limit = 10, const std::string& type = ""); + Json full_text_search(const std::string& q, int limit = 10, const std::string& type = ""); + Json trending(const std::string& type = "", int limit = 0); + Json tags_by_type(RepoType type, const std::string& tag_type = ""); // models or datasets + Json tasks(); + Json spaces_hardware(); + Json spaces_featured(); + Json spaces_semantic_search(const std::string& q); + Json space_runtime(const RepoRef& space); // needs a token + + // Papers. + Json daily_papers(int limit = 0, const std::string& date = ""); + Json papers_search(const std::string& q, int limit = 0); + Json paper(const std::string& paper_id); + + // Inference Providers catalog: which models a provider serves. + Json partner_models(const std::string& provider, const std::string& status = ""); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/platform.cpp b/src/vendors/huggingface/platform.cpp new file mode 100644 index 0000000..571fbb4 --- /dev/null +++ b/src/vendors/huggingface/platform.cpp @@ -0,0 +1,130 @@ +#include "vendors/huggingface/platform.h" + +#include + +#include "clients/transports/http_transport.h" +#include "clients/url.h" +#include "vendors/huggingface/caps.h" +#include "llm/config.h" +#include "llm/error.h" + +namespace molo::huggingface { + +RequestBudget::RequestBudget(int max_calls, int window_seconds, Clock clock) + : max_calls_(max_calls), window_(window_seconds), clock_(std::move(clock)) { + if (!clock_) { + clock_ = [] { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + }; + } + window_start_ = clock_(); +} + +void RequestBudget::spend() { + std::lock_guard lock(mutex_); + const long now = clock_(); + if (now - window_start_ >= window_) { + window_start_ = now; + used_ = 0; + } + if (used_ >= max_calls_) + throw llm::ApiError("huggingface", 429, "LOCAL_BUDGET", + "local request budget spent (" + std::to_string(max_calls_) + " calls per " + + std::to_string(window_) + " s); wait for the next window"); + ++used_; +} + +int RequestBudget::used() const { + std::lock_guard lock(mutex_); + return used_; +} + +std::string Platform::token_from_env() { + std::string t = llm::Config::get(kKeyEnv); + if (t.empty()) t = llm::Config::get(kLegacyKeyEnv); + return t; +} + +llm::ApiClientConfig Platform::config(const std::string& base_url, bool has_token) { + llm::ApiClientConfig c; + c.vendor = "huggingface"; + c.base_url = base_url; + c.key_env = kKeyEnv; + if (has_token) { + c.auth = llm::AuthStyle::kBearer; + } else { + c.auth = llm::AuthStyle::kHeader; + c.auth_header = "X-Molo-Anonymous"; // marker, not a credential + } + return c; +} + +namespace { + +Access make_access(std::unique_ptr transport, const std::string& base_url, + const std::string& token, std::shared_ptr budget) { + Access a; + a.has_token = !token.empty(); + a.client = std::make_shared(std::move(transport), Platform::config(base_url, a.has_token), + a.has_token ? token : std::string("1")); + a.budget = std::move(budget); + return a; +} + +std::shared_ptr default_budget() { + return std::make_shared(HfCaps::kBudgetMaxCalls, HfCaps::kBudgetWindowSeconds); +} + +} // namespace + +Platform Platform::from_env() { + return with(std::make_unique(), + std::make_unique(), + std::make_unique(), token_from_env()); +} + +Platform Platform::with(std::unique_ptr hub_transport, + std::unique_ptr router_transport, + std::unique_ptr endpoints_transport, + const std::string& token, std::shared_ptr budget) { + if (!budget) budget = default_budget(); + Platform p; + p.hub = make_access(std::move(hub_transport), kHubHost, token, budget); + p.router = make_access(std::move(router_transport), kRouterHost, token, budget); + p.endpoints = make_access(std::move(endpoints_transport), kEndpointsHost, token, budget); + return p; +} + +std::string Platform::check_server_url(const std::string& url) { + if (url.empty() || url.size() > HfCaps::kMaxServerUrlChars) + throw llm::ApiError::invalid("huggingface", "server url is empty or too long"); + clients::Url parsed; + try { + parsed = clients::Url::parse(url); + } catch (const std::exception& e) { + throw llm::ApiError::invalid("huggingface", std::string("bad server url: ") + e.what()); + } + if ((parsed.scheme() != "http" && parsed.scheme() != "https") || !parsed.has_host()) + throw llm::ApiError::invalid("huggingface", "server url must be http(s)://host[:port]"); + if (url.find('@') != std::string::npos || url.find('?') != std::string::npos || + url.find('#') != std::string::npos) + throw llm::ApiError::invalid("huggingface", "server url must not carry credentials, a query or a fragment"); + std::string out = url; + while (!out.empty() && out.back() == '/') out.pop_back(); + return out; +} + +Access Platform::server(const std::string& base_url, const std::string& token, + std::shared_ptr budget) { + return server_with(std::make_unique(), base_url, token, std::move(budget)); +} + +Access Platform::server_with(std::unique_ptr transport, const std::string& base_url, + const std::string& token, std::shared_ptr budget) { + if (!budget) budget = default_budget(); + return make_access(std::move(transport), check_server_url(base_url), token, std::move(budget)); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/platform.h b/src/vendors/huggingface/platform.h new file mode 100644 index 0000000..6372c14 --- /dev/null +++ b/src/vendors/huggingface/platform.h @@ -0,0 +1,83 @@ +#pragma once + +// Wiring for the Hugging Face hosts and one shared request budget. +// +// One credential, HF_TOKEN (the legacy name HUGGING_FACE_HUB_TOKEN is also +// read), through llm::Config. The Hub serves public reads without any token, +// but llm::ApiClient cannot send an unauthenticated request. Workaround kept +// inside this package: without a token the clients are built with a harmless +// marker header instead of Authorization (reported in SPEC.md ## Gaps). + +#include +#include +#include +#include + +#include "clients/transport.h" +#include "llm/api_client.h" + +namespace molo::huggingface { + +// Fixed-window call counter. Local ceiling so one process never trips the +// server limit; the clock is injectable for tests. +class RequestBudget { + public: + using Clock = std::function; // monotonic seconds + RequestBudget(int max_calls, int window_seconds, Clock clock = nullptr); + // Throws ApiError 429 LOCAL_BUDGET when the window is spent. + void spend(); + int used() const; + int max_calls() const { return max_calls_; } + + private: + mutable std::mutex mutex_; + int max_calls_; + int window_; + Clock clock_; + long window_start_ = 0; + int used_ = 0; +}; + +// What a service needs: a client, whether a token backs it, the shared budget. +struct Access { + std::shared_ptr client; + bool has_token = false; + std::shared_ptr budget; +}; + +struct Platform { + static constexpr const char* kHubHost = "https://huggingface.co"; + static constexpr const char* kRouterHost = "https://router.huggingface.co"; + static constexpr const char* kEndpointsHost = "https://api.endpoints.huggingface.cloud"; + static constexpr const char* kKeyEnv = "HF_TOKEN"; + static constexpr const char* kLegacyKeyEnv = "HUGGING_FACE_HUB_TOKEN"; + + Access hub; + Access router; + Access endpoints; + + // HF_TOKEN, else HUGGING_FACE_HUB_TOKEN, else empty. + static std::string token_from_env(); + // Real HTTPS transports. A missing token still builds working anonymous + // clients (public Hub reads); calls that need a token fail NOT_CONFIGURED. + static Platform from_env(); + static Platform with(std::unique_ptr hub_transport, + std::unique_ptr router_transport, + std::unique_ptr endpoints_transport, + const std::string& token, + std::shared_ptr budget = nullptr); + bool configured() const { return hub.has_token; } + + // A TGI / TEI server (Inference Endpoint URL, localhost, any host). The token + // is optional: self-hosted servers usually have none. + static Access server(const std::string& base_url, const std::string& token, + std::shared_ptr budget = nullptr); + static Access server_with(std::unique_ptr transport, const std::string& base_url, + const std::string& token, std::shared_ptr budget = nullptr); + // http(s) URL with a host, no userinfo, query or fragment; trailing / removed. + static std::string check_server_url(const std::string& url); + + static llm::ApiClientConfig config(const std::string& base_url, bool has_token); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/providers.cpp b/src/vendors/huggingface/providers.cpp new file mode 100644 index 0000000..42ce25e --- /dev/null +++ b/src/vendors/huggingface/providers.cpp @@ -0,0 +1,188 @@ +#include "vendors/huggingface/providers.h" + +#include "vendors/huggingface/caps.h" +#include "llm/error.h" +#include "security/crypto/Base64.h" + +namespace molo::huggingface { + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +// hf-inference task map, from huggingface_hub inference/_providers/__init__.py +// ("hf-inference" entry) and hf_inference.py (binary input classes). +const TaskInfo kTasks[] = { + {"text-to-image", false, false, "binary image reply"}, + {"text-generation", false, false, ""}, + {"text-classification", false, false, ""}, + {"question-answering", false, false, ""}, + {"audio-classification", true, false, ""}, + {"automatic-speech-recognition", true, false, ""}, + {"fill-mask", false, false, ""}, + {"feature-extraction", false, true, "parameters at the top level"}, + {"image-classification", true, false, ""}, + {"image-segmentation", true, false, ""}, + {"document-question-answering", false, false, ""}, + {"image-to-text", true, false, ""}, + {"object-detection", true, false, ""}, + {"audio-to-audio", true, false, ""}, + {"zero-shot-image-classification", true, false, ""}, + {"zero-shot-classification", false, false, ""}, + {"image-to-image", true, false, "binary image reply"}, + {"sentence-similarity", false, true, ""}, + {"table-question-answering", false, false, ""}, + {"tabular-classification", false, false, ""}, + {"text-to-speech", false, false, "binary audio reply"}, + {"token-classification", false, false, ""}, + {"translation", false, false, ""}, + {"summarization", false, false, ""}, + {"visual-question-answering", true, false, ""}, +}; + +bool is_json_type(const std::string& ct) { + return ct.find("json") != std::string::npos; +} + +InferenceResult to_result(const llm::RawResponse& raw) { + InferenceResult r; + r.content_type = raw.content_type; + if (is_json_type(raw.content_type) || raw.content_type.empty()) { + try { + r.json = Json::parse(raw.body); + r.is_json = true; + return r; + } catch (const std::exception&) { + if (!raw.content_type.empty()) + throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "JSON reply does not parse"); + } + } + r.bytes = raw.body; + return r; +} + +} // namespace + +RouterModel RouterModel::parse(const std::string& text) { + if (text.empty() || text.size() > HfCaps::kMaxModelChars) bad("model is empty or too long"); + RouterModel m; + const std::size_t colon = text.find(':'); + m.id = text.substr(0, colon); + if (colon != std::string::npos) { + m.selector = text.substr(colon + 1); + if (!llm::Service::valid_id(m.selector) || m.selector.find(':') != std::string::npos) + bad("bad routing selector after ':' (fastest, cheapest, preferred or a provider id)"); + } + RequestPlan::check_model(m.id); + return m; +} + +void RouterChat::set_bill_to(const std::string& organization) { + if (!organization.empty() && !RequestPlan::valid_name(organization)) bad("bad organization name"); + bill_to_ = organization; +} + +llm::Headers RouterChat::headers() const { + llm::Headers h; + if (!bill_to_.empty()) h["X-HF-Bill-To"] = bill_to_; + return h; +} + +Json RouterChat::create(const Json& body) { + const Json* model = body.is_object() ? body.find("model") : nullptr; + if (!model || !model->is_string()) bad("model is required"); + RouterModel::parse(model->as_string()); + if (const Json* s = body.find("stream"); s && s->is_bool() && s->as_bool()) bad("use stream() for streaming"); + return call(Call::kRouterChat, PathArgs(), Query(), &body, headers()); +} + +void RouterChat::stream(const Json& body, const llm::SseDecoder::Sink& sink) { + const Json* model = body.is_object() ? body.find("model") : nullptr; + if (!model || !model->is_string()) bad("model is required"); + RouterModel::parse(model->as_string()); + HfService::stream(Call::kRouterChat, PathArgs(), body, sink, headers()); +} + +Json RouterChat::models() { return call(Call::kRouterModels); } + +Json RouterChat::create_hf_inference(const std::string& model, const Json& body) { + PathArgs a; + a.set("model", model); + return call(Call::kHfInferenceChat, a, Query(), &body, headers()); +} + +std::size_t TaskCatalog::count() { return sizeof(kTasks) / sizeof(kTasks[0]); } +const TaskInfo& TaskCatalog::at(std::size_t i) { + if (i >= count()) bad("task index out of range"); + return kTasks[i]; +} +const TaskInfo* TaskCatalog::find(const std::string& name) { + for (const TaskInfo& t : kTasks) + if (name == t.name) return &t; + return nullptr; +} + +InferenceResult InferenceTasks::run_json(const std::string& task, const std::string& model, const Json& inputs, + const Json& parameters) { + const TaskInfo* info = TaskCatalog::find(task); + if (!info) bad("unknown hf-inference task " + task); + if (info->binary_input) bad("task " + task + " takes a file input; use run_binary"); + if (inputs.is_null()) bad("inputs is required"); + RequestPlan::check_model(model); + Json body = Json::object(); + body.set("inputs", inputs); + if (info->pipeline_path) { + if (parameters.is_object()) + for (const auto& [k, v] : parameters.as_object()) body.set(k, v); + } else if (!parameters.is_null()) { + if (!parameters.is_object()) bad("parameters must be an object"); + if (!parameters.as_object().empty()) body.set("parameters", parameters); + } + PathArgs a; + a.set("model", model); + if (info->pipeline_path) { + a.set("task", task); + return to_result(call_raw(Call::kHfInferencePipeline, a, Query(), &body)); + } + return to_result(call_raw(Call::kHfInferenceTask, a, Query(), &body)); +} + +InferenceResult InferenceTasks::run_binary(const std::string& task, const std::string& model, + const std::string& bytes, const std::string& mime, + const Json& parameters) { + const TaskInfo* info = TaskCatalog::find(task); + if (!info) bad("unknown hf-inference task " + task); + if (!info->binary_input) bad("task " + task + " takes JSON input; use run_json"); + if (bytes.empty()) bad("input bytes are empty"); + if (bytes.size() > HfCaps::kMaxInferenceInputBytes) bad("input is over the byte cap"); + RequestPlan::check_model(model); + PathArgs a; + a.set("model", model); + const bool has_params = parameters.is_object() && !parameters.as_object().empty(); + if (!has_params) { + if (mime.empty() || mime.find('\n') != std::string::npos) bad("mime type required"); + return to_result(call_bytes(Call::kHfInferenceTask, a, bytes, mime)); + } + Json body = Json::object(); + body.set("inputs", Json(security::crypto::Base64::encode(bytes))); + body.set("parameters", parameters); + return to_result(call_raw(Call::kHfInferenceTask, a, Query(), &body)); +} + +Json InferenceTasks::embed(const std::string& model, const std::vector& texts, bool normalize) { + if (texts.empty() || texts.size() > HfCaps::kMaxBatchInputs) bad("texts must hold 1..256 entries"); + Json arr = Json::array(); + std::size_t total = 0; + for (const std::string& t : texts) { + total += t.size(); + if (total > HfCaps::kMaxTextChars) bad("texts are over the size cap"); + arr.push(Json(t)); + } + Json params = Json::object(); + if (normalize) params.set("normalize", Json(true)); + InferenceResult r = run_json("feature-extraction", model, texts.size() == 1 ? Json(texts[0]) : arr, params); + if (!r.is_json) throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "embedding reply is not JSON"); + return r.json; +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/providers.h b/src/vendors/huggingface/providers.h new file mode 100644 index 0000000..890223d --- /dev/null +++ b/src/vendors/huggingface/providers.h @@ -0,0 +1,85 @@ +#pragma once + +// Inference Providers (https://router.huggingface.co): the OpenAI-compatible +// chat router plus the hf-inference task APIs (the classic Inference API). +// The classic host api-inference.huggingface.co no longer resolves (probe +// 2026-10-25); the same tasks live under /hf-inference/models/{model}. + +#include +#include + +#include "vendors/huggingface/chat_wire.h" +#include "vendors/huggingface/hf_service.h" + +namespace molo::huggingface { + +// "owner/name" or "owner/name:selector" where selector is fastest, cheapest, +// preferred or a provider id (the router's routing suffix). +struct RouterModel { + std::string id; + std::string selector; + static RouterModel parse(const std::string& text); // throws ApiError::invalid + std::string str() const { return selector.empty() ? id : id + ":" + selector; } +}; + +class RouterChat : public HfService, public ChatEndpoint { + public: + explicit RouterChat(Access access) : HfService(std::move(access), Surface::kRouter) {} + static RouterChat from_platform(const Platform& p) { return RouterChat(p.router); } + + // Chat completions through the router. `bill_to` sets X-HF-Bill-To (an + // organization that pays); empty: the token owner pays. + Json create(const Json& body) override; + void stream(const Json& body, const llm::SseDecoder::Sink& sink) override; + void set_bill_to(const std::string& organization); + + // GET /v1/models: public, no token needed. + Json models(); + + // Chat straight to hf-inference for one model (no provider routing). + Json create_hf_inference(const std::string& model, const Json& body); + + private: + llm::Headers headers() const; + std::string bill_to_; +}; + +// One hf-inference task. +struct TaskInfo { + const char* name; // pipeline tag: text-classification, ... + bool binary_input; // the input is a file (image, audio), not JSON + bool pipeline_path; // /pipeline/{task} (feature-extraction, sentence-similarity) + const char* note; +}; + +struct TaskCatalog { + static std::size_t count(); + static const TaskInfo& at(std::size_t index); + static const TaskInfo* find(const std::string& name); // nullptr when unknown +}; + +struct InferenceResult { + bool is_json = false; + Json json; // when is_json + std::string bytes; // images, audio + std::string content_type; +}; + +class InferenceTasks : public HfService { + public: + explicit InferenceTasks(Access access) : HfService(std::move(access), Surface::kRouter) {} + static InferenceTasks from_platform(const Platform& p) { return InferenceTasks(p.router); } + + // JSON-input tasks: {"inputs": inputs, "parameters": parameters}. For the + // pipeline tasks the parameters go at the top level, as huggingface_hub does. + InferenceResult run_json(const std::string& task, const std::string& model, const Json& inputs, + const Json& parameters = Json()); + // File-input tasks: raw bytes when there are no parameters, else the + // base64 JSON form huggingface_hub sends. + InferenceResult run_binary(const std::string& task, const std::string& model, const std::string& bytes, + const std::string& mime, const Json& parameters = Json()); + // feature-extraction convenience: embeddings for 1..HfCaps::kMaxBatchInputs texts. + Json embed(const std::string& model, const std::vector& texts, bool normalize = false); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/request_plan.cpp b/src/vendors/huggingface/request_plan.cpp new file mode 100644 index 0000000..3f67845 --- /dev/null +++ b/src/vendors/huggingface/request_plan.cpp @@ -0,0 +1,271 @@ +#include "vendors/huggingface/request_plan.h" + +#include +#include + +#include "vendors/huggingface/caps.h" +#include "llm/api_client.h" +#include "llm/error.h" + +namespace molo::huggingface { + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +bool has_control(const std::string& s) { + for (unsigned char c : s) + if (c < 0x20 || c == 0x7f) return true; + return false; +} + +bool all_digits(const std::string& s) { + if (s.empty()) return false; + for (char c : s) + if (!std::isdigit(static_cast(c))) return false; + return true; +} + +std::string segment(const std::string& name, const std::string& value) { + if (value.empty()) bad(name + " must not be empty"); + if (value.size() > HfCaps::kMaxSegmentChars) bad(name + " is too long"); + if (has_control(value)) bad(name + " has control characters"); + if (value == "." || value == "..") bad(name + " must not be . or .."); + return RequestPlan::encode(value); +} + +std::string render_value(const std::string& name, const std::string& value) { + if (name == "namespace" || name == "repo" || name == "name") { + if (!RequestPlan::valid_name(value)) bad("bad " + name + " (letters, digits, - _ . only)"); + return value; + } + if (name == "repoType") { + RepoType t; + if (!parse_repo_type(value, t)) bad("repoType must be models, datasets or spaces"); + return repo_type_plural(t); + } + if (name == "typePrefix") { + RepoType t; + if (!parse_repo_type(value, t)) bad("repoType must be models, datasets or spaces"); + return t == RepoType::kModel ? "" : std::string("/") + repo_type_plural(t); + } + if (name == "path") return RequestPlan::encode_file_path(value); + if (name == "model") return RequestPlan::check_model(value); + if (name == "num") { + if (!all_digits(value) || value.size() > 9) bad("num must be a discussion number"); + return value; + } + if (name == "action") { + if (value != "enable" && value != "disable") bad("action must be enable or disable"); + return value; + } + if (name == "task" || name == "provider") { + if (!llm::Service::valid_id(value)) bad("bad " + name); + return value; + } + return segment(name, value); +} + +std::string query_encode_value(const std::string& v) { + std::string out = RequestPlan::encode(v); + // ApiClient refuses ".." anywhere in the request path, query included. + if (v.find("..") != std::string::npos) { + std::string fixed; + for (char c : out) { + if (c == '.') fixed += "%2E"; + else fixed += c; + } + return fixed; + } + return out; +} + +} // namespace + +const char* repo_type_plural(RepoType type) { + switch (type) { + case RepoType::kModel: return "models"; + case RepoType::kDataset: return "datasets"; + case RepoType::kSpace: return "spaces"; + } + return "models"; +} + +bool parse_repo_type(const std::string& text, RepoType& out) { + if (text == "models" || text == "model") out = RepoType::kModel; + else if (text == "datasets" || text == "dataset") out = RepoType::kDataset; + else if (text == "spaces" || text == "space") out = RepoType::kSpace; + else return false; + return true; +} + +RepoRef RepoRef::parse(const std::string& id) { + const std::size_t slash = id.find('/'); + if (slash == std::string::npos) + bad("repo id must be owner/name (for example openai-community/gpt2); legacy ids without an owner redirect"); + RepoRef r; + r.owner_ = id.substr(0, slash); + r.name_ = id.substr(slash + 1); + if (!RequestPlan::valid_name(r.owner_) || !RequestPlan::valid_name(r.name_)) + bad("bad repo id (owner/name, letters, digits, - _ . only)"); + return r; +} + +PathArgs& PathArgs::set(const std::string& name, std::string value) { + values_[name] = std::move(value); + return *this; +} + +PathArgs& PathArgs::repo(const RepoRef& ref) { + values_["namespace"] = ref.owner(); + values_["repo"] = ref.name(); + return *this; +} + +PathArgs& PathArgs::repo_type(RepoType type) { + values_["repoType"] = repo_type_plural(type); + values_["typePrefix"] = repo_type_plural(type); + return *this; +} + +const std::string* PathArgs::find(const std::string& name) const { + auto it = values_.find(name); + return it == values_.end() ? nullptr : &it->second; +} + +Query& Query::add(const std::string& name, const std::string& value) { + if (items_.size() >= HfCaps::kMaxQueryParams) bad("too many query parameters"); + if (value.size() > HfCaps::kMaxQueryValueChars) bad("query parameter " + name + " is too long"); + if (has_control(value)) bad("query parameter " + name + " has control characters"); + items_.emplace_back(name, value); + return *this; +} + +Query& Query::add(const std::string& name, long value) { return add(name, std::to_string(value)); } + +Query& Query::flag(const std::string& name, bool value) { return add(name, value ? "true" : "false"); } + +bool Query::has(const std::string& name) const { + for (const auto& kv : items_) + if (kv.first == name) return true; + return false; +} + +Query& Query::replace(const std::string& name, const std::string& value) { + std::vector> kept; + for (auto& kv : items_) + if (kv.first != name) kept.push_back(kv); + items_ = std::move(kept); + return add(name, value); +} + +std::string RequestPlan::encode(const std::string& text) { + static const char* hex = "0123456789ABCDEF"; + std::string out; + for (unsigned char c : text) { + if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') { + out += static_cast(c); + } else { + out += '%'; + out += hex[c >> 4]; + out += hex[c & 15]; + } + } + return out; +} + +bool RequestPlan::valid_name(const std::string& text) { + if (text.empty() || text.size() > HfCaps::kMaxSegmentChars || text == "." || text == "..") return false; + for (char c : text) + if (!(std::isalnum(static_cast(c)) || c == '-' || c == '_' || c == '.')) return false; + return true; +} + +std::string RequestPlan::encode_file_path(const std::string& path) { + if (path.empty()) bad("path must not be empty"); + if (path.size() > HfCaps::kMaxFilePathChars) bad("path is too long"); + if (path.front() == '/' || path.back() == '/') bad("path must not start or end with /"); + std::string out; + std::size_t segments = 0, start = 0; + while (start <= path.size()) { + std::size_t end = path.find('/', start); + if (end == std::string::npos) end = path.size(); + std::string seg = path.substr(start, end - start); + if (seg.empty()) bad("path has an empty segment"); + if (++segments > HfCaps::kMaxPathSegments) bad("path has too many segments"); + if (!out.empty()) out += '/'; + out += segment("path segment", seg); + start = end + 1; + } + return out; +} + +std::string RequestPlan::check_model(const std::string& model) { + if (model.empty() || model.size() > HfCaps::kMaxModelChars) bad("model id is empty or too long"); + const std::size_t slash = model.find('/'); + if (slash == std::string::npos) { + if (!valid_name(model)) bad("bad model id"); + return model; + } + if (model.find('/', slash + 1) != std::string::npos) bad("model id must be owner/name"); + if (!valid_name(model.substr(0, slash)) || !valid_name(model.substr(slash + 1))) bad("bad model id"); + return model; +} + +Plan RequestPlan::make(Call call, const PathArgs& args, const Query& query, const clients::body::Json* body) { + const CallSpec& spec = Calls::spec_of(call); + if (spec.has_flag("unsupported")) + bad(std::string("call is listed but not supported: ") + spec.path + " (see SPEC.md ## Gaps)"); + std::string out; + std::set used; + const std::string tpl = spec.path; + for (std::size_t i = 0; i < tpl.size();) { + if (tpl[i] != '{') { + out += tpl[i++]; + continue; + } + const std::size_t close = tpl.find('}', i); + if (close == std::string::npos) bad("malformed call table path"); + const std::string name = tpl.substr(i + 1, close - i - 1); + i = close + 1; + const std::string* v = args.find(name); + const bool last_optional = name == "path" && spec.has_flag("optpath") && i == tpl.size(); + if (!v || (v->empty() && last_optional)) { + if (last_optional) { + if (!out.empty() && out.back() == '/') out.pop_back(); + used.insert(name); + continue; + } + bad("missing path argument " + name); + } + used.insert(name); + out += render_value(name, *v); + } + for (const auto& kv : args.all()) { + // repo_type() sets both repoType and typePrefix; only the ones the path uses count. + if (!used.count(kv.first) && kv.first != "repoType" && kv.first != "typePrefix") + bad("path argument " + kv.first + " is not used by " + spec.path); + } + if (spec.path[0] == '{' && out.empty()) out = "/"; + std::string qs; + for (const auto& kv : query.items()) { + if (!Calls::in_list(spec.query_params, kv.first)) + bad("query parameter " + kv.first + " is not accepted by " + spec.http_method + " " + spec.path); + if (!qs.empty()) qs += '&'; + qs += encode(kv.first) + "=" + query_encode_value(kv.second); + } + if (body) { + if (!spec.has_body()) bad(std::string(spec.http_method) + " " + spec.path + " takes no body"); + if (!body->is_object()) bad("body must be a JSON object"); + for (const auto& kv : body->as_object()) + if (!Calls::in_list(spec.body_fields, kv.first)) + bad("body field " + kv.first + " is not accepted by " + spec.http_method + " " + spec.path); + } + Plan plan; + plan.call = call; + plan.method = Calls::method_of(call); + plan.target = out + (qs.empty() ? "" : "?" + qs); + return plan; +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/request_plan.h b/src/vendors/huggingface/request_plan.h new file mode 100644 index 0000000..a5af32c --- /dev/null +++ b/src/vendors/huggingface/request_plan.h @@ -0,0 +1,85 @@ +#pragma once + +// RequestPlan: turns one row of the call table plus caller arguments into a +// validated request target (path and query). Every id is checked before it is +// placed in a path, every query parameter and body field must be one the table +// lists, every size is capped (caps.h). Nothing here touches the network. + +#include +#include +#include + +#include "clients/body/json.h" +#include "clients/request.h" +#include "vendors/huggingface/calls.h" + +namespace molo::huggingface { + +// Hub repository kinds the module reads. Kernels exist on the Hub but are +// outside this module (SPEC.md ## Gaps). +enum class RepoType { kModel, kDataset, kSpace }; + +const char* repo_type_plural(RepoType type); // models, datasets, spaces +bool parse_repo_type(const std::string& text, RepoType& out); // accepts singular or plural + +// "owner/name" validated once. Legacy ids without an owner (gpt2) are refused +// loudly: the Hub answers them with a redirect to the canonical id. +class RepoRef { + public: + static RepoRef parse(const std::string& id); // throws ApiError::invalid + const std::string& owner() const { return owner_; } + const std::string& name() const { return name_; } + std::string id() const { return owner_ + "/" + name_; } + + private: + std::string owner_; + std::string name_; +}; + +class PathArgs { + public: + PathArgs& set(const std::string& name, std::string value); + PathArgs& repo(const RepoRef& ref); // namespace + repo + PathArgs& repo_type(RepoType type); // repoType + typePrefix + const std::string* find(const std::string& name) const; + const std::map& all() const { return values_; } + + private: + std::map values_; +}; + +class Query { + public: + Query& add(const std::string& name, const std::string& value); // repeats allowed + Query& add(const std::string& name, long value); + Query& flag(const std::string& name, bool value); + bool has(const std::string& name) const; + // Replaces every value of `name` (page walks set the cursor). + Query& replace(const std::string& name, const std::string& value); + const std::vector>& items() const { return items_; } + + private: + std::vector> items_; +}; + +struct Plan { + Call call; + clients::Method method; + std::string target; // "/path?query", percent-encoded +}; + +struct RequestPlan { + // Throws ApiError::invalid (400 INVALID_ARGUMENT) naming the offending part. + static Plan make(Call call, const PathArgs& args, const Query& query, + const clients::body::Json* body = nullptr); + + static std::string encode(const std::string& text); // RFC 3986 unreserved kept + // True for a Hub-style name: 1..256 of [A-Za-z0-9._-] and not "." or "..". + static bool valid_name(const std::string& text); + // Validates a repo-relative file path and returns it percent-encoded per segment. + static std::string encode_file_path(const std::string& path); + // Validates "owner/name" or a bare "name" model id (router, hf-inference). + static std::string check_model(const std::string& model); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/servers.cpp b/src/vendors/huggingface/servers.cpp new file mode 100644 index 0000000..f5566cc --- /dev/null +++ b/src/vendors/huggingface/servers.cpp @@ -0,0 +1,183 @@ +#include "vendors/huggingface/servers.h" + +#include "vendors/huggingface/caps.h" +#include "llm/error.h" + +namespace molo::huggingface { + +namespace { + +[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); } + +Json with_inputs(const Json& inputs, const Json& extra) { + Json body = Json::object(); + body.set("inputs", inputs); + if (extra.is_object()) + for (const auto& [k, v] : extra.as_object()) body.set(k, v); + return body; +} + +bool health_of(HfService& s, const std::function& get) { + (void)s; + try { + get(); + return true; + } catch (const llm::ApiError& e) { + if (e.http_status() == 503) return false; + throw; + } +} + +} // namespace + +bool TgiServer::ready() { + return health_of(*this, [&] { return call_raw(Call::kTgiHealth); }); +} +Json TgiServer::info() { return call(Call::kTgiGetModelInfo); } +std::string TgiServer::metrics() { return call_raw(Call::kTgiMetrics).body; } +Json TgiServer::models() { return call(Call::kTgiOpenaiGetModelInfo); } + +Json TgiServer::generate(const std::string& inputs, const Json& parameters) { + if (inputs.empty() || inputs.size() > HfCaps::kMaxTextChars) bad("inputs is empty or over the size cap"); + Json body = Json::object(); + body.set("inputs", Json(inputs)); + if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters); + return call(Call::kTgiGenerate, PathArgs(), Query(), &body); +} + +void TgiServer::generate_stream(const std::string& inputs, const Json& parameters, + const llm::SseDecoder::Sink& sink) { + if (inputs.empty() || inputs.size() > HfCaps::kMaxTextChars) bad("inputs is empty or over the size cap"); + Json body = Json::object(); + body.set("inputs", Json(inputs)); + if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters); + HfService::stream(Call::kTgiGenerateStream, PathArgs(), body, sink); +} + +Json TgiServer::tokenize(const std::string& inputs, const Json& parameters) { + if (inputs.empty() || inputs.size() > HfCaps::kMaxTextChars) bad("inputs is empty or over the size cap"); + Json body = Json::object(); + body.set("inputs", Json(inputs)); + if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters); + return call(Call::kTgiTokenize, PathArgs(), Query(), &body); +} + +Json TgiServer::chat_tokenize(const Json& chat_body) { + return call(Call::kTgiGetChatTokenize, PathArgs(), Query(), &chat_body); +} + +Json TgiServer::create(const Json& body) { + if (const Json* s = body.is_object() ? body.find("stream") : nullptr; s && s->is_bool() && s->as_bool()) + bad("use stream() for streaming"); + return call(Call::kTgiChatCompletions, PathArgs(), Query(), &body); +} + +void TgiServer::stream(const Json& body, const llm::SseDecoder::Sink& sink) { + HfService::stream(Call::kTgiChatCompletions, PathArgs(), body, sink); +} + +Json TgiServer::completions(const Json& body) { return call(Call::kTgiCompletions, PathArgs(), Query(), &body); } +Json TgiServer::invocations(const Json& body) { return call(Call::kTgiSagemakerCompatibility, PathArgs(), Query(), &body); } + +void TeiServer::check_inputs(const Json& inputs) { + std::size_t chars = 0, count = 1; + if (inputs.is_string()) { + chars = inputs.as_string().size(); + } else if (inputs.is_array()) { + count = inputs.as_array().size(); + if (count == 0 || count > HfCaps::kMaxBatchInputs) bad("inputs must hold 1..256 entries"); + for (const Json& x : inputs.as_array()) { + if (x.is_string()) chars += x.as_string().size(); + else if (x.is_array()) { + for (const Json& y : x.as_array()) + if (y.is_string()) chars += y.as_string().size(); + } else bad("inputs entries must be strings (or pairs of strings for predict)"); + } + } else { + bad("inputs must be a string or an array"); + } + if (chars == 0 || chars > HfCaps::kMaxTextChars) bad("inputs are empty or over the size cap"); +} + +bool TeiServer::ready() { return health_of(*this, [&] { return call_raw(Call::kTeiHealth); }); } +Json TeiServer::info() { return call(Call::kTeiGetModelInfo); } +std::string TeiServer::metrics() { return call_raw(Call::kTeiMetrics).body; } + +Json TeiServer::embed(const Json& inputs, const Json& options) { + check_inputs(inputs); + Json body = with_inputs(inputs, options); + return call(Call::kTeiEmbed, PathArgs(), Query(), &body); +} + +Json TeiServer::embed_all(const Json& inputs, const Json& options) { + check_inputs(inputs); + Json body = with_inputs(inputs, options); + return call(Call::kTeiEmbedAll, PathArgs(), Query(), &body); +} + +Json TeiServer::embed_sparse(const Json& inputs, const Json& options) { + check_inputs(inputs); + Json body = with_inputs(inputs, options); + return call(Call::kTeiEmbedSparse, PathArgs(), Query(), &body); +} + +Json TeiServer::predict(const Json& inputs, const Json& options) { + check_inputs(inputs); + Json body = with_inputs(inputs, options); + return call(Call::kTeiPredict, PathArgs(), Query(), &body); +} + +Json TeiServer::rerank(const std::string& query, const std::vector& texts, bool return_text, + bool raw_scores) { + if (query.empty() || query.size() > HfCaps::kMaxTextChars) bad("query is empty or over the size cap"); + if (texts.empty() || texts.size() > HfCaps::kMaxBatchInputs) bad("texts must hold 1..256 entries"); + Json arr = Json::array(); + std::size_t chars = query.size(); + for (const std::string& t : texts) { + chars += t.size(); + arr.push(Json(t)); + } + if (chars > HfCaps::kMaxTextChars) bad("texts are over the size cap"); + Json body = Json::object(); + body.set("query", Json(query)); + body.set("texts", arr); + if (return_text) body.set("return_text", Json(true)); + if (raw_scores) body.set("raw_scores", Json(true)); + return call(Call::kTeiRerank, PathArgs(), Query(), &body); +} + +Json TeiServer::similarity(const Json& inputs, const Json& parameters) { + if (!inputs.is_object()) bad("similarity inputs must be an object {source_sentence, sentences}"); + Json body = Json::object(); + body.set("inputs", inputs); + if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters); + return call(Call::kTeiSimilarity, PathArgs(), Query(), &body); +} + +Json TeiServer::tokenize(const Json& inputs, const Json& options) { + check_inputs(inputs); + Json body = with_inputs(inputs, options); + return call(Call::kTeiTokenize, PathArgs(), Query(), &body); +} + +Json TeiServer::decode(const std::vector& ids, bool skip_special_tokens) { + if (ids.empty() || ids.size() > 100000) bad("ids must hold 1..100000 entries"); + Json arr = Json::array(); + for (long id : ids) arr.push(Json(id)); + Json body = Json::object(); + body.set("ids", arr); + body.set("skip_special_tokens", Json(skip_special_tokens)); + return call(Call::kTeiDecode, PathArgs(), Query(), &body); +} + +Json TeiServer::openai_embeddings(const Json& input, const std::string& model, int dimensions) { + check_inputs(input); + if (dimensions < 0 || dimensions > 65536) bad("dimensions out of range"); + Json body = Json::object(); + body.set("input", input); + if (!model.empty()) body.set("model", Json(model)); + if (dimensions > 0) body.set("dimensions", Json(dimensions)); + return call(Call::kTeiOpenaiEmbed, PathArgs(), Query(), &body); +} + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/servers.h b/src/vendors/huggingface/servers.h new file mode 100644 index 0000000..efc89f7 --- /dev/null +++ b/src/vendors/huggingface/servers.h @@ -0,0 +1,64 @@ +#pragma once + +// Text Generation Inference (TGI) and Text Embeddings Inference (TEI) server +// APIs. Both run anywhere: an Inference Endpoint URL, a container on +// localhost, a Space. The base URL is given by the caller (Platform::server +// validates it); the token is optional because self-hosted servers usually +// have none. Request shapes follow the servers' own OpenAPI documents. + +#include +#include + +#include "vendors/huggingface/chat_wire.h" +#include "vendors/huggingface/hf_service.h" + +namespace molo::huggingface { + +class TgiServer : public HfService, public ChatEndpoint { + public: + explicit TgiServer(Access access) : HfService(std::move(access), Surface::kTgi) {} + + bool ready(); // GET /health: true on 2xx, false on 503, throws otherwise + Json info(); // GET /info + std::string metrics(); // GET /metrics (Prometheus text) + Json models(); // GET /v1/models + + // Native API. + Json generate(const std::string& inputs, const Json& parameters = Json()); + // Streams token events; sink returns false to stop. + void generate_stream(const std::string& inputs, const Json& parameters, const llm::SseDecoder::Sink& sink); + Json tokenize(const std::string& inputs, const Json& parameters = Json()); + Json chat_tokenize(const Json& chat_body); + + // OpenAI-compatible API. + Json create(const Json& body) override; // POST /v1/chat/completions + void stream(const Json& body, const llm::SseDecoder::Sink& sink) override; + Json completions(const Json& body); // POST /v1/completions + Json invocations(const Json& body); // POST /invocations (SageMaker compatibility) +}; + +class TeiServer : public HfService { + public: + explicit TeiServer(Access access) : HfService(std::move(access), Surface::kTei) {} + + bool ready(); + Json info(); + std::string metrics(); + + // inputs: one string or an array of strings (1..HfCaps::kMaxBatchInputs). + Json embed(const Json& inputs, const Json& options = Json()); + Json embed_all(const Json& inputs, const Json& options = Json()); + Json embed_sparse(const Json& inputs, const Json& options = Json()); + Json predict(const Json& inputs, const Json& options = Json()); + Json rerank(const std::string& query, const std::vector& texts, bool return_text = false, + bool raw_scores = false); + Json similarity(const Json& inputs, const Json& parameters = Json()); + Json tokenize(const Json& inputs, const Json& options = Json()); + Json decode(const std::vector& ids, bool skip_special_tokens = true); + Json openai_embeddings(const Json& input, const std::string& model = "", int dimensions = 0); + + // Counts and size of an `inputs` value: refuses anything over the caps. + static void check_inputs(const Json& inputs); +}; + +} // namespace molo::huggingface diff --git a/src/vendors/huggingface/tools/check_spec.py b/src/vendors/huggingface/tools/check_spec.py new file mode 100644 index 0000000..212c81a --- /dev/null +++ b/src/vendors/huggingface/tools/check_spec.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +"""Checks calls.cpp (the module's one call table) against Hugging Face's +published documents. Python 3 stdlib only, no credentials. + + python3 -I src/vendors/huggingface/tools/check_spec.py [--cache DIR] [--probe] + +Documents (all fetched unless found in --cache): + hub https://huggingface.co/.well-known/openapi.json (Hub OpenAPI 3.1) + ie https://api.endpoints.huggingface.cloud/openapi.json (Inference Endpoints 2.0.0) + tgi text-generation-inference docs/openapi.json (main) + tei text-embeddings-inference docs/openapi.json (main) + src huggingface_hub hf_api.py, hf_inference.py, constants.py (main; the Hub OpenAPI + omits the repo list / info routes, so those rows cite the SDK source) + task huggingface.js packages/tasks/src/tasks//spec/input.json + +Per row it requires: the HTTP method and path exist (a {repoType} row is +expanded to models, datasets, spaces; {typePrefix} to "", /datasets, /spaces), +the operationId matches when the document has one, every query parameter and +body field in the row is present in the document. Coverage: for ie, tgi and tei +every document operation, query parameter and body field must be in the table; +for hub every operation must be in the table or match an explicit EXCLUDED rule +(counted per reason in the output); for hub the unlisted query parameters and +body fields are reported (not failed: the Hub document is large and +the table lists what the module sends). With --probe, rows whose spec is +"src" or "none" are probed without a credential: a 404 fails. +Exit 0: all rows verified. Exit 1: at least one mismatch (printed). +""" +import hashlib +import json +import os +import re +import sys +import urllib.error +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +TABLE = os.path.join(HERE, "..", "calls.cpp") +RAW = "https://raw.githubusercontent.com/huggingface/" +URLS = { + "hub.json": "https://huggingface.co/.well-known/openapi.json", + "ie.json": "https://api.endpoints.huggingface.cloud/openapi.json", + "tgi.json": RAW + "text-generation-inference/main/docs/openapi.json", + "tei.json": RAW + "text-embeddings-inference/main/docs/openapi.json", + "hf_api.py": RAW + "huggingface_hub/main/src/huggingface_hub/hf_api.py", + "hf_inference.py": RAW + "huggingface_hub/main/src/huggingface_hub/inference/_providers/hf_inference.py", + "constants.py": RAW + "huggingface_hub/main/src/huggingface_hub/constants.py", +} +TASK_URL = RAW + "huggingface.js/main/packages/tasks/src/tasks/%s/spec/input.json" +HOSTS = { + "Hub": "https://huggingface.co", + "Router": "https://router.huggingface.co", + "Endpoints": "https://api.endpoints.huggingface.cloud", +} +S = r'"((?:[^"\\]|\\.)*)"' +ROW = re.compile(r"^\s*\{Call::(\w+), Surface::k(\w+), " + ", ".join([S] * 8) + r"\},?\s*$") +METHODS = ("get", "post", "put", "patch", "delete") +REPO_TYPES = ["models", "datasets", "spaces"] +PREFIXES = ["", "/datasets", "/spaces"] + +# Hub operations deliberately not wrapped: (reason, regex over "METHOD /path"). +EXCLUDED = [ + ("repository writes (commit, upload, LFS, branches, tags, squash, duplicate, settings, move, create)", + r"^(POST|PUT|DELETE) /api/(models|datasets|spaces)/\{namespace\}/\{repo\}/(commit|preupload|lfs-files|tag|branch|super-squash|settings|resource-group|volumes|hardware|sleeptime|dev-mode|custom-domain|secrets|variables)"), + ("repository writes (commit, upload, LFS, branches, tags, squash, duplicate, settings, move, create)", + r"^(POST|PUT|DELETE|GET) /api/(repos|\{repoType\})/.*(create|move|duplicate)"), + ("repository admin reads (LFS file list, xet tokens, jwt, scan, notebook, leaderboard, resource group)", + r"^GET /api/(models|datasets|spaces)/\{namespace\}/\{repo\}/(lfs-files|xet-write-token|xet-read-token|jwt|scan|notebook|leaderboard|resource-group|logs|events|metrics)"), + ("gated repo access requests", r"(ask-access|user-access|access-request)"), + ("resolve-cache and viewer helpers", r"^GET /api/resolve-cache/"), + ("secrets and variables of a Space (credentials)", r"^(GET|POST|DELETE) /api/spaces/\{namespace\}/\{repo\}/(secrets|variables)"), + ("rarely used discussion moderation (pin, merge, ignore, storage, ref delete, comment edit, hide, reaction)", + r"^(GET|POST|DELETE) /api/\{repoType\}/\{namespace\}/\{repo\}/discussions/\{num\}/(pin|merge|ignore|storage|ref|comment/)"), + ("collection item update and delete (need the slug-id split, see SPEC Gaps) and resource groups", + r"^(PATCH|DELETE|POST|GET) /api/collections/.*(items/|items/batch|resource-group|\{slug\}-\{id\})"), + ("comments on blogs, papers, posts", r"^(POST|DELETE) /api/(blog|papers|posts)/"), + ("paper curation (index, links, media, claim)", r"^POST /api/(papers/index|papers/\{paperId\}/(links|media)|settings/papers/claim)"), + ("account settings, billing and notifications", r"^(GET|POST|PUT|PATCH|DELETE) /api/(settings/(?!webhooks)|notifications|credentials)"), + ("organizations, users, avatars, SCIM, service accounts, resource groups", + r"^[A-Z]+ /api/(orgs|organizations|users|avatars|scim|service-accounts|resource-groups)"), + ("Hub Jobs and scheduled jobs (separate product)", r"^[A-Z]+ /api/(jobs|scheduled-jobs)"), + ("buckets, containers, kernels (separate repo kinds)", r"^[A-Z]+ /(api/)?(buckets|containers|kernels)"), + ("Spaces runtime control", r"^[A-Z]+ /api/spaces/(zero-gpu|\{namespace\}/\{repo\}/(runtime|restart|pause))"), + ("partner provider mapping writes and inference-provider account settings", + r"^(POST|PUT) /api/(partners|settings/inference-providers)"), + ("Inference Endpoints auth-check helper (internal)", r"/api/inference-endpoints/"), + ("OAuth, docs search, agents, sql console, registry token, repo search POST variant", + r"^[A-Z]+ /(oauth|api/(docs|agent-harnesses|registry|sql-console|\{repoType\}/\{namespace\}/\{repo\}/sql-console|quicksearch$))"), + ("blog, MCP settings, likes", r"^[A-Z]+ /api/(blog|settings|trending-.*)"), +] + + +def load(cache, name): + if cache: + p = os.path.join(cache, name) + if os.path.exists(p): + return open(p, "rb").read() + url = URLS.get(name) or name + req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"}) + with urllib.request.urlopen(req, timeout=120) as r: + data = r.read() + if cache: + os.makedirs(os.path.dirname(os.path.join(cache, name)), exist_ok=True) + open(os.path.join(cache, name), "wb").write(data) + return data + + +def resolve(doc, node): + seen = 0 + while isinstance(node, dict) and "$ref" in node and seen < 20: + cur = doc + for p in node["$ref"].lstrip("#/").split("/"): + cur = cur[p] + node = cur + seen += 1 + return node + + +def props(doc, schema): + schema = resolve(doc, schema) + if not isinstance(schema, dict): + return {} + out = dict(schema.get("properties", {})) + for key in ("allOf", "anyOf", "oneOf"): + for sub in schema.get(key, []): + out.update(props(doc, sub)) + return out + + +def op_fields(doc, op): + q = set() + for p in op.get("parameters", []): + p = resolve(doc, p) + if p.get("in") == "query": + q.add(p["name"]) + body = set() + rb = resolve(doc, op.get("requestBody", {})) + if rb: + content = rb.get("content", {}) + ct = next(iter(content), None) + if ct: + body = set(props(doc, content[ct].get("schema")).keys()) + return q, body + + +def unesc(s): + return s.replace('\\"', '"').replace("\\\\", "\\") + + +def variants(path, doc_paths=()): + # The document may carry {repoType} literally (discussions) or one path per + # repo type (everything else); a literal match wins. + if path in doc_paths: + return [path] + if "{repoType}" in path: + return [path.replace("{repoType}", t) for t in REPO_TYPES] + if "{typePrefix}" in path: + return [path.replace("{typePrefix}", p) for p in PREFIXES] + return [path] + + +def csv(s): + return [x for x in s.split(",") if x] + + +def probe_url(surface, path): + subs = {"{namespace}": "openai-community", "{repo}": "gpt2", "{rev}": "main", "{path}": "config.json", + "{model}": "openai-community/gpt2", "{task}": "feature-extraction", "{repoType}": "models"} + for k, v in subs.items(): + path = path.replace(k, v) + return HOSTS[surface] + path + + +def main(): + cache = None + if "--cache" in sys.argv: + cache = sys.argv[sys.argv.index("--cache") + 1] + text = open(TABLE).read() + begin, end = text.index("// CALLS-BEGIN"), text.index("// CALLS-END") + rows = [] + for line in text[begin:end].splitlines(): + m = ROW.match(line) + if m: + g = m.groups() + rows.append(dict(name=g[0], surface=g[1], method=unesc(g[2]), path=unesc(g[3]), spec=unesc(g[4]), + ref=unesc(g[5]), auth=unesc(g[6]), flags=unesc(g[7]), body=csv(unesc(g[8])), + query=csv(unesc(g[9])))) + if not rows: + print("no rows parsed from calls.cpp") + return 1 + docs = {} + shas = {} + for key in ("hub", "ie", "tgi", "tei"): + raw = load(cache, key + ".json") + shas[key] = hashlib.sha256(raw).hexdigest()[:12] + docs[key] = json.loads(raw) + sources = "" + for name in ("hf_api.py", "hf_inference.py", "constants.py"): + raw = load(cache, name) + shas[name] = hashlib.sha256(raw).hexdigest()[:12] + sources += raw.decode("utf-8", "replace") + ops = {} + for key, doc in docs.items(): + ops[key] = {} + for path, item in doc["paths"].items(): + for method, op in item.items(): + if method in METHODS: + ops[key][(method.upper(), path)] = op + bad = [] + names = set() + covered = {k: set() for k in docs} + notes = {"hub_unlisted_query": 0, "hub_unlisted_body": 0} + for r in rows: + if r["name"] in names: + bad.append("duplicate row %s" % r["name"]) + names.add(r["name"]) + spec = r["spec"] + if spec in docs: + union_q, union_b, found = set(), set(), False + for v in variants(r["path"], docs[spec]["paths"]): + op = ops[spec].get((r["method"], v)) + if op is None: + bad.append("%s: %s %s not in the %s document" % (r["name"], r["method"], v, spec)) + continue + found = True + covered[spec].add((r["method"], v)) + if spec != "hub" and r["ref"] != "-" and op.get("operationId") != r["ref"]: + bad.append("%s: operationId %s, table says %s" % (r["name"], op.get("operationId"), r["ref"])) + q, b = op_fields(docs[spec], op) + union_q |= q + union_b |= b + if not found: + continue + for f in r["query"]: + if f not in union_q: + bad.append("%s: query parameter %s absent from the document" % (r["name"], f)) + for f in r["body"]: + if f not in union_b: + bad.append("%s: body field %s absent from the document" % (r["name"], f)) + if spec == "hub": + notes["hub_unlisted_query"] += len(union_q - set(r["query"])) + notes["hub_unlisted_body"] += len(union_b - set(r["body"])) + else: + for f in union_q - set(r["query"]): + bad.append("%s: document query parameter %s not in the table" % (r["name"], f)) + for f in union_b - set(r["body"]): + bad.append("%s: document body field %s not in the table" % (r["name"], f)) + elif spec == "src": + if r["ref"] not in sources: + bad.append("%s: source needle %r not found in huggingface_hub" % (r["name"], r["ref"])) + elif spec == "task": + try: + schema = json.loads(load(cache, "tasks/%s.json" % r["ref"]) if cache and os.path.exists(os.path.join(cache, "tasks/%s.json" % r["ref"])) else load(None, TASK_URL % r["ref"])) + except Exception as e: # noqa + bad.append("%s: cannot load the %s task schema (%s)" % (r["name"], r["ref"], e)) + continue + allowed = set(schema.get("properties", {}).keys()) + for f in r["body"]: + if f not in allowed: + bad.append("%s: body field %s absent from the %s task schema" % (r["name"], f, r["ref"])) + for f in allowed - set(r["body"]): + bad.append("%s: task schema field %s not in the table" % (r["name"], f)) + elif spec != "none": + bad.append("%s: unknown spec kind %s" % (r["name"], spec)) + # Coverage. + for key in ("ie", "tgi", "tei"): + for k in ops[key]: + if k not in covered[key]: + bad.append("unwrapped %s operation %s %s" % (key, k[0], k[1])) + excluded_count = {} + unwrapped_hub = 0 + for (method, path), op in sorted(ops["hub"].items()): + if (method, path) in covered["hub"]: + continue + label = "%s %s" % (method, path) + reason = None + for why, rx in EXCLUDED: + if re.search(rx, label): + reason = why + break + if reason is None: + bad.append("unwrapped hub operation %s (tags %s)" % (label, op.get("tags"))) + else: + excluded_count[reason] = excluded_count.get(reason, 0) + 1 + unwrapped_hub += 1 + if "--probe" in sys.argv: + for r in rows: + if r["spec"] not in ("src", "none") or r["surface"] not in HOSTS or r["method"] != "GET": + continue + if "{" in r["path"].replace("{namespace}", "").replace("{repo}", "").replace("{rev}", "").replace("{path}", "").replace("{model}", ""): + continue + url = probe_url(r["surface"], r["path"]) + req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"}) + try: + status = urllib.request.urlopen(req, timeout=60).status + except urllib.error.HTTPError as e: + status = e.code + print("probe GET %s -> %d" % (url, status)) + if status == 404: + bad.append("%s: probe answered 404" % r["name"]) + for b in bad: + print("FAIL " + b) + print("sources: " + ", ".join("%s=%s" % kv for kv in sorted(shas.items()))) + print("hub operations excluded (%d):" % unwrapped_hub) + for why, n in sorted(excluded_count.items()): + print(" %3d %s" % (n, why)) + print("hub columns the module does not send: %d query parameters, %d body fields (information)" + % (notes["hub_unlisted_query"], notes["hub_unlisted_body"])) + counts = {k: len(v) for k, v in ops.items()} + print("%d rows checked; spec operations hub=%d ie=%d tgi=%d tei=%d; %d problems" + % (len(rows), counts["hub"], counts["ie"], counts["tgi"], counts["tei"], len(bad))) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/example/ai/huggingface/page_test.cpp b/tests/example/ai/huggingface/page_test.cpp new file mode 100644 index 0000000..d619012 --- /dev/null +++ b/tests/example/ai/huggingface/page_test.cpp @@ -0,0 +1,40 @@ +// Routes: GET /ai/huggingface (page), /status, /calls, /tasks (no credential needed) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + unsetenv("HF_TOKEN"); + unsetenv("HUGGING_FACE_HUB_TOKEN"); + Fixture f; + auto page = f.client->get("/ai/huggingface"); + check(page.status == 200, "page 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "html"); + check(contains(page.body, "Hugging Face") && contains(page.body, "viewport"), "title and mobile viewport"); + check(contains(page.body, "/embedded/ai_huggingface.js"), "script wired"); + check(f.client->post("/ai/huggingface", "{}", "application/json").status == 405, "post page is 405"); + + auto st = f.client->get("/ai/huggingface/status"); + check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without a token"); + check(contains(st.body, "HF_TOKEN") && contains(st.body, "router.huggingface.co") && + contains(st.body, "api.endpoints.huggingface.cloud"), + "status names credential env and hosts"); + check(contains(st.body, "\"calls\":132") && contains(st.body, "\"tasks\":25"), "status counts calls and tasks"); + check(f.client->post("/ai/huggingface/status", "{}", "application/json").status == 405, "post status is 405"); + + auto calls = f.client->get("/ai/huggingface/calls"); + check(calls.status == 200 && contains(calls.body, "/v1/chat/completions") && + contains(calls.body, "/v2/endpoint/{namespace}/{name}/pause") && contains(calls.body, "/embed") && + contains(calls.body, "/api/whoami-v2"), + "call table listed across surfaces"); + check(f.client->post("/ai/huggingface/calls", "{}", "application/json").status == 405, "post calls is 405"); + + auto tasks = f.client->get("/ai/huggingface/tasks"); + check(tasks.status == 200 && contains(tasks.body, "feature-extraction") && + contains(tasks.body, "\"binary_input\":true"), + "task catalog listed"); + return finish("ai/huggingface/page"); +} diff --git a/tests/example/ai/huggingface/routes_test.cpp b/tests/example/ai/huggingface/routes_test.cpp new file mode 100644 index 0000000..a875e57 --- /dev/null +++ b/tests/example/ai/huggingface/routes_test.cpp @@ -0,0 +1,87 @@ +// Routes: POST /ai/huggingface/* (405, 422, dry runs, 503 without a credential) +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + setenv("MOLO_ENV_FILES", "off", 1); + unsetenv("HF_TOKEN"); + unsetenv("HUGGING_FACE_HUB_TOKEN"); + Fixture f; + auto post = [&](const std::string& route, const std::string& body) { + return f.client->post("/ai/huggingface/" + route, body, "application/json"); + }; + const char* all[] = {"hub/list", "hub/repo", "hub/search", "hub/whoami", "hub/discussions", "hub/collections", + "hub/webhooks", "chat", "chat/stream", "task", "endpoints", "tgi", "tei"}; + for (const char* r : all) { + std::string route = std::string("/ai/huggingface/") + r; + check(f.client->get(route).status == 405, route + " get is 405"); + check(post(r, "not json").status == 422, route + " bad json is 422"); + check(post(r, "[1]").status == 422, route + " non-object is 422"); + } + + // 422: input rules run before anything else + check(post("hub/list", "{\"kind\":\"kernels\"}").status == 422, "unknown kind"); + check(post("hub/list", "{\"limit\":101}").status == 422, "limit over 100"); + check(post("hub/list", "{\"pipeline_tag\":\"x\",\"kind\":\"datasets\"}").status == 422, "pipeline_tag is models only"); + check(post("hub/repo", "{}").status == 422, "repo needs action"); + check(post("hub/repo", "{\"action\":\"info\",\"id\":\"gpt2\"}").status == 422, "legacy id without owner"); + check(post("hub/repo", "{\"action\":\"info\",\"id\":\"../x/y\"}").status == 422, "path traversal id"); + check(post("hub/repo", "{\"action\":\"fly\",\"id\":\"a/b\"}").status == 422, "unknown repo action"); + check(post("hub/repo", "{\"action\":\"file\",\"id\":\"a/b\",\"path\":\"../etc\"}").status == 422, "file path traversal"); + check(post("hub/search", "{\"action\":\"quicksearch\"}").status == 422, "quicksearch needs q"); + check(post("hub/search", "{\"action\":\"nope\"}").status == 422, "unknown search action"); + check(post("hub/discussions", "{\"action\":\"get\",\"id\":\"a/b\"}").status == 422, "discussion get needs num"); + check(post("hub/discussions", "{\"action\":\"create\",\"id\":\"a/b\"}").status == 422, "discussion create needs title"); + check(post("hub/collections", "{\"action\":\"add_item\",\"owner\":\"me\",\"slug\":\"x-1\",\"item_type\":\"video\",\"item_id\":\"x\"}").status == 422, + "bad collection item type"); + check(post("hub/webhooks", "{\"action\":\"create\",\"url\":\"https://example.com/hook\"}").status == 422, "webhook needs watched"); + check(post("chat", "{\"model\":\"a/b\"}").status == 422, "chat needs prompt"); + check(post("chat", "{\"prompt\":\"hi\"}").status == 422, "chat needs model"); + check(post("chat", "{\"model\":\"a b\",\"prompt\":\"hi\"}").status == 422, "bad model id"); + check(post("chat", "{\"model\":\"a/b\",\"prompt\":\"hi\",\"max_tokens\":0}").status == 422, "max_tokens 0"); + check(post("task", "{\"task\":\"nope\",\"model\":\"a/b\",\"inputs\":\"x\"}").status == 422, "unknown task"); + check(post("task", "{\"task\":\"translation\",\"model\":\"a/b\"}").status == 422, "task needs inputs"); + check(post("task", "{\"task\":\"image-classification\",\"model\":\"a/b\",\"data_base64\":\"@@\",\"mime\":\"image/png\"}").status == 422, + "bad base64"); + check(post("endpoints", "{\"action\":\"get\",\"namespace\":\"acme\"}").status == 422, "endpoint get needs name"); + check(post("endpoints", "{\"action\":\"get\",\"namespace\":\"acme\",\"name\":\"Bad_Name\"}").status == 422, "bad endpoint name"); + check(post("endpoints", "{\"action\":\"create\",\"namespace\":\"acme\",\"spec\":{\"name\":\"x\"}}").status == 422, "incomplete endpoint spec"); + check(post("tgi", "{\"action\":\"health\",\"url\":\"ftp://x\"}").status == 422, "tgi url must be http(s)"); + check(post("tei", "{\"action\":\"embed\",\"url\":\"http://127.0.0.1:1\"}").status == 422, "tei embed needs inputs"); + + // dry runs: paid or resource-changing routes show the exact request + auto dry = post("chat", "{\"model\":\"openai/gpt-oss-20b:fastest\",\"prompt\":\"hi\"}"); + check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "/v1/chat/completions") && + contains(dry.body, "openai/gpt-oss-20b:fastest"), + "chat is a dry run without confirm"); + check(contains(post("chat/stream", "{\"model\":\"a/b\",\"prompt\":\"hi\"}").body, "\"stream\":true"), "stream preview"); + check(contains(post("task", "{\"task\":\"translation\",\"model\":\"t5/x\",\"inputs\":\"hi\"}").body, "\"dry_run\":true"), "task is a dry run"); + check(contains(post("hub/discussions", "{\"action\":\"create\",\"id\":\"a/b\",\"title\":\"Typo\"}").body, "\"dry_run\":true"), + "discussion create is a dry run"); + check(contains(post("hub/discussions", "{\"action\":\"remove\",\"id\":\"a/b\",\"num\":3}").body, "DELETE"), "discussion remove is a dry run"); + check(contains(post("hub/collections", "{\"action\":\"create\",\"owner\":\"me\",\"title\":\"T\"}").body, "\"dry_run\":true"), + "collection create is a dry run"); + check(contains(post("hub/webhooks", "{\"action\":\"remove\",\"id\":\"w1\"}").body, "\"dry_run\":true"), "webhook remove is a dry run"); + check(contains(post("endpoints", "{\"action\":\"pause\",\"namespace\":\"acme\",\"name\":\"ep\"}").body, "\"dry_run\":true"), + "endpoint pause is a dry run"); + check(contains(post("endpoints", "{\"action\":\"remove\",\"namespace\":\"acme\",\"name\":\"ep\"}").body, "DELETE"), + "endpoint remove is a dry run"); + + // 503: reads and confirmed calls need the credential, before any network traffic + for (const char* r : {"hub/whoami"}) { + auto res = post(r, "{}"); + check(res.status == 503 && contains(res.body, "huggingface_not_configured"), std::string(r) + " without a token is 503"); + } + check(post("hub/list", "{}").status == 503, "list without a token is 503"); + check(post("hub/repo", "{\"action\":\"info\",\"id\":\"openai-community/gpt2\"}").status == 503, "repo info without a token is 503"); + check(post("hub/search", "{\"action\":\"tasks\"}").status == 503, "search without a token is 503"); + check(post("endpoints", "{\"action\":\"vendors\"}").status == 503, "endpoints without a token is 503"); + check(post("tgi", "{\"action\":\"health\",\"url\":\"http://127.0.0.1:1\"}").status == 503, "tgi without a token is 503"); + check(post("chat", "{\"model\":\"a/b\",\"prompt\":\"hi\",\"confirm\":true}").status == 503, "confirmed chat without a token is 503"); + check(post("endpoints", "{\"action\":\"remove\",\"namespace\":\"acme\",\"name\":\"ep\",\"confirm\":true}").status == 503, + "confirmed endpoint remove without a token is 503"); + return finish("ai/huggingface/routes"); +} diff --git a/tests/src/vendors/huggingface/support.h b/tests/src/vendors/huggingface/support.h new file mode 100644 index 0000000..83843c3 --- /dev/null +++ b/tests/src/vendors/huggingface/support.h @@ -0,0 +1,63 @@ +#pragma once + +// Shared rig for the Hugging Face tests: a Platform over three MockTransports +// (hub, router, endpoints). The mock only records the requests the real client +// builds and answers with canned bodies. Canned bodies are either RECORDED +// (copied from an unauthenticated probe on 2026-10-25, trimmed) or SHAPE +// (written by us to the documented schema, not recorded). Nothing here +// verifies vendor behavior (see src/vendors/huggingface/SPEC.md ## Unverified). + +#include +#include + +#include "clients/transports/mock_transport.h" +#include "framework.h" +#include "vendors/huggingface/platform.h" + +namespace hf_test { + +using molo::clients::ClientResponse; +using molo::clients::HeaderMap; +using molo::clients::Method; +using molo::clients::transports::MockExpectation; +using molo::clients::transports::MockTransport; +using Json = molo::clients::body::Json; + +inline ClientResponse reply(const std::string& body, int status = 200, + const std::string& type = "application/json") { + HeaderMap h; + h.set("Content-Type", type); + return ClientResponse(status, h, body); +} + +inline ClientResponse sse(const std::string& body) { return reply(body, 200, "text/event-stream"); } + +inline const char* kToken = "hf_TESTTOKEN_DO_NOT_LEAK"; + +struct Rig { + MockTransport* hub = nullptr; + MockTransport* router = nullptr; + MockTransport* endpoints = nullptr; + molo::huggingface::Platform platform; + + explicit Rig(const std::string& token = kToken, + std::shared_ptr budget = nullptr) { + auto a = std::make_unique(); + auto b = std::make_unique(); + auto c = std::make_unique(); + hub = a.get(); + router = b.get(); + endpoints = c.get(); + platform = molo::huggingface::Platform::with(std::move(a), std::move(b), std::move(c), token, budget); + } + + static std::string header(const MockTransport::SeenRequest& r, const std::string& name) { + return r.headers.get_or(name, ""); + } +}; + +inline bool has(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +} // namespace hf_test diff --git a/tests/src/vendors/huggingface/test_calls.cpp b/tests/src/vendors/huggingface/test_calls.cpp new file mode 100644 index 0000000..b877367 --- /dev/null +++ b/tests/src/vendors/huggingface/test_calls.cpp @@ -0,0 +1,212 @@ +// The call table and request planning: structure of the table, id and path +// validation, query and body field checks, caps. Pure local logic. +#include + +#include "support.h" +#include "vendors/huggingface/calls.h" +#include "vendors/huggingface/request_plan.h" + +using namespace molo::huggingface; +using namespace hf_test; +using molo::llm::ApiError; + +TEST(huggingface_calls, table_is_in_enum_order_and_unique) { + std::set seen; + for (std::size_t i = 0; i < Calls::count(); ++i) { + const CallSpec& s = Calls::at(i); + ASSERT_EQ(static_cast(s.call), i); + ASSERT_TRUE(&Calls::spec_of(s.call) == &s); + std::string key = std::string(surface_name(s.surface)) + " " + s.http_method + " " + s.path; + ASSERT_TRUE(seen.insert(key).second); + } + ASSERT_EQ(Calls::count(), static_cast(132)); +} + +TEST(huggingface_calls, rows_use_known_vocabulary) { + const std::set specs = {"none", "hub", "ie", "tgi", "tei", "src", "task"}; + const std::set auths = {"none", "optional", "token"}; + const std::set methods = {"GET", "POST", "PUT", "PATCH", "DELETE"}; + for (std::size_t i = 0; i < Calls::count(); ++i) { + const CallSpec& s = Calls::at(i); + ASSERT_TRUE(specs.count(s.spec) == 1); + ASSERT_TRUE(auths.count(s.auth) == 1); + ASSERT_TRUE(methods.count(s.http_method) == 1); + ASSERT_TRUE(s.path[0] == '/' || s.path[0] == '{'); + ASSERT_TRUE(std::string(s.ref).size() > 0); + // braces balance + int depth = 0; + for (const char* p = s.path; *p; ++p) { + if (*p == '{') ++depth; + if (*p == '}') --depth; + ASSERT_TRUE(depth == 0 || depth == 1); + } + ASSERT_EQ(depth, 0); + } +} + +TEST(huggingface_calls, writes_need_a_token_and_deletes_are_marked_destructive) { + std::size_t deletes = 0; + for (std::size_t i = 0; i < Calls::count(); ++i) { + const CallSpec& s = Calls::at(i); + if (s.has_flag("write")) ASSERT_EQ(std::string(s.auth), std::string("token")); + if (std::string(s.http_method) == "DELETE") { + ++deletes; + ASSERT_TRUE(s.has_flag("destructive")); + } + if (s.has_flag("destructive")) ASSERT_TRUE(s.has_flag("write")); + } + ASSERT_EQ(deletes, static_cast(7)); // 3 Hub (discussion, collection, webhook) + 4 Endpoints +} + +TEST(huggingface_calls, per_surface_counts_match_the_published_documents) { + std::size_t counts[5] = {0, 0, 0, 0, 0}; + std::size_t unsupported = 0; + for (std::size_t i = 0; i < Calls::count(); ++i) { + ++counts[static_cast(Calls::at(i).surface)]; + if (Calls::at(i).has_flag("unsupported")) ++unsupported; + } + ASSERT_EQ(counts[static_cast(Surface::kEndpoints)], static_cast(48)); // IE openapi 2.0.0 + ASSERT_EQ(counts[static_cast(Surface::kTgi)], static_cast(12)); // TGI 3.3.6-dev0 + ASSERT_EQ(counts[static_cast(Surface::kTei)], static_cast(12)); // TEI 1.9.4 + ASSERT_EQ(unsupported, static_cast(3)); // the three GET SSE streams +} + +TEST(huggingface_paths, repo_ref_needs_owner_and_name) { + RepoRef r = RepoRef::parse("openai-community/gpt2"); + ASSERT_EQ(r.owner(), std::string("openai-community")); + ASSERT_EQ(r.name(), std::string("gpt2")); + ASSERT_THROW(RepoRef::parse("gpt2"), ApiError); + ASSERT_THROW(RepoRef::parse("a/b/c"), ApiError); + ASSERT_THROW(RepoRef::parse("../etc"), ApiError); + ASSERT_THROW(RepoRef::parse("a/.."), ApiError); + ASSERT_THROW(RepoRef::parse("a/b?x=1"), ApiError); + ASSERT_THROW(RepoRef::parse("a/b c"), ApiError); + ASSERT_THROW(RepoRef::parse(std::string("a/") + std::string(257, 'x')), ApiError); +} + +TEST(huggingface_paths, renders_typed_repo_paths) { + PathArgs a; + a.repo(RepoRef::parse("squad/x")).repo_type(RepoType::kDataset).set("rev", "main").set("path", ""); + Plan p = RequestPlan::make(Call::kRepoTree, a, Query()); + ASSERT_EQ(p.target, std::string("/api/datasets/squad/x/tree/main")); + ASSERT_TRUE(p.method == Method::kGet); + a.set("path", "data/train 1.parquet"); + p = RequestPlan::make(Call::kRepoTree, a, Query()); + ASSERT_EQ(p.target, std::string("/api/datasets/squad/x/tree/main/data/train%201.parquet")); +} + +TEST(huggingface_paths, revision_is_one_encoded_segment) { + PathArgs a; + a.repo(RepoRef::parse("o/n")).repo_type(RepoType::kModel).set("rev", "refs/pr/1"); + Plan p = RequestPlan::make(Call::kRepoCommits, a, Query()); + ASSERT_EQ(p.target, std::string("/api/models/o/n/commits/refs%2Fpr%2F1")); + a.set("rev", ".."); + ASSERT_THROW(RequestPlan::make(Call::kRepoCommits, a, Query()), ApiError); + a.set("rev", ""); + ASSERT_THROW(RequestPlan::make(Call::kRepoCommits, a, Query()), ApiError); + a.set("rev", "a\nb"); + ASSERT_THROW(RequestPlan::make(Call::kRepoCommits, a, Query()), ApiError); +} + +TEST(huggingface_paths, resolve_uses_the_type_prefix) { + PathArgs a; + a.repo(RepoRef::parse("o/n")).set("rev", "main").set("path", "config.json"); + a.repo_type(RepoType::kModel); + ASSERT_EQ(RequestPlan::make(Call::kFileResolve, a, Query()).target, std::string("/o/n/resolve/main/config.json")); + a.repo_type(RepoType::kDataset); + ASSERT_EQ(RequestPlan::make(Call::kFileResolve, a, Query()).target, + std::string("/datasets/o/n/resolve/main/config.json")); + a.repo_type(RepoType::kSpace); + ASSERT_EQ(RequestPlan::make(Call::kFileResolve, a, Query()).target, std::string("/spaces/o/n/resolve/main/config.json")); + a.set("path", ""); // a file path is required for resolve + ASSERT_THROW(RequestPlan::make(Call::kFileResolve, a, Query()), ApiError); +} + +TEST(huggingface_paths, file_path_rules) { + ASSERT_EQ(RequestPlan::encode_file_path("a/b.txt"), std::string("a/b.txt")); + ASSERT_THROW(RequestPlan::encode_file_path("/abs"), ApiError); + ASSERT_THROW(RequestPlan::encode_file_path("a//b"), ApiError); + ASSERT_THROW(RequestPlan::encode_file_path("a/../b"), ApiError); + ASSERT_THROW(RequestPlan::encode_file_path("a/./b"), ApiError); + ASSERT_THROW(RequestPlan::encode_file_path(std::string(1025, 'x')), ApiError); + std::string deep; + for (int i = 0; i < 33; ++i) deep += "d/"; + ASSERT_THROW(RequestPlan::encode_file_path(deep + "f"), ApiError); +} + +TEST(huggingface_paths, query_and_body_are_checked_against_the_table) { + PathArgs a; + a.repo(RepoRef::parse("o/n")).repo_type(RepoType::kModel); + Query ok; + ok.flag("include_prs", true); + ASSERT_EQ(RequestPlan::make(Call::kRepoRefs, a, ok).target, std::string("/api/models/o/n/refs?include_prs=true")); + Query unknown; + unknown.add("evil", "1"); + ASSERT_THROW(RequestPlan::make(Call::kRepoRefs, a, unknown), ApiError); + Json body = Json::object(); + ASSERT_THROW(RequestPlan::make(Call::kRepoRefs, a, Query(), &body), ApiError); // GET takes no body + PathArgs b; + b.repo(RepoRef::parse("o/n")).repo_type(RepoType::kModel).set("rev", "main"); + Json paths = Json::object(); + paths.set("paths", Json::array()); + ASSERT_TRUE(!RequestPlan::make(Call::kRepoPathsInfo, b, Query(), &paths).target.empty()); + paths.set("sneaky", Json(1)); + ASSERT_THROW(RequestPlan::make(Call::kRepoPathsInfo, b, Query(), &paths), ApiError); +} + +TEST(huggingface_paths, query_values_are_encoded_and_dots_survive_the_client_guard) { + Query q; + q.add("search", "a b&c=d"); + Plan p = RequestPlan::make(Call::kModelsList, PathArgs(), q); + ASSERT_EQ(p.target, std::string("/api/models?search=a%20b%26c%3Dd")); + Query dots; + dots.add("search", "x..y"); + p = RequestPlan::make(Call::kModelsList, PathArgs(), dots); + ASSERT_EQ(p.target, std::string("/api/models?search=x%2E%2Ey")); + ASSERT_TRUE(p.target.find("..") == std::string::npos); +} + +TEST(huggingface_paths, missing_and_unused_arguments_are_refused) { + PathArgs none; + ASSERT_THROW(RequestPlan::make(Call::kModelInfo, none, Query()), ApiError); + PathArgs extra; + extra.repo(RepoRef::parse("o/n")).set("surplus", "1"); + ASSERT_THROW(RequestPlan::make(Call::kModelInfo, extra, Query()), ApiError); +} + +TEST(huggingface_paths, caps_on_queries) { + Query q; + for (std::size_t i = 0; i < 32; ++i) q.add("filter", "x"); + ASSERT_THROW(q.add("filter", "x"), ApiError); + Query big; + ASSERT_THROW(big.add("search", std::string(2049, 'x')), ApiError); + ASSERT_THROW(big.add("search", std::string("a\r\nb")), ApiError); +} + +TEST(huggingface_paths, unsupported_calls_are_refused_with_a_reason) { + PathArgs a; + a.set("namespace", "acme").set("name", "ep"); + try { + RequestPlan::make(Call::kIeGetEndpointSse, a, Query()); + FAIL("expected a refusal"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 400); + ASSERT_TRUE(molo::huggingface::Calls::spec_of(Call::kIeGetEndpointSse).has_flag("unsupported")); + } +} + +TEST(huggingface_paths, special_placeholders) { + PathArgs a; + a.set("webhookId", "abc123").set("action", "enable"); + ASSERT_EQ(RequestPlan::make(Call::kWebhookToggle, a, Query()).target, std::string("/api/settings/webhooks/abc123/enable")); + a.set("action", "explode"); + ASSERT_THROW(RequestPlan::make(Call::kWebhookToggle, a, Query()), ApiError); + PathArgs d; + d.repo(RepoRef::parse("o/n")).repo_type(RepoType::kSpace).set("num", "12"); + ASSERT_EQ(RequestPlan::make(Call::kDiscussionGet, d, Query()).target, std::string("/api/spaces/o/n/discussions/12")); + d.set("num", "12x"); + ASSERT_THROW(RequestPlan::make(Call::kDiscussionGet, d, Query()), ApiError); + ASSERT_EQ(RequestPlan::check_model("gpt2"), std::string("gpt2")); + ASSERT_THROW(RequestPlan::check_model("a/b/c"), ApiError); + ASSERT_THROW(RequestPlan::check_model(""), ApiError); +} diff --git a/tests/src/vendors/huggingface/test_community.cpp b/tests/src/vendors/huggingface/test_community.cpp new file mode 100644 index 0000000..fd994e9 --- /dev/null +++ b/tests/src/vendors/huggingface/test_community.cpp @@ -0,0 +1,135 @@ +// Discussions, collections, webhooks: validation and the exact requests built. +#include "support.h" +#include "vendors/huggingface/community.h" + +using namespace molo::huggingface; +using namespace hf_test; +using molo::llm::ApiError; + +namespace { +// RECORDED 2026-10-25: GET /api/collections/nvidia/flora-6ad6a8986320a55467825cf5 (first fields) +const char* kCollection = + "{\"slug\":\"nvidia/flora-6ad6a8986320a55467825cf5\",\"title\":\"Flora\",\"gating\":false," + "\"owner\":{\"name\":\"nvidia\",\"type\":\"org\"}}"; +} // namespace + +TEST(huggingface_community, discussion_reads_are_public_and_writes_need_a_token) { + Rig anon(""); + Discussions d(anon.platform.hub); + anon.hub->expect_response(Method::kGet, "/api/models/o/n/discussions", reply("{\"discussions\":[]}")); + DiscussionFilter f; + f.status = "open"; + f.page = 2; + d.list(RepoType::kModel, RepoRef::parse("o/n"), f); + ASSERT_EQ(anon.hub->seen().back().target, std::string("/api/models/o/n/discussions?p=2&status=open")); + try { + d.create(RepoType::kModel, RepoRef::parse("o/n"), "A title", "body", false); + FAIL("expected NOT_CONFIGURED"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED")); + } + ASSERT_EQ(anon.hub->send_count(), static_cast(1)); +} + +TEST(huggingface_community, discussion_writes_build_the_documented_bodies) { + Rig rig; + Discussions d(rig.platform.hub); + RepoRef r = RepoRef::parse("o/n"); + rig.hub->expect_response(Method::kPost, "/api/datasets/o/n/discussions", reply("{\"num\":7}")); + d.create(RepoType::kDataset, r, "Typo in card", "details", true); + ASSERT_EQ(rig.hub->seen().back().body, std::string("{\"description\":\"details\",\"pullRequest\":true,\"title\":\"Typo in card\"}")); + rig.hub->expect_response(Method::kPost, "/api/datasets/o/n/discussions/7/status", reply("{}")); + d.change_status(RepoType::kDataset, r, 7, "closed", "done"); + ASSERT_EQ(rig.hub->seen().back().body, std::string("{\"comment\":\"done\",\"status\":\"closed\"}")); + rig.hub->expect_response(Method::kDelete, "/api/datasets/o/n/discussions/7", reply("{}")); + d.remove_discussion(RepoType::kDataset, r, 7); + ASSERT_THROW(d.create(RepoType::kModel, r, "ab", "x", false), ApiError); // title 3..200 + ASSERT_THROW(d.change_status(RepoType::kModel, r, 1, "maybe"), ApiError); + ASSERT_THROW(d.comment(RepoType::kModel, r, 0, "x"), ApiError); + ASSERT_THROW(d.comment(RepoType::kModel, r, 1, ""), ApiError); +} + +TEST(huggingface_community, collections_read_and_write) { + Rig rig; + Collections c(rig.platform.hub); + rig.hub->expect_response(Method::kGet, "/api/collections/nvidia/flora-6ad6a8986320a55467825cf5", reply(kCollection)); + Json got = c.get("nvidia", "flora-6ad6a8986320a55467825cf5"); + ASSERT_EQ(got.find("title")->as_string(), std::string("Flora")); + rig.hub->expect_response(Method::kPost, "/api/collections", reply("{\"slug\":\"me/x-1\"}")); + c.create("me", "My list", "d", true); + ASSERT_EQ(rig.hub->seen().back().body, + std::string("{\"description\":\"d\",\"namespace\":\"me\",\"private\":true,\"title\":\"My list\"}")); + ASSERT_THROW(c.create("me", "", "", false), ApiError); + ASSERT_THROW(c.create("me", std::string(61, 'x'), "", false), ApiError); + ASSERT_THROW(c.create("me", "t", std::string(151, 'x'), false), ApiError); + CollectionItem bad_item; + bad_item.type = "video"; + bad_item.id = "x"; + ASSERT_THROW(c.add_item("me", "x-1", bad_item), ApiError); + CollectionItem item; + item.type = "model"; + item.id = "openai-community/gpt2"; + rig.hub->expect_response(Method::kPost, "/api/collections/me/x-1/items", reply("{}")); + c.add_item("me", "x-1", item, "note"); + ASSERT_EQ(rig.hub->seen().back().body, + std::string("{\"item\":{\"id\":\"openai-community/gpt2\",\"type\":\"model\"},\"note\":\"note\"}")); + ASSERT_THROW(c.update("me", "x-1", Json::object()), ApiError); +} + +TEST(huggingface_community, collection_list_walks_with_item_filter) { + Rig rig(""); + Collections c(rig.platform.hub); + rig.hub->expect_response(Method::kGet, "/api/collections", reply("[{\"slug\":\"a/b-1\"}]")); + CollectionListParams p; + p.owner = "nvidia"; + p.limit = 5; + p.item.type = "model"; + p.item.id = "openai-community/gpt2"; + Page page = c.list(p, 10, 2); + ASSERT_EQ(page.items.as_array().size(), static_cast(1)); + ASSERT_EQ(rig.hub->seen().back().target, std::string("/api/collections?owner=nvidia&limit=5&item=model%2Fopenai-community%2Fgpt2")); +} + +TEST(huggingface_community, webhook_spec_is_validated_before_anything_is_sent) { + Rig rig; + Webhooks w(rig.platform.hub); + WebhookSpec s; + s.url = "https://example.com/hook"; + s.watched.push_back({"model", "openai-community/gpt2"}); + s.domains = {"repo"}; + rig.hub->expect_response(Method::kPost, "/api/settings/webhooks", reply("{\"webhook\":{\"id\":\"w1\"}}")); + w.create(s); + ASSERT_EQ(rig.hub->seen().back().body, + std::string("{\"domains\":[\"repo\"],\"url\":\"https://example.com/hook\",\"watched\":[{\"name\":\"openai-community/gpt2\",\"type\":\"model\"}]}")); + WebhookSpec bad = s; + bad.url = "ftp://x/y/z/w"; + ASSERT_THROW(w.create(bad), ApiError); + bad = s; + bad.watched.clear(); + ASSERT_THROW(w.create(bad), ApiError); + bad = s; + bad.watched[0].type = "planet"; + ASSERT_THROW(w.create(bad), ApiError); + bad = s; + bad.domains = {"everything"}; + ASSERT_THROW(w.create(bad), ApiError); + bad = s; + bad.secret = "caf\xc3\xa9"; + ASSERT_THROW(w.create(bad), ApiError); + ASSERT_EQ(rig.hub->send_count(), static_cast(1)); +} + +TEST(huggingface_community, webhook_lifecycle_paths) { + Rig rig; + Webhooks w(rig.platform.hub); + rig.hub->expect_response(Method::kPost, "/api/settings/webhooks/w1/enable", reply("{}")); + rig.hub->expect_response(Method::kPost, "/api/settings/webhooks/w1/disable", reply("{}")); + rig.hub->expect_response(Method::kPost, "/api/settings/webhooks/w1/replay/l9", reply("{}")); + rig.hub->expect_response(Method::kDelete, "/api/settings/webhooks/w1", reply("{}")); + w.enable("w1"); + w.disable("w1"); + w.replay("w1", "l9"); + w.remove_webhook("w1"); + rig.hub->verify(); + ASSERT_THROW(w.get("../x"), ApiError); +} diff --git a/tests/src/vendors/huggingface/test_hub.cpp b/tests/src/vendors/huggingface/test_hub.cpp new file mode 100644 index 0000000..1a8e6d6 --- /dev/null +++ b/tests/src/vendors/huggingface/test_hub.cpp @@ -0,0 +1,229 @@ +// Hub reads and the shared service rules (token, budget, paging) through the +// real ApiClient over a mock transport. Replies are RECORDED trims or SHAPE. +#include "support.h" +#include "vendors/huggingface/hub.h" +#include "vendors/huggingface/platform.h" + +using namespace molo::huggingface; +using namespace hf_test; +using molo::llm::ApiError; + +namespace { +// RECORDED 2026-10-25: GET https://huggingface.co/api/models/openai-community/gpt2/refs (no token) +const char* kRefs = + "{\"tags\":[],\"branches\":[{\"name\":\"main\",\"ref\":\"refs/heads/main\"," + "\"targetCommit\":\"607a30d783dfa663caf39e06633721c8d4cfcd7e\"}],\"converts\":[]}"; +// RECORDED 2026-10-25: GET /api/models/openai-community/gpt2/tree/main (first 3 entries) +const char* kTree = + "[{\"type\":\"directory\",\"oid\":\"d03ec5ec179df58241d27d55f92a674f5f44197f\",\"size\":0,\"path\":\"onnx\"}," + "{\"type\":\"file\",\"oid\":\"602b71f15d40ed68c5f96330e3f3175a76a32126\",\"size\":445,\"path\":\".gitattributes\"}]"; +// RECORDED 2026-10-25: GET /api/whoami-v2 without a token -> 401 +const char* kUnauth = "{\"error\":\"Invalid username or password.\"}"; +// RECORDED 2026-10-25: GET /api/models?limit=1 (trimmed fields) +const char* kModel = + "{\"_id\":\"621ffdc036468d709f17434d\",\"id\":\"openai-community/gpt2\",\"likes\":3891,\"private\":false," + "\"downloads\":20867408,\"pipeline_tag\":\"text-generation\",\"library_name\":\"transformers\"}"; + +ClientResponse paged(const std::string& body, const std::string& next_cursor) { + HeaderMap h; + h.set("Content-Type", "application/json"); + if (!next_cursor.empty()) + h.set("Link", "; rel=\"next\""); + return ClientResponse(200, h, body); +} +} // namespace + +TEST(huggingface_hub, anonymous_reads_send_no_authorization) { + Rig rig(""); + rig.hub->expect_response(Method::kGet, "/api/models/openai-community/gpt2/refs", reply(kRefs)); + Hub hub = Hub::from_platform(rig.platform); + ASSERT_FALSE(hub.has_token()); + Json refs = hub.refs(RepoType::kModel, RepoRef::parse("openai-community/gpt2")); + ASSERT_EQ(refs.find("branches")->as_array().front().find("name")->as_string(), std::string("main")); + auto seen = rig.hub->seen(); + ASSERT_EQ(seen.size(), static_cast(1)); + ASSERT_EQ(Rig::header(seen[0], "Authorization"), std::string("")); + ASSERT_EQ(Rig::header(seen[0], "X-Molo-Anonymous"), std::string("1")); +} + +TEST(huggingface_hub, token_travels_as_bearer_only) { + Rig rig; + rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs)); + Hub hub = Hub::from_platform(rig.platform); + hub.refs(RepoType::kModel, RepoRef::parse("o/n"), true); + auto seen = rig.hub->seen(); + ASSERT_EQ(Rig::header(seen[0], "Authorization"), std::string("Bearer ") + kToken); + ASSERT_TRUE(seen[0].target.find(kToken) == std::string::npos); + ASSERT_EQ(seen[0].target, std::string("/api/models/o/n/refs?include_prs=true")); +} + +TEST(huggingface_hub, whoami_without_token_fails_before_any_traffic) { + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + try { + hub.whoami(); + FAIL("expected NOT_CONFIGURED"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 503); + ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED")); + ASSERT_TRUE(has(e.message(), "HF_TOKEN")); + } + ASSERT_EQ(rig.hub->send_count(), static_cast(0)); +} + +TEST(huggingface_hub, hub_error_envelope_is_kept) { + Rig rig; + rig.hub->expect_response(Method::kGet, "/api/whoami-v2", reply(kUnauth, 401)); + Hub hub = Hub::from_platform(rig.platform); + try { + hub.whoami(); + FAIL("expected 401"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 401); + ASSERT_EQ(e.message(), std::string("Invalid username or password.")); + ASSERT_TRUE(!has(e.message(), kToken)); + } +} + +TEST(huggingface_hub, list_walks_link_cursors_and_reports_page_truncation) { + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + // Three pages exist; the walk is capped at two pages. + rig.hub->expect_response(Method::kGet, "/api/models", paged(std::string("[") + kModel + "]", "AAA")); + rig.hub->expect_response(Method::kGet, "/api/models", paged(std::string("[") + kModel + "]", "BBB")); + RepoListParams p; + p.limit = 1; + p.sort = "downloads"; + Page page = hub.list_repos(RepoType::kModel, p, 100, 2); + ASSERT_EQ(page.items.as_array().size(), static_cast(2)); + ASSERT_TRUE(page.truncated); + ASSERT_EQ(page.truncated_by, std::string("pages")); + ASSERT_EQ(page.next_cursor, std::string("BBB")); + auto seen = rig.hub->seen(); + ASSERT_EQ(seen[0].target, std::string("/api/models?sort=downloads&limit=1")); + ASSERT_EQ(seen[1].target, std::string("/api/models?sort=downloads&limit=1&cursor=AAA")); +} + +TEST(huggingface_hub, list_reports_item_truncation_and_clean_end) { + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + rig.hub->expect_response(Method::kGet, "/api/datasets", + paged(std::string("[") + kModel + "," + kModel + "," + kModel + "]", "")); + Page cut = hub.list_repos(RepoType::kDataset, RepoListParams(), 2, 5); + ASSERT_EQ(cut.items.as_array().size(), static_cast(2)); + ASSERT_TRUE(cut.truncated); + ASSERT_EQ(cut.truncated_by, std::string("items")); + rig.hub->expect_response(Method::kGet, "/api/spaces", paged(std::string("[") + kModel + "]", "")); + Page done = hub.list_repos(RepoType::kSpace, RepoListParams(), 10, 5); + ASSERT_FALSE(done.truncated); + ASSERT_EQ(done.pages, static_cast(1)); +} + +TEST(huggingface_hub, link_header_to_another_host_is_refused) { + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + HeaderMap h; + h.set("Content-Type", "application/json"); + h.set("Link", "; rel=\"next\""); + rig.hub->expect_response(Method::kGet, "/api/models", ClientResponse(200, h, "[]")); + ASSERT_THROW(hub.list_repos(RepoType::kModel, RepoListParams()), ApiError); +} + +TEST(huggingface_hub, link_header_parsing) { + ASSERT_EQ(next_cursor_of("; rel=\"next\"", "huggingface.co"), + std::string("abc=")); + ASSERT_EQ(next_cursor_of("; rel=\"prev\"", "huggingface.co"), std::string("")); + ASSERT_EQ(next_cursor_of("", "huggingface.co"), std::string("")); + ASSERT_EQ(next_cursor_of("; rel=\"prev\", ; rel=\"next\"", + "huggingface.co"), + std::string("b")); +} + +TEST(huggingface_hub, list_params_are_validated) { + RepoListParams p; + p.limit = 1001; + ASSERT_THROW(p.to_query(), ApiError); + p.limit = -1; + ASSERT_THROW(p.to_query(), ApiError); + RepoListParams q; + q.pipeline_tag = "text-generation"; + q.expand = {"downloads", "likes"}; + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + // pipeline_tag is a models-only filter: the datasets row does not list it. + ASSERT_THROW(hub.list_repos(RepoType::kDataset, q), ApiError); + ASSERT_EQ(rig.hub->send_count(), static_cast(0)); +} + +TEST(huggingface_hub, tree_paths_info_and_download) { + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + rig.hub->expect_response(Method::kGet, "/api/models/o/n/tree/main", reply(kTree)); + Json t = hub.tree(RepoType::kModel, RepoRef::parse("o/n"), "main"); + ASSERT_EQ(t.as_array().size(), static_cast(2)); + + rig.hub->expect_response(Method::kPost, "/api/models/o/n/paths-info/main", + reply("[{\"type\":\"file\",\"oid\":\"10c6\",\"size\":665,\"path\":\"config.json\"}]")); + hub.paths_info(RepoType::kModel, RepoRef::parse("o/n"), "main", {"config.json"}); + auto seen = rig.hub->seen(); + ASSERT_EQ(seen.back().body, std::string("{\"paths\":[\"config.json\"]}")); + ASSERT_THROW(hub.paths_info(RepoType::kModel, RepoRef::parse("o/n"), "main", {}), ApiError); + ASSERT_THROW(hub.paths_info(RepoType::kModel, RepoRef::parse("o/n"), "main", {"../x"}), ApiError); + + HeaderMap h; + h.set("Content-Type", "text/plain; charset=utf-8"); + h.set("X-Repo-Commit", "607a30d7"); + h.set("ETag", "\"abc\""); + rig.hub->expect_response(Method::kGet, "/o/n/resolve/main/config.json", ClientResponse(200, h, "{\"a\":1}")); + Download d = hub.download(RepoType::kModel, RepoRef::parse("o/n"), "main", "config.json"); + ASSERT_EQ(d.bytes, std::string("{\"a\":1}")); + ASSERT_EQ(d.commit, std::string("607a30d7")); + ASSERT_EQ(d.etag, std::string("\"abc\"")); +} + +TEST(huggingface_hub, search_and_catalogs_validate_input) { + Rig rig(""); + Hub hub = Hub::from_platform(rig.platform); + ASSERT_THROW(hub.quicksearch(""), ApiError); + ASSERT_THROW(hub.quicksearch("bert", 5000), ApiError); + ASSERT_THROW(hub.tags_by_type(RepoType::kSpace), ApiError); + ASSERT_THROW(hub.paper("../x"), ApiError); + ASSERT_THROW(hub.partner_models("a/b"), ApiError); + rig.hub->expect_response(Method::kGet, "/api/quicksearch", reply("{\"models\":[],\"datasets\":[],\"spaces\":[]}")); + hub.quicksearch("bert", 2, "model"); + ASSERT_EQ(rig.hub->seen().back().target, std::string("/api/quicksearch?q=bert&limit=2&type=model")); +} + +TEST(huggingface_budget, spends_down_and_resets_with_the_window) { + long now = 1000; + auto budget = std::make_shared(2, 300, [&] { return now; }); + Rig rig("", budget); + Hub hub = Hub::from_platform(rig.platform); + rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs)); + rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs)); + rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs)); + RepoRef r = RepoRef::parse("o/n"); + hub.refs(RepoType::kModel, r); + hub.refs(RepoType::kModel, r); + try { + hub.refs(RepoType::kModel, r); + FAIL("expected the local budget to stop the call"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 429); + ASSERT_EQ(e.type(), std::string("LOCAL_BUDGET")); + } + ASSERT_EQ(rig.hub->send_count(), static_cast(2)); + now += 301; + hub.refs(RepoType::kModel, r); + ASSERT_EQ(rig.hub->send_count(), static_cast(3)); +} + +TEST(huggingface_platform, server_urls_are_validated) { + ASSERT_EQ(Platform::check_server_url("http://127.0.0.1:8080/"), std::string("http://127.0.0.1:8080")); + ASSERT_EQ(Platform::check_server_url("https://x.endpoints.huggingface.cloud"), std::string("https://x.endpoints.huggingface.cloud")); + ASSERT_THROW(Platform::check_server_url(""), ApiError); + ASSERT_THROW(Platform::check_server_url("ftp://x"), ApiError); + ASSERT_THROW(Platform::check_server_url("http://user:pw@host"), ApiError); + ASSERT_THROW(Platform::check_server_url("http://host/?a=b"), ApiError); + ASSERT_THROW(Platform::check_server_url("not a url"), ApiError); +} diff --git a/tests/src/vendors/huggingface/test_live.cpp b/tests/src/vendors/huggingface/test_live.cpp new file mode 100644 index 0000000..57927ee --- /dev/null +++ b/tests/src/vendors/huggingface/test_live.cpp @@ -0,0 +1,136 @@ +// Opt-in live checks. MOLO_LIVE_NET=1 enables the network. Public Hub reads, +// the router model list and the Inference Endpoints vendor list need NO key +// and run under MOLO_LIVE_NET=1 alone. Token-gated reads also need HF_TOKEN +// (via llm::Config); the one tiny chat completion additionally needs +// MOLO_LIVE_PAID=1. Without them each test prints a loud SKIP. Nothing is +// created, changed or deleted. +#include +#include + +#include "clients/transports/http_transport.h" +#include "support.h" +#include "vendors/huggingface/chat_model.h" +#include "vendors/huggingface/community.h" +#include "vendors/huggingface/endpoints.h" +#include "vendors/huggingface/hub.h" +#include "vendors/huggingface/providers.h" + +using namespace molo::huggingface; +using molo::llm::ApiError; + +namespace { +bool net() { return std::getenv("MOLO_LIVE_NET") != nullptr; } +bool paid() { return std::getenv("MOLO_LIVE_PAID") != nullptr; } +Platform anonymous_platform() { + return Platform::with(std::make_unique(), + std::make_unique(), + std::make_unique(), ""); +} +} // namespace + +TEST(huggingface_live, public_hub_reads_without_a_key) { + if (!net()) { + std::cout << "SKIP huggingface_live.public_hub_reads_without_a_key (set MOLO_LIVE_NET=1; no key needed)\n"; + return; + } + Platform p = anonymous_platform(); + Hub hub = Hub::from_platform(p); + ASSERT_FALSE(hub.has_token()); + RepoRef gpt2 = RepoRef::parse("openai-community/gpt2"); + + RepoListParams params; + params.limit = 2; + params.sort = "downloads"; + Page models = hub.list_repos(RepoType::kModel, params, 4, 2); + std::cout << " models listed: " << models.items.as_array().size() << " pages=" << models.pages + << " truncated=" << models.truncated << "\n"; + ASSERT_TRUE(models.items.as_array().size() >= 2); + + Json info = hub.repo_info(RepoType::kModel, gpt2); + ASSERT_EQ(info.find("id")->as_string(), std::string("openai-community/gpt2")); + Json refs = hub.refs(RepoType::kModel, gpt2); + ASSERT_TRUE(refs.find("branches") != nullptr); + Json tree = hub.tree(RepoType::kModel, gpt2, "main"); + ASSERT_TRUE(tree.is_array() && !tree.as_array().empty()); + Download cfg = hub.download(RepoType::kModel, gpt2, "main", "config.json"); + std::cout << " config.json: " << cfg.bytes.size() << " bytes commit=" << cfg.commit << "\n"; + ASSERT_TRUE(cfg.bytes.find("GPT2LMHeadModel") != std::string::npos); + Json ds = hub.repo_info(RepoType::kDataset, RepoRef::parse("rajpurkar/squad")); + ASSERT_EQ(ds.find("id")->as_string(), std::string("rajpurkar/squad")); + Json sp = hub.repo_info(RepoType::kSpace, RepoRef::parse("gradio/hello_world")); + ASSERT_EQ(sp.find("id")->as_string(), std::string("gradio/hello_world")); + Json search = hub.quicksearch("bert", 2); + ASSERT_TRUE(search.is_object()); + Json hw = hub.spaces_hardware(); + ASSERT_TRUE(hw.is_array()); + Json trending = hub.trending("model", 3); + ASSERT_TRUE(trending.is_object()); + Json mapping = hub.partner_models("groq"); + ASSERT_TRUE(mapping.is_object()); + + Discussions d(p.hub); + Json disc = d.list(RepoType::kModel, gpt2); + ASSERT_TRUE(disc.is_object()); + Collections c(p.hub); + Json coll = c.get("nvidia", "flora-6ad6a8986320a55467825cf5"); + ASSERT_EQ(coll.find("title")->as_string(), std::string("Flora")); +} + +TEST(huggingface_live, router_models_and_endpoint_vendors_without_a_key) { + if (!net()) { + std::cout << "SKIP huggingface_live.router_models_and_endpoint_vendors_without_a_key (set MOLO_LIVE_NET=1; no key needed)\n"; + return; + } + Platform p = anonymous_platform(); + Json models = RouterChat::from_platform(p).models(); + std::cout << " router models: " << models.find("data")->as_array().size() << "\n"; + ASSERT_TRUE(models.find("data")->as_array().size() > 10); + Json vendors = Endpoints::from_platform(p).vendors(); + ASSERT_TRUE(vendors.find("vendors")->as_array().size() >= 1); +} + +TEST(huggingface_live, invalid_token_answers_401_on_whoami_and_endpoints) { + if (!net()) { + std::cout << "SKIP huggingface_live.invalid_token_answers_401_on_whoami_and_endpoints (set MOLO_LIVE_NET=1)\n"; + return; + } + Platform p = Platform::with(std::make_unique(), + std::make_unique(), + std::make_unique(), "hf_invalid_molo_probe"); + try { + Hub::from_platform(p).whoami(); + FAIL("expected 401"); + } catch (const ApiError& e) { + std::cout << " GET /api/whoami-v2 with an invalid token: " << e.http_status() << " " << e.message() << "\n"; + ASSERT_EQ(e.http_status(), 401); + } + try { + Endpoints::from_platform(p).list("huggingface"); + FAIL("expected 401"); + } catch (const ApiError& e) { + std::cout << " GET /v2/endpoint/huggingface with an invalid token: " << e.http_status() << " " << e.message() << "\n"; + ASSERT_EQ(e.http_status(), 401); + } +} + +TEST(huggingface_live, token_reads_and_one_tiny_completion) { + Platform p = Platform::from_env(); + if (!net() || !p.configured()) { + std::cout << "SKIP huggingface_live.token_reads_and_one_tiny_completion (set MOLO_LIVE_NET=1 and HF_TOKEN)\n"; + return; + } + Json who = Hub::from_platform(p).whoami(); + ASSERT_TRUE(who.find("name") != nullptr); + std::cout << " whoami ok\n"; + if (!paid()) { + std::cout << "SKIP huggingface_live.token_reads_and_one_tiny_completion chat part (also set MOLO_LIVE_PAID=1)\n"; + return; + } + auto model = HuggingfaceChatModel::over_router(p, "openai/gpt-oss-20b"); + molo::llm::ChatRequest req; + req.messages.push_back(molo::llm::Message::user("Reply with the single word: ok")); + req.max_tokens = 16; + molo::llm::ChatResponse r = model->generate(req); + std::cout << " completion finish=" << r.finish_reason << " tokens=" << r.usage.total_tokens << "\n"; + ASSERT_TRUE(r.usage.total_tokens > 0); +} diff --git a/tests/src/vendors/huggingface/test_providers.cpp b/tests/src/vendors/huggingface/test_providers.cpp new file mode 100644 index 0000000..aa16b8c --- /dev/null +++ b/tests/src/vendors/huggingface/test_providers.cpp @@ -0,0 +1,330 @@ +// Inference Providers router, hf-inference tasks, the chat wire and +// HuggingfaceChatModel. Replies are SHAPE (documented schema) or RECORDED. +#include "support.h" +#include "vendors/huggingface/chat_model.h" +#include "vendors/huggingface/providers.h" + +using namespace molo::huggingface; +using namespace hf_test; +using molo::llm::ApiError; + +namespace { +// SHAPE: chat completion per huggingface.js tasks/chat-completion/spec/output.json +const char* kChat = + "{\"id\":\"cmpl-1\",\"model\":\"openai/gpt-oss-20b\",\"created\":1,\"object\":\"chat.completion\"," + "\"choices\":[{\"index\":0,\"finish_reason\":\"stop\",\"message\":{\"role\":\"assistant\",\"content\":\"Hello!\"," + "\"reasoning\":\"greeting\"}}],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":3,\"total_tokens\":12," + "\"completion_tokens_details\":{\"reasoning_tokens\":2}}}"; +const char* kToolReply = + "{\"id\":\"cmpl-2\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":{\"role\":\"assistant\"," + "\"content\":null,\"tool_calls\":[{\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\"," + "\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}},{\"id\":\"call_2\",\"type\":\"function\",\"function\":{\"name\":\"f\"," + "\"arguments\":\"{broken\"}}]}}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}"; +// SHAPE: chat.completion.chunk stream, as OpenAI-compatible providers send it +const char* kStream = + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"He\"}}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"llo\"}}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\"\"}}]}}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\":1}\"}}]}}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":2,\"total_tokens\":6}}\n\n" + "data: [DONE]\n\n"; +// RECORDED 2026-10-25: GET https://router.huggingface.co/v1/models (first entry, 2 of its providers) +const char* kRouterModels = + "{\"object\":\"list\",\"data\":[{\"id\":\"deepseek-ai/DeepSeek-V4.1-Flash-1017\",\"object\":\"model\",\"created\":1792204838," + "\"owned_by\":\"deepseek-ai\",\"providers\":[{\"provider\":\"novita\",\"status\":\"live\",\"context_length\":1048576," + "\"supports_tools\":true}]}]}"; + +struct FakeEndpoint : ChatEndpoint { + Json last_body; + std::string reply_json; + std::string stream_text; + Json create(const Json& body) override { + last_body = body; + return Json::parse(reply_json); + } + void stream(const Json& body, const molo::llm::SseDecoder::Sink& sink) override { + last_body = body; + molo::llm::SseDecoder dec("huggingface", sink); + dec.feed(stream_text); + dec.finish(); + } +}; +} // namespace + +TEST(huggingface_router, model_selectors) { + RouterModel m = RouterModel::parse("openai/gpt-oss-20b:fastest"); + ASSERT_EQ(m.id, std::string("openai/gpt-oss-20b")); + ASSERT_EQ(m.selector, std::string("fastest")); + ASSERT_EQ(m.str(), std::string("openai/gpt-oss-20b:fastest")); + ASSERT_EQ(RouterModel::parse("a/b").selector, std::string("")); + ASSERT_THROW(RouterModel::parse("a/b:x:y"), ApiError); + ASSERT_THROW(RouterModel::parse("a/b:"), ApiError) ; + ASSERT_THROW(RouterModel::parse("a b"), ApiError); + ASSERT_THROW(RouterModel::parse(""), ApiError); +} + +TEST(huggingface_router, chat_request_and_bill_to_header) { + Rig rig; + RouterChat chat = RouterChat::from_platform(rig.platform); + chat.set_bill_to("acme"); + rig.router->expect_response(Method::kPost, "/v1/chat/completions", reply(kChat)); + Json body = Json::object(); + body.set("model", Json("openai/gpt-oss-20b:cheapest")); + Json msgs = Json::array(); + Json m = Json::object(); + m.set("role", Json("user")); + m.set("content", Json("hi")); + msgs.push(m); + body.set("messages", msgs); + Json out = chat.create(body); + ASSERT_EQ(out.find("id")->as_string(), std::string("cmpl-1")); + auto seen = rig.router->seen(); + ASSERT_EQ(Rig::header(seen[0], "X-HF-Bill-To"), std::string("acme")); + ASSERT_EQ(Rig::header(seen[0], "Authorization"), std::string("Bearer ") + kToken); + ASSERT_THROW(chat.set_bill_to("a b"), ApiError); +} + +TEST(huggingface_router, chat_refuses_bad_bodies_before_sending) { + Rig rig; + RouterChat chat = RouterChat::from_platform(rig.platform); + Json no_model = Json::object(); + ASSERT_THROW(chat.create(no_model), ApiError); + Json unknown = Json::object(); + unknown.set("model", Json("a/b")); + unknown.set("provider_secret_sauce", Json(1)); + ASSERT_THROW(chat.create(unknown), ApiError); + Json streaming = Json::object(); + streaming.set("model", Json("a/b")); + streaming.set("stream", Json(true)); + ASSERT_THROW(chat.create(streaming), ApiError); + ASSERT_EQ(rig.router->send_count(), static_cast(0)); +} + +TEST(huggingface_router, chat_without_token_is_not_configured) { + Rig rig(""); + RouterChat chat = RouterChat::from_platform(rig.platform); + Json b = Json::object(); + b.set("model", Json("a/b")); + try { + chat.create(b); + FAIL("expected NOT_CONFIGURED"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED")); + } + ASSERT_EQ(rig.router->send_count(), static_cast(0)); +} + +TEST(huggingface_router, models_list_is_public) { + Rig rig(""); + RouterChat chat = RouterChat::from_platform(rig.platform); + rig.router->expect_response(Method::kGet, "/v1/models", reply(kRouterModels)); + Json m = chat.models(); + ASSERT_EQ(m.find("data")->as_array().front().find("owned_by")->as_string(), std::string("deepseek-ai")); + ASSERT_EQ(Rig::header(rig.router->seen()[0], "Authorization"), std::string("")); +} + +TEST(huggingface_tasks, catalog_matches_the_hf_inference_map) { + ASSERT_EQ(TaskCatalog::count(), static_cast(25)); + ASSERT_TRUE(TaskCatalog::find("feature-extraction")->pipeline_path); + ASSERT_TRUE(TaskCatalog::find("sentence-similarity")->pipeline_path); + ASSERT_TRUE(TaskCatalog::find("image-classification")->binary_input); + ASSERT_FALSE(TaskCatalog::find("translation")->binary_input); + ASSERT_TRUE(TaskCatalog::find("conversational") == nullptr); // chat has its own surface +} + +TEST(huggingface_tasks, json_task_request) { + Rig rig; + InferenceTasks t = InferenceTasks::from_platform(rig.platform); + rig.router->expect_response(Method::kPost, "/hf-inference/models/distilbert/distilbert-base-uncased-finetuned-sst-2-english", + reply("[[{\"label\":\"POSITIVE\",\"score\":0.99}]]")); + Json params = Json::object(); + params.set("top_k", Json(1)); + InferenceResult r = t.run_json("text-classification", "distilbert/distilbert-base-uncased-finetuned-sst-2-english", + Json("I like you"), params); + ASSERT_TRUE(r.is_json); + ASSERT_EQ(rig.router->seen()[0].body, std::string("{\"inputs\":\"I like you\",\"parameters\":{\"top_k\":1}}")); +} + +TEST(huggingface_tasks, pipeline_tasks_put_parameters_at_the_top_level) { + Rig rig; + InferenceTasks t = InferenceTasks::from_platform(rig.platform); + rig.router->expect_response(Method::kPost, "/hf-inference/models/BAAI/bge-small-en-v1.5/pipeline/feature-extraction", + reply("[[0.1,0.2]]")); + Json e = t.embed("BAAI/bge-small-en-v1.5", {"hello"}, true); + ASSERT_TRUE(e.is_array()); + ASSERT_EQ(rig.router->seen()[0].body, std::string("{\"inputs\":\"hello\",\"normalize\":true}")); + ASSERT_THROW(t.embed("BAAI/bge-small-en-v1.5", {}), ApiError); + ASSERT_THROW(t.embed("BAAI/bge-small-en-v1.5", std::vector(257, "x")), ApiError); +} + +TEST(huggingface_tasks, binary_tasks_send_raw_bytes_or_base64_with_parameters) { + Rig rig; + InferenceTasks t = InferenceTasks::from_platform(rig.platform); + rig.router->expect_response(Method::kPost, "/hf-inference/models/google/vit-base-patch16-224", reply("[{\"label\":\"cat\",\"score\":0.9}]")); + rig.router->expect_response(Method::kPost, "/hf-inference/models/google/vit-base-patch16-224", reply("[]")); + t.run_binary("image-classification", "google/vit-base-patch16-224", "\x89PNG", "image/png"); + auto first = rig.router->seen()[0]; + ASSERT_EQ(first.body, std::string("\x89PNG")); + ASSERT_EQ(Rig::header(first, "Content-Type"), std::string("image/png")); + Json p = Json::object(); + p.set("top_k", Json(2)); + t.run_binary("image-classification", "google/vit-base-patch16-224", "abc", "image/png", p); + ASSERT_EQ(rig.router->seen()[1].body, std::string("{\"inputs\":\"YWJj\",\"parameters\":{\"top_k\":2}}")); + ASSERT_THROW(t.run_json("image-classification", "m", Json("x")), ApiError); + ASSERT_THROW(t.run_binary("translation", "m", "x", "text/plain"), ApiError); + ASSERT_THROW(t.run_binary("image-classification", "m", "", "image/png"), ApiError); + ASSERT_THROW(t.run_json("no-such-task", "m", Json("x")), ApiError); +} + +TEST(huggingface_tasks, binary_replies_are_returned_as_bytes) { + Rig rig; + InferenceTasks t = InferenceTasks::from_platform(rig.platform); + rig.router->expect_response(Method::kPost, "/hf-inference/models/black-forest-labs/FLUX.1-dev", + reply("\x89PNG\r\n", 200, "image/png")); + InferenceResult r = t.run_json("text-to-image", "black-forest-labs/FLUX.1-dev", Json("a cat")); + ASSERT_FALSE(r.is_json); + ASSERT_EQ(r.content_type, std::string("image/png")); + ASSERT_EQ(r.bytes.size(), static_cast(6)); +} + +TEST(huggingface_chat, neutral_request_maps_to_the_wire_body) { + molo::llm::ChatRequest req; + req.model = "openai/gpt-oss-20b:fastest"; + req.system = "be brief"; + req.messages.push_back(molo::llm::Message::user("hi")); + req.max_tokens = 32; + req.temperature = 0.5; + req.stop = {"END"}; + req.json_mode = true; + req.reasoning_effort = "low"; + molo::llm::ToolSpec tool; + tool.name = "get_weather"; + tool.description = "weather"; + req.tools.push_back(tool); + req.tool_choice = "get_weather"; + Json b = ChatWire::to_body(req, "ignored", false); + ASSERT_EQ(b.find("model")->as_string(), std::string("openai/gpt-oss-20b:fastest")); + ASSERT_EQ(b.find("messages")->as_array().size(), static_cast(2)); + ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system")); + ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object")); + ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather")); + ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low")); + ASSERT_TRUE(b.find("stream") == nullptr); + Json s = ChatWire::to_body(req, "x", true); + ASSERT_TRUE(s.find("stream")->as_bool()); + ASSERT_TRUE(s.find("stream_options")->find("include_usage")->as_bool()); +} + +TEST(huggingface_chat, requests_are_validated) { + molo::llm::ChatRequest req; + req.messages.push_back(molo::llm::Message::user("hi")); + ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError); // no model anywhere + req.model = "a/b"; + req.stop = {"1", "2", "3", "4", "5"}; + ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError); + req.stop.clear(); + req.json_mode = true; + req.response_schema = Json::object(); + ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError); + req.json_mode = false; + req.response_schema = Json(); + req.temperature = 2.5; + ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError); + req.temperature.reset(); + req.max_tokens = 0; + ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError); + req.max_tokens.reset(); + req.extra = Json::object(); + req.extra.set("seed", Json(7)); + ASSERT_TRUE(ChatWire::to_body(req, "", false).find("seed") != nullptr); + molo::llm::ChatRequest empty; + empty.model = "a/b"; + ASSERT_THROW(ChatWire::to_body(empty, "", false), ApiError); // no messages +} + +TEST(huggingface_chat, response_parsing_with_reasoning_and_usage) { + molo::llm::ChatResponse r = ChatWire::to_response(Json::parse(kChat)); + ASSERT_EQ(r.text(), std::string("Hello!")); + ASSERT_EQ(r.finish_reason, std::string("stop")); + ASSERT_EQ(r.usage.input_tokens, 9); + ASSERT_EQ(r.usage.output_tokens, 3); + ASSERT_EQ(r.usage.reasoning_tokens, 2); + ASSERT_EQ(r.message.parts.front().reasoning, std::string("greeting")); + ASSERT_THROW(ChatWire::to_response(Json::parse("{\"choices\":[]}")), ApiError); +} + +TEST(huggingface_chat, tool_calls_keep_invalid_json_visible) { + molo::llm::ChatResponse r = ChatWire::to_response(Json::parse(kToolReply)); + std::vector calls = r.tool_calls(); + ASSERT_EQ(calls.size(), static_cast(2)); + ASSERT_EQ(calls[0].name, std::string("get_weather")); + ASSERT_EQ(calls[0].arguments.find("city")->as_string(), std::string("Paris")); + ASSERT_EQ(calls[1].arguments.find("_invalid_json")->as_string(), std::string("{broken")); + ASSERT_EQ(r.finish_reason, std::string("tool_calls")); +} + +TEST(huggingface_chat, model_over_a_fake_endpoint) { + auto ep = std::make_shared(); + ep->reply_json = kChat; + HuggingfaceChatModel model(ep, "openai/gpt-oss-20b"); + molo::llm::ChatRequest req; + req.messages.push_back(molo::llm::Message::user("hi")); + molo::llm::ChatResponse r = model.generate(req); + ASSERT_EQ(r.text(), std::string("Hello!")); + ASSERT_EQ(ep->last_body.find("model")->as_string(), std::string("openai/gpt-oss-20b")); + ASSERT_EQ(model.vendor(), std::string("huggingface")); + ASSERT_THROW(HuggingfaceChatModel(nullptr, "m"), ApiError); +} + +TEST(huggingface_chat, stream_transcript_and_early_stop) { + auto ep = std::make_shared(); + ep->stream_text = kStream; + HuggingfaceChatModel model(ep, "a/b"); + molo::llm::ChatRequest req; + req.messages.push_back(molo::llm::Message::user("hi")); + std::string text, args, id, finish; + int total = 0; + model.stream(req, [&](const molo::llm::ChatChunk& c) { + text += c.text_delta; + args += c.tool_args_delta; + if (!c.tool_call_id.empty()) id = c.tool_call_id; + if (!c.finish_reason.empty()) finish = c.finish_reason; + if (c.usage) total = c.usage->total_tokens; + return true; + }); + ASSERT_EQ(text, std::string("Hello")); + ASSERT_EQ(args, std::string("{\"a\":1}")); + ASSERT_EQ(id, std::string("call_1")); + ASSERT_EQ(finish, std::string("stop")); + ASSERT_EQ(total, 6); + ASSERT_TRUE(ep->last_body.find("stream")->as_bool()); + + int seen = 0; + model.stream(req, [&](const molo::llm::ChatChunk&) { return ++seen < 2; }); + ASSERT_EQ(seen, 2); +} + +TEST(huggingface_chat, error_event_in_stream_throws) { + auto ep = std::make_shared(); + ep->stream_text = "data: {\"error\":{\"message\":\"provider failed\"}}\n\n"; + HuggingfaceChatModel model(ep, "a/b"); + molo::llm::ChatRequest req; + req.messages.push_back(molo::llm::Message::user("hi")); + ASSERT_THROW(model.stream(req, [](const molo::llm::ChatChunk&) { return true; }), ApiError); +} + +TEST(huggingface_chat, router_model_streams_over_the_real_client) { + Rig rig; + rig.router->expect_response(Method::kPost, "/v1/chat/completions", sse(kStream)); + auto model = HuggingfaceChatModel::over_router(rig.platform, "a/b"); + molo::llm::ChatRequest req; + req.messages.push_back(molo::llm::Message::user("hi")); + std::string text; + model->stream(req, [&](const molo::llm::ChatChunk& c) { + text += c.text_delta; + return true; + }); + ASSERT_EQ(text, std::string("Hello")); + ASSERT_TRUE(has(rig.router->seen()[0].body, "\"stream\":true")); +} diff --git a/tests/src/vendors/huggingface/test_servers.cpp b/tests/src/vendors/huggingface/test_servers.cpp new file mode 100644 index 0000000..c5e899a --- /dev/null +++ b/tests/src/vendors/huggingface/test_servers.cpp @@ -0,0 +1,200 @@ +// Inference Endpoints management, TGI and TEI. Requests are checked against +// the call table; replies are SHAPE or RECORDED. +#include "support.h" +#include "vendors/huggingface/endpoints.h" +#include "vendors/huggingface/servers.h" + +using namespace molo::huggingface; +using namespace hf_test; +using molo::llm::ApiError; + +namespace { +// RECORDED 2026-10-25: GET https://api.endpoints.huggingface.cloud/v2/provider (no token), trimmed +const char* kVendors = + "{\"vendors\":[{\"name\":\"aws\",\"status\":\"available\",\"regions\":[{\"name\":\"eu-west-1\",\"label\":\"Ireland\"," + "\"status\":\"available\",\"computes\":[{\"id\":\"aws-eu-west-1-intel-spr-x1\",\"accelerator\":\"cpu\"," + "\"instanceType\":\"intel-spr\",\"instanceSize\":\"x1\",\"status\":\"available\",\"pricePerHour\":0.033}]}]}]}"; +// RECORDED 2026-10-25: any call without a valid token -> 401 +const char* kIeUnauth = "{\"error\":\"401 Unauthorized\",\"code\":\"UNAUTHORIZED\"}"; + +Json spec() { + return Json::parse( + "{\"name\":\"my-ep\",\"type\":\"protected\",\"provider\":{\"vendor\":\"aws\",\"region\":\"us-east-1\"}," + "\"compute\":{},\"model\":{}}"); +} +} // namespace + +TEST(huggingface_endpoints, vendors_list_is_public_everything_else_needs_a_token) { + Rig rig(""); + Endpoints ie = Endpoints::from_platform(rig.platform); + rig.endpoints->expect_response(Method::kGet, "/v2/provider", reply(kVendors)); + Json v = ie.vendors(); + ASSERT_EQ(v.find("vendors")->as_array().front().find("name")->as_string(), std::string("aws")); + try { + ie.list("acme"); + FAIL("expected NOT_CONFIGURED"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED")); + } + ASSERT_EQ(rig.endpoints->send_count(), static_cast(1)); +} + +TEST(huggingface_endpoints, error_code_is_kept) { + Rig rig; + Endpoints ie = Endpoints::from_platform(rig.platform); + rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/huggingface", reply(kIeUnauth, 401)); + try { + ie.list("huggingface"); + FAIL("expected 401"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 401); + ASSERT_TRUE(has(e.message(), "401 Unauthorized")); + } +} + +TEST(huggingface_endpoints, reads_build_the_documented_paths) { + Rig rig; + Endpoints ie = Endpoints::from_platform(rig.platform); + rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme", reply("{\"items\":[]}")); + rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep", reply("{}")); + rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/replica", reply("{}")); + rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/logs", reply("line1\nline2\n", 200, "text/plain")); + rig.endpoints->expect_response(Method::kGet, "/v2/search/acme/states", reply("{}")); + EndpointListParams p; + p.limit = 5; + p.state = "running"; + ie.list("acme", p); + ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme?state=running&limit=5")); + ie.get("acme", "ep"); + ie.replicas("acme", "ep"); + ASSERT_EQ(ie.logs("acme", "ep", 50), std::string("line1\nline2\n")); + ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme/ep/logs?tail=50")); + ie.state_counts("acme"); + ASSERT_THROW(ie.get("acme", "../x"), ApiError); + ASSERT_THROW(ie.list("acme", EndpointListParams{"", "", "", "", "", "", 101}), ApiError); +} + +TEST(huggingface_endpoints, metrics_use_post_with_a_time_window) { + Rig rig; + Endpoints ie = Endpoints::from_platform(rig.platform); + rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/metrics/inference/ttft", reply("{}")); + ie.metrics(MetricKind::kTtft, "acme", "ep", "2021-01-01T00:00:00Z", "2021-01-02T00:00:00Z", "r1"); + ASSERT_EQ(rig.endpoints->seen().back().body, + std::string("{\"replicaId\":\"r1\",\"start\":\"2021-01-01T00:00:00Z\",\"stop\":\"2021-01-02T00:00:00Z\"}")); + ASSERT_THROW(ie.metrics(MetricKind::kTtft, "acme", "ep", "yesterday", "today"), ApiError); + MetricKind k; + ASSERT_TRUE(parse_metric_kind("kv-cache", k)); + ASSERT_FALSE(parse_metric_kind("nope", k)); +} + +TEST(huggingface_endpoints, writes_are_validated_and_explicit) { + Rig rig; + Endpoints ie = Endpoints::from_platform(rig.platform); + ASSERT_THROW(ie.create("acme", spec()), ApiError); // type "protected" is not a type + Json good = spec(); + good.set("type", Json("private")); + rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme", reply("{}")); + ie.create("acme", good); + Json bad_name = good; + bad_name.set("name", Json("My_Endpoint")); + ASSERT_THROW(ie.create("acme", bad_name), ApiError); + Json missing = Json::object(); + missing.set("name", Json("x")); + ASSERT_THROW(ie.create("acme", missing), ApiError); + rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/pause", reply("{}")); + rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/resume", reply("{}")); + rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/scale-to-zero", reply("{}")); + rig.endpoints->expect_response(Method::kDelete, "/v2/endpoint/acme/ep", reply("{}")); + ie.pause("acme", "ep"); + ie.resume("acme", "ep"); + ie.scale_to_zero("acme", "ep"); + ie.remove_endpoint("acme", "ep"); + rig.endpoints->verify(); +} + +TEST(huggingface_endpoints, sse_streams_are_listed_but_refused) { + Rig rig; + Endpoints ie = Endpoints::from_platform(rig.platform); + ASSERT_EQ(rig.endpoints->send_count(), static_cast(0)); + for (std::size_t i = 0; i < Calls::count(); ++i) + if (Calls::at(i).has_flag("unsupported")) ASSERT_TRUE(std::string(Calls::at(i).path).find("sse") != std::string::npos || + std::string(Calls::at(i).path).find("/sse") != std::string::npos); +} + +TEST(huggingface_tgi, native_and_openai_routes) { + Rig rig; // transports unused: the server has its own + auto mock = std::make_unique(); + MockTransport* t = mock.get(); + TgiServer tgi(Platform::server_with(std::move(mock), "http://127.0.0.1:8080/", "")); + ASSERT_FALSE(tgi.has_token()); + t->expect_response(Method::kGet, "/health", reply("", 200, "text/plain")); + MockExpectation starting; // the client retries a 503 GET, so allow it to answer more than once + starting.method = Method::kGet; + starting.path = "/health"; + starting.response = reply("starting", 503, "text/plain"); + starting.times = 5; + t->expect(starting); + ASSERT_TRUE(tgi.ready()); + ASSERT_FALSE(tgi.ready()); + t->expect_response(Method::kPost, "/generate", reply("{\"generated_text\":\"hi\"}")); + Json params = Json::object(); + params.set("max_new_tokens", Json(8)); + tgi.generate("Hello", params); + ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"Hello\",\"parameters\":{\"max_new_tokens\":8}}")); + ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("")); + ASSERT_THROW(tgi.generate("", params), ApiError); + Json bad = Json::object(); + bad.set("max_new_tokens", Json(8)); + ASSERT_THROW(tgi.completions(bad), ApiError); // body needs a table field (prompt is optional, max_new_tokens is not a field) +} + +TEST(huggingface_tgi, generate_stream_decodes_token_events) { + auto mock = std::make_unique(); + MockTransport* t = mock.get(); + TgiServer tgi(Platform::server_with(std::move(mock), "http://tgi.local:8080", "hf_tok")); + ASSERT_TRUE(tgi.has_token()); + // SHAPE: TGI streams one StreamResponse per `data:` line + t->expect_response(Method::kPost, "/generate_stream", + sse("data:{\"index\":1,\"token\":{\"id\":1,\"text\":\"a\",\"logprob\":-0.1,\"special\":false},\"generated_text\":null}\n\n" + "data:{\"index\":2,\"token\":{\"id\":2,\"text\":\"b\",\"logprob\":-0.2,\"special\":false},\"generated_text\":\"ab\"}\n\n")); + std::string text; + tgi.generate_stream("x", Json(), [&](const molo::llm::SseEvent& e) { + text += e.data.find("token")->find("text")->as_string(); + return true; + }); + ASSERT_EQ(text, std::string("ab")); + ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("Bearer hf_tok")); +} + +TEST(huggingface_tei, embed_rerank_and_validation) { + auto mock = std::make_unique(); + MockTransport* t = mock.get(); + TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", "")); + t->expect_response(Method::kPost, "/embed", reply("[[0.1,0.2]]")); + Json opts = Json::object(); + opts.set("normalize", Json(true)); + tei.embed(Json("hello"), opts); + ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"hello\",\"normalize\":true}")); + t->expect_response(Method::kPost, "/rerank", reply("[{\"index\":0,\"score\":0.9}]")); + tei.rerank("q", {"a", "b"}, true); + ASSERT_EQ(t->seen().back().body, std::string("{\"query\":\"q\",\"return_text\":true,\"texts\":[\"a\",\"b\"]}")); + ASSERT_THROW(tei.embed(Json(""), Json()), ApiError); + ASSERT_THROW(tei.embed(Json(7), Json()), ApiError); + ASSERT_THROW(tei.rerank("q", {}), ApiError); + Json many = Json::array(); + for (int i = 0; i < 257; ++i) many.push(Json("x")); + ASSERT_THROW(tei.embed(many, Json()), ApiError); + Json unknown = Json::object(); + unknown.set("hack", Json(1)); + ASSERT_THROW(tei.embed(Json("x"), unknown), ApiError); + ASSERT_THROW(tei.decode({}, true), ApiError); + ASSERT_THROW(tei.openai_embeddings(Json("x"), "", -1), ApiError); +} + +TEST(huggingface_servers, call_from_the_wrong_surface_is_refused) { + auto mock = std::make_unique(); + MockTransport* t = mock.get(); + TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", "")); + ASSERT_EQ(t->send_count(), static_cast(0)); + ASSERT_TRUE(tei.surface() == Surface::kTei); +}