feat(clients): huggingface: Hub, Inference Providers, Endpoints, TGI, TEI, chat model, demo
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,13 +1,749 @@
|
||||
#include "../AiDemos.hpp"
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include "../../fw/Router.hpp"
|
||||
#include "../../fw/Templates.hpp"
|
||||
#include "clients/url.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "vendors/huggingface/calls.h"
|
||||
#include "vendors/huggingface/chat_model.h"
|
||||
#include "vendors/huggingface/community.h"
|
||||
#include "vendors/huggingface/endpoints.h"
|
||||
#include "vendors/huggingface/hub.h"
|
||||
#include "vendors/huggingface/platform.h"
|
||||
#include "vendors/huggingface/providers.h"
|
||||
#include "vendors/huggingface/servers.h"
|
||||
|
||||
namespace molo::example::app {
|
||||
|
||||
// huggingface. Stub: the package agent fills this in.
|
||||
// Hugging Face over the real src/vendors/huggingface package. GET routes need
|
||||
// no credential. Every route that calls Hugging Face is a POST: 422 on bad
|
||||
// input, 503 huggingface_not_configured without HF_TOKEN. Routes that cost
|
||||
// money (chat, tasks, creating endpoints) or change resources (discussions,
|
||||
// collections, webhooks, endpoint lifecycle) are dry runs that show the exact
|
||||
// request unless the body has "confirm":true.
|
||||
|
||||
namespace {
|
||||
|
||||
namespace hf = huggingface;
|
||||
using Json = clients::body::Json;
|
||||
|
||||
constexpr std::size_t kMaxBody = 8 * 1024 * 1024; // inference inputs travel as base64
|
||||
|
||||
const char kPage[] = R"HUGGINGFACE({% extends "base.html" %}
|
||||
{% block content %}
|
||||
<h1>Hugging Face</h1>
|
||||
<p class="hint">Hub (models, datasets, spaces, repos, search, papers, discussions, collections,
|
||||
webhooks), Inference Providers router chat, hf-inference tasks, Inference Endpoints management,
|
||||
TGI and TEI servers. Paid and changing calls are dry runs until the body says
|
||||
<code>"confirm":true</code>.</p>
|
||||
<p id="hf-config" class="hint">Checking credential...</p>
|
||||
<section class="hf">
|
||||
<label>Route
|
||||
<select id="hf-route"></select></label>
|
||||
<label>Body (JSON object) <textarea id="hf-body" rows="8"></textarea></label>
|
||||
<div class="hf-actions">
|
||||
<button id="hf-send" type="button" class="primary">Send</button>
|
||||
<button id="hf-confirm" type="button">Send with confirm</button>
|
||||
</div>
|
||||
<p id="hf-error" class="error" role="alert"></p>
|
||||
<pre id="hf-out" class="hf-out"></pre>
|
||||
</section>
|
||||
<style>
|
||||
.hf label{display:block;margin:.5rem 0}
|
||||
.hf select,.hf textarea{width:100%;box-sizing:border-box}
|
||||
.hf-actions{display:flex;flex-wrap:wrap;gap:.5rem;margin:.5rem 0}
|
||||
.hf-out{overflow:auto;max-height:24rem;white-space:pre-wrap;word-break:break-word}
|
||||
</style>
|
||||
<script src="/embedded/ai_huggingface.js"></script>
|
||||
{% endblock %}
|
||||
)HUGGINGFACE";
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw fw::HttpError(422, "validation_error", why); }
|
||||
|
||||
fw::Response json_out(const Json& j, int status = 200) {
|
||||
fw::Response res = fw::JsonResponse(j.dump(), status);
|
||||
res.headers.set("Cache-Control", "no-store");
|
||||
return res;
|
||||
}
|
||||
|
||||
Json parse_body(const fw::Request& req) {
|
||||
if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large");
|
||||
if (req.body.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(req.body);
|
||||
if (!j.is_object()) bad("body must be a JSON object");
|
||||
return j;
|
||||
} catch (const clients::body::JsonError&) {
|
||||
bad("body is not valid JSON");
|
||||
}
|
||||
}
|
||||
|
||||
std::string text(const Json& b, const char* key, bool required = false) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) {
|
||||
if (required) bad(std::string(key) + " is required");
|
||||
return "";
|
||||
}
|
||||
if (!v->is_string()) bad(std::string(key) + " must be a string");
|
||||
if (required && v->as_string().empty()) bad(std::string(key) + " is required");
|
||||
return v->as_string();
|
||||
}
|
||||
|
||||
int integer(const Json& b, const char* key, int fallback, int lo, int hi) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return fallback;
|
||||
if (!v->is_number()) bad(std::string(key) + " must be a number");
|
||||
long n = v->as_int();
|
||||
if (n < lo || n > hi) bad(std::string(key) + " must be " + std::to_string(lo) + ".." + std::to_string(hi));
|
||||
return static_cast<int>(n);
|
||||
}
|
||||
|
||||
bool flag(const Json& b, const char* key) {
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return false;
|
||||
if (!v->is_bool()) bad(std::string(key) + " must be true or false");
|
||||
return v->as_bool();
|
||||
}
|
||||
|
||||
std::vector<std::string> strings(const Json& b, const char* key) {
|
||||
std::vector<std::string> out;
|
||||
const Json* v = b.find(key);
|
||||
if (!v || v->is_null()) return out;
|
||||
if (!v->is_array()) bad(std::string(key) + " must be an array of strings");
|
||||
for (const Json& s : v->as_array()) {
|
||||
if (!s.is_string()) bad(std::string(key) + " must be an array of strings");
|
||||
out.push_back(s.as_string());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
hf::RepoType repo_type(const Json& b) {
|
||||
std::string k = text(b, "kind");
|
||||
if (k.empty()) return hf::RepoType::kModel;
|
||||
hf::RepoType t;
|
||||
if (!hf::parse_repo_type(k, t)) bad("kind must be models, datasets or spaces");
|
||||
return t;
|
||||
}
|
||||
|
||||
hf::RepoRef repo_ref(const Json& b, const char* key = "id") {
|
||||
try {
|
||||
return hf::RepoRef::parse(text(b, key, true));
|
||||
} catch (const llm::ApiError& e) {
|
||||
bad(e.message());
|
||||
}
|
||||
}
|
||||
|
||||
Json dry_run(const Json& request, const std::string& path) {
|
||||
Json o = Json::object();
|
||||
o.set("dry_run", Json(true));
|
||||
o.set("would_send_to", Json(path));
|
||||
o.set("request", request);
|
||||
o.set("hint", Json("send confirm:true to run it"));
|
||||
return o;
|
||||
}
|
||||
|
||||
Json page_json(const hf::Page& p) {
|
||||
Json o = Json::object();
|
||||
o.set("items", p.items);
|
||||
o.set("pages", Json(static_cast<int>(p.pages)));
|
||||
o.set("truncated", Json(p.truncated));
|
||||
o.set("truncated_by", Json(p.truncated_by));
|
||||
o.set("next_cursor", Json(p.next_cursor));
|
||||
return o;
|
||||
}
|
||||
|
||||
struct Plan {
|
||||
Json request; // what would be sent (shown in a dry run)
|
||||
std::string path; // for the dry-run message
|
||||
bool needs_confirm = false; // paid or changes a resource
|
||||
std::function<Json()> call;
|
||||
};
|
||||
|
||||
// Servers (TGI, TEI) get the token only when they are Hugging Face hosted.
|
||||
std::string token_for(const std::string& url, const std::string& token) {
|
||||
clients::Url u = clients::Url::parse(url);
|
||||
const std::string& h = u.host();
|
||||
auto ends = [&](const char* suffix) {
|
||||
std::string s(suffix);
|
||||
return h.size() >= s.size() && h.compare(h.size() - s.size(), s.size(), s) == 0;
|
||||
};
|
||||
return (ends(".huggingface.cloud") || ends(".hf.space")) ? token : std::string();
|
||||
}
|
||||
|
||||
class HuggingfaceRoutes {
|
||||
public:
|
||||
HuggingfaceRoutes() : p_(hf::Platform::from_env()), token_(hf::Platform::token_from_env()) {}
|
||||
const hf::Platform& platform() const { return p_; }
|
||||
|
||||
using Planner = std::function<Plan(const Json&)>;
|
||||
|
||||
fw::Response run(const fw::Request& req, const Planner& planner) const {
|
||||
Json body = parse_body(req);
|
||||
Plan plan;
|
||||
try {
|
||||
plan = planner(body);
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
if (plan.needs_confirm && !flag(body, "confirm")) return json_out(dry_run(plan.request, plan.path));
|
||||
if (!p_.configured()) throw fw::HttpError(503, "huggingface_not_configured", "no credential configured (set HF_TOKEN)");
|
||||
try {
|
||||
return json_out(plan.call());
|
||||
} catch (const llm::ApiError& e) {
|
||||
map(e);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Hub --------------------------------------------------------------------
|
||||
Plan hub_list(const Json& b) const {
|
||||
hf::RepoType type = repo_type(b);
|
||||
hf::RepoListParams lp;
|
||||
lp.search = text(b, "search");
|
||||
lp.author = text(b, "author");
|
||||
lp.sort = text(b, "sort");
|
||||
lp.direction = text(b, "direction");
|
||||
lp.pipeline_tag = text(b, "pipeline_tag");
|
||||
lp.filter = strings(b, "filter");
|
||||
lp.limit = integer(b, "limit", 20, 1, 100);
|
||||
const int pages = integer(b, "pages", 1, 1, 5);
|
||||
hf::RequestPlan::make(hf::Hub::list_call(type), hf::PathArgs(), lp.to_query());
|
||||
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
|
||||
Json req = Json::object();
|
||||
req.set("kind", Json(hf::repo_type_plural(type)));
|
||||
return Plan{req, std::string("GET /api/") + hf::repo_type_plural(type), false, [=] {
|
||||
return page_json(hub->list_repos(type, lp, static_cast<std::size_t>(lp.limit) * pages,
|
||||
static_cast<std::size_t>(pages)));
|
||||
}};
|
||||
}
|
||||
|
||||
Plan hub_repo(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
hf::RepoType type = repo_type(b);
|
||||
hf::RepoRef ref = repo_ref(b);
|
||||
const std::string rev = text(b, "rev").empty() ? "main" : text(b, "rev");
|
||||
const std::string path = text(b, "path");
|
||||
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
req.set("id", Json(ref.id()));
|
||||
auto make = [&](std::function<Json()> fn) { return Plan{req, "GET /api/" + std::string(hf::repo_type_plural(type)) + "/" + ref.id(), false, std::move(fn)}; };
|
||||
if (action == "info") {
|
||||
const std::string revision = text(b, "revision");
|
||||
return make([=] { return hub->repo_info(type, ref, revision, strings(b, "expand")); });
|
||||
}
|
||||
if (action == "tree") {
|
||||
const bool rec = flag(b, "recursive");
|
||||
const int lim = integer(b, "limit", 0, 0, 1000);
|
||||
return make([=] { return hub->tree(type, ref, rev, path, rec, lim); });
|
||||
}
|
||||
if (action == "refs") return make([=] { return hub->refs(type, ref, flag(b, "include_prs")); });
|
||||
if (action == "commits") {
|
||||
const int lim = integer(b, "limit", 20, 1, 100);
|
||||
return make([=] { return hub->commits(type, ref, rev, lim); });
|
||||
}
|
||||
if (action == "compare") {
|
||||
const std::string cmp = text(b, "compare", true);
|
||||
return make([=] { return hub->compare(type, ref, cmp); });
|
||||
}
|
||||
if (action == "likers") return make([=] { return hub->likers(type, ref); });
|
||||
if (action == "paths_info") {
|
||||
std::vector<std::string> paths = strings(b, "paths");
|
||||
return make([=] { return hub->paths_info(type, ref, rev, paths); });
|
||||
}
|
||||
if (action == "safetensors") {
|
||||
const std::string p = text(b, "path", true);
|
||||
return make([=] { return hub->safetensors_metadata(ref, p); });
|
||||
}
|
||||
if (action == "file") {
|
||||
const std::string p = text(b, "path", true);
|
||||
hf::RequestPlan::encode_file_path(p);
|
||||
return make([=] {
|
||||
hf::Download d = hub->download(type, ref, rev, p);
|
||||
Json o = Json::object();
|
||||
o.set("bytes", Json(static_cast<int>(d.bytes.size())));
|
||||
o.set("content_type", Json(d.content_type));
|
||||
o.set("commit", Json(d.commit));
|
||||
o.set("etag", Json(d.etag));
|
||||
const bool texty = d.content_type.find("text") != std::string::npos || d.content_type.find("json") != std::string::npos;
|
||||
o.set("preview", Json(texty ? d.bytes.substr(0, 20000) : std::string()));
|
||||
return o;
|
||||
});
|
||||
}
|
||||
bad("action must be info, tree, refs, commits, compare, likers, paths_info, safetensors or file");
|
||||
}
|
||||
|
||||
Plan hub_search(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
auto make = [&](std::function<Json()> fn) { return Plan{req, "GET /api (" + action + ")", false, std::move(fn)}; };
|
||||
const int lim = integer(b, "limit", 10, 1, 100);
|
||||
if (action == "quicksearch") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->quicksearch(q, lim, t); }); }
|
||||
if (action == "fulltext") { std::string q = text(b, "q", true); std::string t = text(b, "type"); return make([=] { return hub->full_text_search(q, lim, t); }); }
|
||||
if (action == "trending") { std::string t = text(b, "type"); return make([=] { return hub->trending(t, lim); }); }
|
||||
if (action == "tags") { hf::RepoType t = repo_type(b); return make([=] { return hub->tags_by_type(t); }); }
|
||||
if (action == "tasks") return make([=] { return hub->tasks(); });
|
||||
if (action == "hardware") return make([=] { return hub->spaces_hardware(); });
|
||||
if (action == "featured") return make([=] { return hub->spaces_featured(); });
|
||||
if (action == "semantic") { std::string q = text(b, "q", true); return make([=] { return hub->spaces_semantic_search(q); }); }
|
||||
if (action == "papers_daily") return make([=] { return hub->daily_papers(lim, ""); });
|
||||
if (action == "papers_search") { std::string q = text(b, "q", true); return make([=] { return hub->papers_search(q, lim); }); }
|
||||
if (action == "paper") { std::string id = text(b, "id", true); return make([=] { return hub->paper(id); }); }
|
||||
if (action == "partners") { std::string pr = text(b, "provider", true); return make([=] { return hub->partner_models(pr, text(b, "status")); }); }
|
||||
bad("action must be quicksearch, fulltext, trending, tags, tasks, hardware, featured, semantic, papers_daily, papers_search, paper or partners");
|
||||
}
|
||||
|
||||
Plan hub_whoami(const Json&) const {
|
||||
auto hub = std::make_shared<hf::Hub>(hf::Hub::from_platform(p_));
|
||||
return Plan{Json::object(), "GET /api/whoami-v2", false, [hub] { return hub->whoami(); }};
|
||||
}
|
||||
|
||||
Plan hub_discussions(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
hf::RepoType type = repo_type(b);
|
||||
hf::RepoRef ref = repo_ref(b);
|
||||
auto api = std::make_shared<hf::Discussions>(p_.hub);
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
req.set("id", Json(ref.id()));
|
||||
const std::string base = std::string("/api/") + hf::repo_type_plural(type) + "/" + ref.id() + "/discussions";
|
||||
const int num = integer(b, "num", 0, 0, 999999999);
|
||||
if (action == "list") {
|
||||
hf::DiscussionFilter f;
|
||||
f.page = integer(b, "page", 0, 0, 100000);
|
||||
f.kind = text(b, "type");
|
||||
f.status = text(b, "status");
|
||||
f.author = text(b, "author");
|
||||
f.search = text(b, "search");
|
||||
return Plan{req, "GET " + base, false, [=] { return api->list(type, ref, f); }};
|
||||
}
|
||||
if (action == "get") {
|
||||
if (!num) bad("num is required");
|
||||
return Plan{req, "GET " + base + "/{num}", false, [=] { return api->get(type, ref, num); }};
|
||||
}
|
||||
if (action == "create") {
|
||||
std::string title = text(b, "title", true), desc = text(b, "description");
|
||||
bool pr = flag(b, "pull_request");
|
||||
req.set("title", Json(title));
|
||||
return Plan{req, "POST " + base, true, [=] { return api->create(type, ref, title, desc, pr); }};
|
||||
}
|
||||
if (!num) bad("num is required");
|
||||
req.set("num", Json(num));
|
||||
if (action == "comment") {
|
||||
std::string c = text(b, "comment", true);
|
||||
return Plan{req, "POST " + base + "/{num}/comment", true, [=] { return api->comment(type, ref, num, c); }};
|
||||
}
|
||||
if (action == "status") {
|
||||
std::string s = text(b, "status", true), c = text(b, "comment");
|
||||
return Plan{req, "POST " + base + "/{num}/status", true, [=] { return api->change_status(type, ref, num, s, c); }};
|
||||
}
|
||||
if (action == "rename") {
|
||||
std::string t = text(b, "title", true);
|
||||
return Plan{req, "POST " + base + "/{num}/title", true, [=] { return api->rename(type, ref, num, t); }};
|
||||
}
|
||||
if (action == "remove")
|
||||
return Plan{req, "DELETE " + base + "/{num}", true, [=] { return api->remove_discussion(type, ref, num); }};
|
||||
bad("action must be list, get, create, comment, status, rename or remove");
|
||||
}
|
||||
|
||||
Plan hub_collections(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
auto api = std::make_shared<hf::Collections>(p_.hub);
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
if (action == "list") {
|
||||
hf::CollectionListParams lp;
|
||||
lp.owner = text(b, "owner");
|
||||
lp.q = text(b, "q");
|
||||
lp.sort = text(b, "sort");
|
||||
lp.limit = integer(b, "limit", 20, 1, 100);
|
||||
lp.to_query();
|
||||
return Plan{req, "GET /api/collections", false, [=] { return page_json(api->list(lp, 100, 2)); }};
|
||||
}
|
||||
const std::string owner = text(b, "owner", true), slug = text(b, "slug");
|
||||
req.set("owner", Json(owner));
|
||||
if (action == "create") {
|
||||
std::string title = text(b, "title", true), desc = text(b, "description");
|
||||
bool priv = flag(b, "private");
|
||||
req.set("title", Json(title));
|
||||
return Plan{req, "POST /api/collections", true, [=] { return api->create(owner, title, desc, priv); }};
|
||||
}
|
||||
if (slug.empty()) bad("slug is required");
|
||||
req.set("slug", Json(slug));
|
||||
if (action == "get") return Plan{req, "GET /api/collections/{owner}/{slug}", false, [=] { return api->get(owner, slug); }};
|
||||
if (action == "update") {
|
||||
const Json* f = b.find("fields");
|
||||
if (!f || !f->is_object()) bad("fields must be an object");
|
||||
Json fields = *f;
|
||||
return Plan{req, "PATCH /api/collections/{owner}/{slug}", true, [=] { return api->update(owner, slug, fields); }};
|
||||
}
|
||||
if (action == "add_item") {
|
||||
hf::CollectionItem item;
|
||||
item.type = text(b, "item_type", true);
|
||||
item.id = text(b, "item_id", true);
|
||||
item.to_json();
|
||||
std::string note = text(b, "note");
|
||||
return Plan{req, "POST /api/collections/{owner}/{slug}/items", true, [=] { return api->add_item(owner, slug, item, note); }};
|
||||
}
|
||||
if (action == "remove")
|
||||
return Plan{req, "DELETE /api/collections/{owner}/{slug}", true, [=] { return api->remove_collection(owner, slug); }};
|
||||
bad("action must be list, get, create, update, add_item or remove");
|
||||
}
|
||||
|
||||
Plan hub_webhooks(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
auto api = std::make_shared<hf::Webhooks>(p_.hub);
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
auto spec_of = [&]() {
|
||||
hf::WebhookSpec s;
|
||||
s.url = text(b, "url", true);
|
||||
s.domains = strings(b, "domains");
|
||||
s.secret = text(b, "secret");
|
||||
const Json* w = b.find("watched");
|
||||
if (!w || !w->is_array()) bad("watched must be an array of {type,name}");
|
||||
for (const Json& x : w->as_array()) {
|
||||
if (!x.is_object()) bad("watched entries must be objects");
|
||||
s.watched.push_back({text(x, "type", true), text(x, "name", true)});
|
||||
}
|
||||
s.to_json();
|
||||
return s;
|
||||
};
|
||||
if (action == "list") return Plan{req, "GET /api/settings/webhooks", false, [=] { return api->list(); }};
|
||||
if (action == "create") {
|
||||
hf::WebhookSpec s = spec_of();
|
||||
req.set("url", Json(s.url));
|
||||
return Plan{req, "POST /api/settings/webhooks", true, [=] { return api->create(s); }};
|
||||
}
|
||||
const std::string id = text(b, "id", true);
|
||||
req.set("id", Json(id));
|
||||
if (action == "get") return Plan{req, "GET /api/settings/webhooks/{id}", false, [=] { return api->get(id); }};
|
||||
if (action == "update") {
|
||||
hf::WebhookSpec s = spec_of();
|
||||
return Plan{req, "POST /api/settings/webhooks/{id}", true, [=] { return api->update(id, s); }};
|
||||
}
|
||||
if (action == "enable") return Plan{req, "POST /api/settings/webhooks/{id}/enable", true, [=] { return api->enable(id); }};
|
||||
if (action == "disable") return Plan{req, "POST /api/settings/webhooks/{id}/disable", true, [=] { return api->disable(id); }};
|
||||
if (action == "replay") {
|
||||
std::string log = text(b, "log_id", true);
|
||||
return Plan{req, "POST /api/settings/webhooks/{id}/replay/{log_id}", true, [=] { return api->replay(id, log); }};
|
||||
}
|
||||
if (action == "remove") return Plan{req, "DELETE /api/settings/webhooks/{id}", true, [=] { return api->remove_webhook(id); }};
|
||||
bad("action must be list, get, create, update, enable, disable, replay or remove");
|
||||
}
|
||||
|
||||
// ---- Inference ---------------------------------------------------------------
|
||||
llm::ChatRequest chat_request(const Json& b) const {
|
||||
llm::ChatRequest r;
|
||||
r.model = text(b, "model", true);
|
||||
hf::RouterModel::parse(r.model);
|
||||
r.system = text(b, "system");
|
||||
r.messages.push_back(llm::Message::user(text(b, "prompt", true)));
|
||||
if (b.find("max_tokens")) r.max_tokens = integer(b, "max_tokens", 256, 1, hf::HfCaps::kMaxTokensCeiling);
|
||||
if (const Json* t = b.find("temperature"); t && t->is_number()) r.temperature = t->as_number();
|
||||
r.json_mode = flag(b, "json");
|
||||
r.reasoning_effort = text(b, "effort");
|
||||
return r;
|
||||
}
|
||||
|
||||
Plan chat(const Json& b) const {
|
||||
llm::ChatRequest r = chat_request(b);
|
||||
auto model = hf::HuggingfaceChatModel::over_router(p_, r.model);
|
||||
Json body = model->to_body(r, false);
|
||||
return Plan{body, "/v1/chat/completions", true, [model, r] {
|
||||
llm::ChatResponse res = model->generate(r);
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(res.id));
|
||||
o.set("model", Json(res.model));
|
||||
o.set("text", Json(res.text()));
|
||||
o.set("finish_reason", Json(res.finish_reason));
|
||||
Json u = Json::object();
|
||||
u.set("input_tokens", Json(res.usage.input_tokens));
|
||||
u.set("output_tokens", Json(res.usage.output_tokens));
|
||||
u.set("total_tokens", Json(res.usage.total_tokens));
|
||||
o.set("usage", u);
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
|
||||
Plan chat_stream(const Json& b) const {
|
||||
llm::ChatRequest r = chat_request(b);
|
||||
auto model = hf::HuggingfaceChatModel::over_router(p_, r.model);
|
||||
Json body = model->to_body(r, true);
|
||||
return Plan{body, "/v1/chat/completions (stream)", true, [model, r] {
|
||||
std::string content;
|
||||
std::size_t chunks = 0;
|
||||
std::string finish;
|
||||
model->stream(r, [&](const llm::ChatChunk& c) {
|
||||
content += c.text_delta;
|
||||
if (!c.finish_reason.empty()) finish = c.finish_reason;
|
||||
return ++chunks < hf::HfCaps::kMaxStreamEvents;
|
||||
});
|
||||
Json o = Json::object();
|
||||
o.set("text", Json(content));
|
||||
o.set("finish_reason", Json(finish));
|
||||
o.set("chunks", Json(static_cast<int>(chunks)));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
|
||||
Plan task(const Json& b) const {
|
||||
const std::string task = text(b, "task", true), model = text(b, "model", true);
|
||||
const hf::TaskInfo* info = hf::TaskCatalog::find(task);
|
||||
if (!info) bad("unknown task");
|
||||
hf::RequestPlan::check_model(model);
|
||||
Json params = b.find("parameters") ? *b.find("parameters") : Json();
|
||||
auto api = std::make_shared<hf::InferenceTasks>(hf::InferenceTasks::from_platform(p_));
|
||||
Json req = Json::object();
|
||||
req.set("task", Json(task));
|
||||
req.set("model", Json(model));
|
||||
auto result_json = [](const hf::InferenceResult& r) {
|
||||
Json o = Json::object();
|
||||
o.set("content_type", Json(r.content_type));
|
||||
if (r.is_json) {
|
||||
o.set("result", r.json);
|
||||
} else {
|
||||
o.set("bytes", Json(static_cast<int>(r.bytes.size())));
|
||||
o.set("data_base64", Json(security::crypto::Base64::encode(r.bytes.substr(0, 1024 * 1024))));
|
||||
}
|
||||
return o;
|
||||
};
|
||||
if (info->binary_input) {
|
||||
std::string bytes;
|
||||
if (!security::crypto::Base64::decode(text(b, "data_base64", true), bytes)) bad("data_base64 is not valid base64");
|
||||
std::string mime = text(b, "mime", true);
|
||||
req.set("bytes", Json(static_cast<int>(bytes.size())));
|
||||
return Plan{req, "POST /hf-inference/models/{model}", true,
|
||||
[=] { return result_json(api->run_binary(task, model, bytes, mime, params)); }};
|
||||
}
|
||||
const Json* in = b.find("inputs");
|
||||
if (!in || in->is_null()) bad("inputs is required");
|
||||
Json inputs = *in;
|
||||
req.set("inputs", inputs);
|
||||
return Plan{req, "POST /hf-inference/models/{model}", true,
|
||||
[=] { return result_json(api->run_json(task, model, inputs, params)); }};
|
||||
}
|
||||
|
||||
Plan endpoints(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
const std::string ns = text(b, "namespace");
|
||||
const std::string name = text(b, "name");
|
||||
auto api = std::make_shared<hf::Endpoints>(hf::Endpoints::from_platform(p_));
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
if (!ns.empty()) req.set("namespace", Json(ns));
|
||||
if (!name.empty()) req.set("name", Json(name));
|
||||
auto need_ns = [&] { if (ns.empty()) bad("namespace is required"); };
|
||||
auto need_name = [&] { need_ns(); if (name.empty()) bad("name is required"); hf::Endpoints::validate_name(name); };
|
||||
if (action == "vendors") return Plan{req, "GET /v2/provider", false, [=] { return api->vendors(); }};
|
||||
if (action == "list") { need_ns(); return Plan{req, "GET /v2/endpoint/{ns}", false, [=] { return api->list(ns); }}; }
|
||||
if (action == "search") { need_ns(); std::string q = text(b, "q"); return Plan{req, "GET /v2/search/{ns}", false, [=] { return api->search(ns, q); }}; }
|
||||
if (action == "states") { need_ns(); return Plan{req, "GET /v2/search/{ns}/states", false, [=] { return api->state_counts(ns); }}; }
|
||||
if (action == "quotas") { need_ns(); return Plan{req, "GET /v2/provider/quotas/{ns}", false, [=] { return api->quotas(ns); }}; }
|
||||
if (action == "audit") { need_ns(); return Plan{req, "GET /v2/audit-logs/{ns}", false, [=] { return api->audit_logs(ns); }}; }
|
||||
if (action == "get") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}", false, [=] { return api->get(ns, name); }}; }
|
||||
if (action == "replicas") { need_name(); return Plan{req, "GET /v2/endpoint/{ns}/{name}/replica", false, [=] { return api->replicas(ns, name); }}; }
|
||||
if (action == "logs") {
|
||||
need_name();
|
||||
int tail = integer(b, "tail", 100, 1, 1000);
|
||||
return Plan{req, "GET /v2/endpoint/{ns}/{name}/logs", false, [=] {
|
||||
Json o = Json::object();
|
||||
o.set("logs", Json(api->logs(ns, name, tail)));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
if (action == "metrics") {
|
||||
need_name();
|
||||
hf::MetricKind k;
|
||||
if (!hf::parse_metric_kind(text(b, "kind").empty() ? "all" : text(b, "kind"), k)) bad("unknown metric kind");
|
||||
std::string start = text(b, "start", true), stop = text(b, "stop", true);
|
||||
return Plan{req, "POST /v2/endpoint/{ns}/{name}/metrics", false, [=] { return api->metrics(k, ns, name, start, stop); }};
|
||||
}
|
||||
if (action == "create") {
|
||||
need_ns();
|
||||
const Json* s = b.find("spec");
|
||||
if (!s || !s->is_object()) bad("spec must be an object");
|
||||
Json spec = *s;
|
||||
hf::Endpoints::validate_spec(spec, true);
|
||||
req.set("spec", spec);
|
||||
return Plan{req, "POST /v2/endpoint/{ns} (costs money while it runs)", true, [=] { return api->create(ns, spec); }};
|
||||
}
|
||||
if (action == "update") {
|
||||
need_name();
|
||||
const Json* s = b.find("spec");
|
||||
if (!s || !s->is_object()) bad("spec must be an object");
|
||||
Json spec = *s;
|
||||
hf::Endpoints::validate_spec(spec, false);
|
||||
return Plan{req, "PUT /v2/endpoint/{ns}/{name}", true, [=] { return api->update(ns, name, spec); }};
|
||||
}
|
||||
if (action == "pause") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/pause", true, [=] { return api->pause(ns, name); }}; }
|
||||
if (action == "resume") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/resume", true, [=] { return api->resume(ns, name); }}; }
|
||||
if (action == "scale_to_zero") { need_name(); return Plan{req, "POST /v2/endpoint/{ns}/{name}/scale-to-zero", true, [=] { return api->scale_to_zero(ns, name); }}; }
|
||||
if (action == "remove") { need_name(); return Plan{req, "DELETE /v2/endpoint/{ns}/{name}", true, [=] { return api->remove_endpoint(ns, name); }}; }
|
||||
bad("action must be vendors, list, search, states, quotas, audit, get, replicas, logs, metrics, create, update, pause, resume, scale_to_zero or remove");
|
||||
}
|
||||
|
||||
hf::Access server_access(const Json& b) const {
|
||||
const std::string url = text(b, "url", true);
|
||||
try {
|
||||
return hf::Platform::server(url, token_for(hf::Platform::check_server_url(url), token_));
|
||||
} catch (const clients::UrlError& e) {
|
||||
bad(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
Plan tgi(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
auto api = std::make_shared<hf::TgiServer>(server_access(b));
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }};
|
||||
if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }};
|
||||
if (action == "models") return Plan{req, "GET /v1/models", false, [=] { return api->models(); }};
|
||||
if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }};
|
||||
Json params = b.find("parameters") ? *b.find("parameters") : Json();
|
||||
if (action == "generate") { std::string in = text(b, "inputs", true); return Plan{req, "POST /generate", false, [=] { return api->generate(in, params); }}; }
|
||||
if (action == "tokenize") { std::string in = text(b, "inputs", true); return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(in, params); }}; }
|
||||
if (action == "chat") {
|
||||
llm::ChatRequest r = chat_request(b);
|
||||
hf::ChatWire::to_body(r, "tgi", false);
|
||||
auto model = std::make_shared<hf::HuggingfaceChatModel>(api, r.model);
|
||||
return Plan{req, "POST /v1/chat/completions", false, [=] {
|
||||
Json o = Json::object();
|
||||
o.set("text", Json(model->generate(r).text()));
|
||||
return o;
|
||||
}};
|
||||
}
|
||||
bad("action must be health, info, models, metrics, generate, tokenize or chat");
|
||||
}
|
||||
|
||||
Plan tei(const Json& b) const {
|
||||
const std::string action = text(b, "action", true);
|
||||
auto api = std::make_shared<hf::TeiServer>(server_access(b));
|
||||
Json req = Json::object();
|
||||
req.set("action", Json(action));
|
||||
if (action == "health") return Plan{req, "GET /health", false, [=] { Json o = Json::object(); o.set("ready", Json(api->ready())); return o; }};
|
||||
if (action == "info") return Plan{req, "GET /info", false, [=] { return api->info(); }};
|
||||
if (action == "metrics") return Plan{req, "GET /metrics", false, [=] { Json o = Json::object(); o.set("metrics", Json(api->metrics())); return o; }};
|
||||
if (action == "rerank") {
|
||||
std::string q = text(b, "query", true);
|
||||
std::vector<std::string> texts = strings(b, "texts");
|
||||
return Plan{req, "POST /rerank", false, [=] { return api->rerank(q, texts, flag(b, "return_text")); }};
|
||||
}
|
||||
const Json* in = b.find("inputs");
|
||||
if (!in || in->is_null()) bad("inputs is required");
|
||||
Json inputs = *in;
|
||||
hf::TeiServer::check_inputs(inputs);
|
||||
if (action == "embed") return Plan{req, "POST /embed", false, [=] { return api->embed(inputs); }};
|
||||
if (action == "predict") return Plan{req, "POST /predict", false, [=] { return api->predict(inputs); }};
|
||||
if (action == "tokenize") return Plan{req, "POST /tokenize", false, [=] { return api->tokenize(inputs); }};
|
||||
bad("action must be health, info, metrics, embed, predict, rerank or tokenize");
|
||||
}
|
||||
|
||||
private:
|
||||
[[noreturn]] static void map(const llm::ApiError& e) {
|
||||
if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED")
|
||||
throw fw::HttpError(503, "huggingface_not_configured", e.message());
|
||||
if (e.type() == "INVALID_ARGUMENT" && e.http_status() == 400 && e.request_id().empty())
|
||||
throw fw::HttpError(422, "validation_error", e.message());
|
||||
if (e.type() == "LOCAL_BUDGET") throw fw::HttpError(429, "huggingface_local_budget", e.message());
|
||||
throw fw::HttpError(502, "huggingface_error", e.type() + ": " + e.message());
|
||||
}
|
||||
|
||||
hf::Platform p_;
|
||||
std::string token_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
void register_ai_huggingface_demo(fw::Router& router, Services& services) {
|
||||
(void)router;
|
||||
(void)services;
|
||||
auto routes = std::make_shared<HuggingfaceRoutes>();
|
||||
fw::Environment* env = services.templates.get();
|
||||
services.templates->add_template("ai_huggingface.html", kPage);
|
||||
services.add_feature("ai-huggingface", "/ai/huggingface");
|
||||
|
||||
router.get("/ai/huggingface", "ai-huggingface-page", [env](fw::Request req) {
|
||||
(void)req;
|
||||
fw::TemplateCtx ctx;
|
||||
ctx["title"] = std::string("Hugging Face");
|
||||
return fw::Response(200, env->render("ai_huggingface.html", ctx), "text/html; charset=utf-8");
|
||||
});
|
||||
|
||||
router.get("/ai/huggingface/status", "ai-huggingface-status", [routes](fw::Request req) {
|
||||
(void)req;
|
||||
Json o = Json::object();
|
||||
o.set("configured", Json(routes->platform().configured()));
|
||||
o.set("credential_env", Json(hf::Platform::kKeyEnv));
|
||||
o.set("hub", Json(hf::Platform::kHubHost));
|
||||
o.set("router", Json(hf::Platform::kRouterHost));
|
||||
o.set("endpoints", Json(hf::Platform::kEndpointsHost));
|
||||
o.set("calls", Json(static_cast<int>(hf::Calls::count())));
|
||||
o.set("tasks", Json(static_cast<int>(hf::TaskCatalog::count())));
|
||||
o.set("budget_calls", Json(hf::HfCaps::kBudgetMaxCalls));
|
||||
o.set("budget_window_seconds", Json(hf::HfCaps::kBudgetWindowSeconds));
|
||||
o.set("max_pages", Json(static_cast<int>(hf::HfCaps::kMaxPages)));
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
router.get("/ai/huggingface/calls", "ai-huggingface-calls", [](fw::Request req) {
|
||||
(void)req;
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < hf::Calls::count(); ++i) {
|
||||
const hf::CallSpec& s = hf::Calls::at(i);
|
||||
Json row = Json::object();
|
||||
row.set("surface", Json(hf::surface_name(s.surface)));
|
||||
row.set("http", Json(s.http_method));
|
||||
row.set("path", Json(s.path));
|
||||
row.set("spec", Json(s.spec));
|
||||
row.set("auth", Json(s.auth));
|
||||
row.set("flags", Json(s.flags));
|
||||
arr.push(row);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("calls", arr);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
router.get("/ai/huggingface/tasks", "ai-huggingface-tasks", [](fw::Request req) {
|
||||
(void)req;
|
||||
Json arr = Json::array();
|
||||
for (std::size_t i = 0; i < hf::TaskCatalog::count(); ++i) {
|
||||
const hf::TaskInfo& t = hf::TaskCatalog::at(i);
|
||||
Json row = Json::object();
|
||||
row.set("task", Json(t.name));
|
||||
row.set("binary_input", Json(t.binary_input));
|
||||
row.set("pipeline_path", Json(t.pipeline_path));
|
||||
row.set("note", Json(t.note));
|
||||
arr.push(row);
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("tasks", arr);
|
||||
return json_out(o);
|
||||
});
|
||||
|
||||
using Member = HuggingfaceRoutes::Planner;
|
||||
HuggingfaceRoutes* r = routes.get();
|
||||
auto add = [&](const char* path, const char* name, Plan (HuggingfaceRoutes::*m)(const Json&) const) {
|
||||
Member planner = [r, m](const Json& b) { return (r->*m)(b); };
|
||||
router.post(path, name, [routes, planner](fw::Request req) { return routes->run(req, planner); });
|
||||
};
|
||||
add("/ai/huggingface/hub/list", "ai-huggingface-hub-list", &HuggingfaceRoutes::hub_list);
|
||||
add("/ai/huggingface/hub/repo", "ai-huggingface-hub-repo", &HuggingfaceRoutes::hub_repo);
|
||||
add("/ai/huggingface/hub/search", "ai-huggingface-hub-search", &HuggingfaceRoutes::hub_search);
|
||||
add("/ai/huggingface/hub/whoami", "ai-huggingface-hub-whoami", &HuggingfaceRoutes::hub_whoami);
|
||||
add("/ai/huggingface/hub/discussions", "ai-huggingface-hub-discussions", &HuggingfaceRoutes::hub_discussions);
|
||||
add("/ai/huggingface/hub/collections", "ai-huggingface-hub-collections", &HuggingfaceRoutes::hub_collections);
|
||||
add("/ai/huggingface/hub/webhooks", "ai-huggingface-hub-webhooks", &HuggingfaceRoutes::hub_webhooks);
|
||||
add("/ai/huggingface/chat", "ai-huggingface-chat", &HuggingfaceRoutes::chat);
|
||||
add("/ai/huggingface/chat/stream", "ai-huggingface-chat-stream", &HuggingfaceRoutes::chat_stream);
|
||||
add("/ai/huggingface/task", "ai-huggingface-task", &HuggingfaceRoutes::task);
|
||||
add("/ai/huggingface/endpoints", "ai-huggingface-endpoints", &HuggingfaceRoutes::endpoints);
|
||||
add("/ai/huggingface/tgi", "ai-huggingface-tgi", &HuggingfaceRoutes::tgi);
|
||||
add("/ai/huggingface/tei", "ai-huggingface-tei", &HuggingfaceRoutes::tei);
|
||||
}
|
||||
|
||||
} // namespace molo::example::app
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
// Hugging Face page: thin client over /ai/huggingface/* (all limits server side).
|
||||
(function () {
|
||||
"use strict";
|
||||
var $ = function (id) { return document.getElementById(id); };
|
||||
var out = $("hf-out"), err = $("hf-error");
|
||||
var templates = {
|
||||
"hub/list": { kind: "models", search: "bert", sort: "downloads", limit: 10 },
|
||||
"hub/repo": { action: "info", kind: "models", id: "openai-community/gpt2" },
|
||||
"hub/search": { action: "quicksearch", q: "llama", limit: 5 },
|
||||
"hub/whoami": {},
|
||||
"hub/discussions": { action: "list", kind: "models", id: "openai-community/gpt2" },
|
||||
"hub/collections": { action: "list", owner: "nvidia", limit: 5 },
|
||||
"hub/webhooks": { action: "list" },
|
||||
"chat": { model: "openai/gpt-oss-20b:fastest", prompt: "Hello!", max_tokens: 64 },
|
||||
"chat/stream": { model: "openai/gpt-oss-20b:fastest", prompt: "Count to five.", max_tokens: 64 },
|
||||
"task": { task: "text-classification", model: "distilbert/distilbert-base-uncased-finetuned-sst-2-english", inputs: "I like you" },
|
||||
"endpoints": { action: "vendors" },
|
||||
"tgi": { url: "http://127.0.0.1:8080", action: "health" },
|
||||
"tei": { url: "http://127.0.0.1:8081", action: "health" }
|
||||
};
|
||||
var sel = $("hf-route");
|
||||
Object.keys(templates).forEach(function (k) {
|
||||
var o = document.createElement("option");
|
||||
o.value = k; o.textContent = k;
|
||||
sel.appendChild(o);
|
||||
});
|
||||
function fill() { $("hf-body").value = JSON.stringify(templates[sel.value], null, 1); }
|
||||
sel.onchange = fill;
|
||||
fill();
|
||||
|
||||
function send(confirm) {
|
||||
var body;
|
||||
try { body = JSON.parse($("hf-body").value || "{}"); }
|
||||
catch (e) { err.textContent = "Body is not valid JSON"; return; }
|
||||
if (confirm) body.confirm = true;
|
||||
err.textContent = "";
|
||||
out.textContent = "...";
|
||||
fetch("/ai/huggingface/" + sel.value, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body)
|
||||
}).then(function (res) {
|
||||
return res.text().then(function (t) {
|
||||
var j;
|
||||
try { j = JSON.parse(t); } catch (e) { j = { message: t }; }
|
||||
if (!res.ok) { err.textContent = j.message || ("HTTP " + res.status); out.textContent = ""; }
|
||||
else { out.textContent = JSON.stringify(j, null, 2); }
|
||||
});
|
||||
}).catch(function (e) { err.textContent = String(e); out.textContent = ""; });
|
||||
}
|
||||
$("hf-send").onclick = function () { send(false); };
|
||||
$("hf-confirm").onclick = function () { send(true); };
|
||||
|
||||
fetch("/ai/huggingface/status").then(function (r) { return r.json(); }).then(function (s) {
|
||||
$("hf-config").textContent = s.configured
|
||||
? "HF_TOKEN found. Calls are real; paid and changing ones need confirm."
|
||||
: "No HF_TOKEN: calls answer 503 until one is configured; dry runs still work.";
|
||||
});
|
||||
})();
|
||||
Vendored
+234
@@ -0,0 +1,234 @@
|
||||
# Hugging Face - Specification
|
||||
|
||||
Owned directory: `src/vendors/huggingface/`. Namespace `molo::huggingface`.
|
||||
Depends on `molo::llm` (`ApiClient`, `Service`, `SseDecoder`, `ChatModel`,
|
||||
`Config`) and `molo::clients`. Built by `make ai` (the vendor glob in
|
||||
`src/ai.mk` picks the directory up). Demo: `example/app/ai/HuggingfaceDemo.cpp`,
|
||||
routes under `/ai/huggingface`. Plans: `plans/17-llm-vendor-platforms.md`,
|
||||
`plans/18-breadth.md`.
|
||||
|
||||
## 1. Scope
|
||||
|
||||
Hugging Face is five HTTP surfaces on four hosts. All are covered:
|
||||
|
||||
| Surface | Host | Class | What |
|
||||
|---|---|---|---|
|
||||
| Hub | `https://huggingface.co` | `Hub`, `Discussions`, `Collections`, `Webhooks` | models, datasets, spaces (list, info, revision), repo tree / refs / commits / compare / paths-info / likers / safetensors metadata, file download (`/resolve`), search, trending, tags, tasks, papers, Spaces catalogs, Inference Provider mappings (`/api/partners/{provider}/models`), whoami, discussions, collections, webhooks |
|
||||
| Router (Inference Providers) | `https://router.huggingface.co` | `RouterChat`, `InferenceTasks` | OpenAI-compatible chat (`/v1/chat/completions`, routing suffix `:fastest` `:cheapest` `:preferred` `:<provider>`), `/v1/models`, hf-inference task APIs (25 tasks, the classic Inference API), hf-inference chat |
|
||||
| Inference Endpoints | `https://api.endpoints.huggingface.cloud` | `Endpoints` | the whole management API v2 / v3: endpoints (list, get, create, update, pause, resume, scale to zero, delete), logs, 15 metric series, replicas, vendors and quotas, search, CIDR lists, custom domains, private services, audit logs |
|
||||
| TGI server | any URL | `TgiServer` | native (`/generate`, `/generate_stream`, `/tokenize`, `/info`, `/health`, `/metrics`) and OpenAI-compatible (`/v1/chat/completions`, `/v1/completions`, `/v1/models`, `/invocations`) |
|
||||
| TEI server | any URL | `TeiServer` | `/embed`, `/embed_all`, `/embed_sparse`, `/predict`, `/rerank`, `/similarity`, `/tokenize`, `/decode`, `/v1/embeddings`, `/info`, `/health`, `/metrics` |
|
||||
|
||||
Plus `HuggingfaceChatModel : llm::ChatModel` over the router (default), a TGI
|
||||
server, or any `ChatEndpoint`.
|
||||
|
||||
Not covered (explicit, see `tools/check_spec.py` EXCLUDED and section 8):
|
||||
repository writes (commit, upload, LFS, branches, tags, settings), organizations
|
||||
and users, billing, Jobs, buckets / containers / kernels, gated-access requests,
|
||||
SCIM, OAuth, `datasets-server.huggingface.co`, the `huggingface.co/api/blog`
|
||||
and posts comment APIs, the provider-specific router paths.
|
||||
|
||||
## 2. Module map (one class per concern)
|
||||
|
||||
| File | Concern |
|
||||
|---|---|
|
||||
| `caps.h` | `HfCaps`: static ceilings |
|
||||
| `calls.h/.cpp` | `Call` enum and `kCalls`: the ONE table of 132 HTTP calls (surface, method, path, spec kind, reference, auth, flags, body fields, query parameters) |
|
||||
| `request_plan.h/.cpp` | `RequestPlan`, `PathArgs`, `Query`, `RepoRef`, `RepoType`: renders and validates one call from the table |
|
||||
| `platform.h/.cpp` | `Platform`, `Access`, `RequestBudget`: hosts, `HF_TOKEN`, anonymous clients, shared call budget, server URL check |
|
||||
| `hf_service.h/.cpp` | `HfService : llm::Service`: surface check, token check, budget, JSON / raw / bytes / paged / SSE calls |
|
||||
| `hub.h/.cpp` | `Hub` |
|
||||
| `community.h/.cpp` | `Discussions`, `Collections`, `Webhooks` (+ `CollectionItem`, `WebhookSpec`) |
|
||||
| `providers.h/.cpp` | `RouterChat`, `RouterModel`, `TaskCatalog`, `InferenceTasks` |
|
||||
| `endpoints.h/.cpp` | `Endpoints`, `MetricKind` |
|
||||
| `servers.h/.cpp` | `TgiServer`, `TeiServer` |
|
||||
| `chat_wire.h/.cpp` | `ChatWire`, `ChatEndpoint`: neutral chat <-> OpenAI-compatible wire |
|
||||
| `chat_model.h/.cpp` | `HuggingfaceChatModel` |
|
||||
| `tools/check_spec.py` | table vs published documents |
|
||||
|
||||
## 3. Static caps (caps.h)
|
||||
|
||||
| Knob | Value |
|
||||
|---|---|
|
||||
| Path segment / repo-relative file path / segments | 256 / 1024 chars / 32 |
|
||||
| Query parameters / one query value | 32 / 2048 chars |
|
||||
| Model id / server URL | 256 / 2048 chars |
|
||||
| Local call budget | 1000 calls per 300 s per process (observed server limit: 500 per 300 s, `RateLimit-Policy: "fixed window";"api";q=500;w=300`) |
|
||||
| Page walk | 20 pages, 5000 items, page `limit` 1000 |
|
||||
| `paths-info` paths | 256 |
|
||||
| Download | 64 MiB (the `llm` response cap; larger fails loudly) |
|
||||
| Inference input (binary) | 16 MiB |
|
||||
| Batch inputs (TEI, embeddings) | 256 entries, 4 MiB text |
|
||||
| Stream events | 200000 |
|
||||
| Messages / tools / stop sequences | 4096 / 128 / 4 |
|
||||
| Webhook domains | 8 |
|
||||
|
||||
## 4. Contracts
|
||||
|
||||
- **Token.** `HF_TOKEN` (or the legacy `HUGGING_FACE_HUB_TOKEN`) via
|
||||
`llm::Config`. It travels as `Authorization: Bearer` only, never in a URL or an
|
||||
error. A row with `auth: token` and no token throws `NotConfiguredError`
|
||||
(503 `NOT_CONFIGURED`) before any traffic. Rows with `auth: none` or
|
||||
`optional` (public Hub reads, `GET /v1/models`, `GET /v2/provider`, TGI / TEI)
|
||||
run without a token.
|
||||
- **Anonymous requests.** `llm::ApiClient` cannot send an unauthenticated
|
||||
request, so without a token `Platform` builds the client with a marker header
|
||||
`X-Molo-Anonymous: 1` instead of `Authorization`. Foundation gap, worked
|
||||
around inside the package.
|
||||
- **One table.** Every path, query parameter and body field comes from `kCalls`.
|
||||
`RequestPlan::make` refuses a query parameter or body field the row does not
|
||||
list, an unused or missing path argument, a body on a bodiless row, and rows
|
||||
flagged `unsupported`.
|
||||
- **Ids.** `namespace`, `repo`, `name`: `[A-Za-z0-9._-]`, 1..256, not `.` / `..`.
|
||||
Repo ids must be `owner/name`; legacy ids without an owner (`gpt2`) are refused
|
||||
because the Hub answers them with a redirect to the canonical id. A revision or
|
||||
tag is one percent-encoded segment (`refs/pr/1` -> `refs%2Fpr%2F1`). File
|
||||
paths are validated per segment and percent-encoded. Query values are
|
||||
percent-encoded; a value containing `..` has its dots encoded because
|
||||
`ApiClient` refuses `..` anywhere in the request path.
|
||||
- **Lists.** Hub list calls follow `Link: <...>; rel="next"` cursors, accept only
|
||||
the `cursor` of a Link to the same host, and return `Page` with `truncated`,
|
||||
`truncated_by` (`items` or `pages`) and `next_cursor`. Never silent.
|
||||
- **Writes and deletes.** Rows flagged `write` need a token. Deletes are
|
||||
`remove_*` methods with no defaults, flagged `destructive` in the table
|
||||
(tested: 7 DELETE rows, all flagged). The demo runs every paid or changing
|
||||
action as a dry run unless the body has `"confirm":true`.
|
||||
- **Chat wire.** `ChatWire` documents its mapping in `chat_wire.h`: tool
|
||||
arguments that are not JSON arrive as `{"_invalid_json": raw}`; reasoning is
|
||||
read from `reasoning_content` then `reasoning`; `response_schema` maps to a
|
||||
strict `json_schema`; usage is requested in streams.
|
||||
- **Errors.** Everything is an `ApiError`; the Hub error body
|
||||
`{"error":"..."}` and the Endpoints body
|
||||
`{"error":"401 Unauthorized","code":"UNAUTHORIZED"}` keep their message and
|
||||
status. A local budget stop is `429 LOCAL_BUDGET`.
|
||||
- **Tests.** `MOLO_ENV_FILES=off`; `MOLO_LIVE_NET=1` enables the live tests;
|
||||
`MOLO_LIVE_PAID=1` additionally allows the one chat completion.
|
||||
|
||||
## 5. Credentials
|
||||
|
||||
| Env var | Page |
|
||||
|---|---|
|
||||
| `HF_TOKEN` | https://huggingface.co/settings/tokens (docs: https://huggingface.co/docs/hub/security-tokens). A fine-grained token needs "Make calls to Inference Providers" for the router, "Make calls to Inference Endpoints" and "Manage Inference Endpoints" for the endpoints API, and write scopes for discussions / collections / webhooks. Inference Providers bills the token owner (or the organization named by `RouterChat::set_bill_to`) |
|
||||
|
||||
No key exists in the development environment: see sections 7 and 8.
|
||||
|
||||
## 6. Sources
|
||||
|
||||
Machine-readable documents, fetched 2026-10-25:
|
||||
|
||||
- Hub OpenAPI 3.1 `https://huggingface.co/.well-known/openapi.json` (374
|
||||
operations, 307 paths; also `/.well-known/openapi.md`). It omits the
|
||||
list / info routes of models, datasets and spaces (`GET /api/models`,
|
||||
`/api/models/{ns}/{repo}`, ...), `/api/tasks` and `/api/spaces/{ns}/{repo}/runtime`;
|
||||
those 11 Hub rows cite `huggingface_hub` `hf_api.py` (spec kind `src`); the
|
||||
three hf-inference router rows cite `hf_inference.py` (also `src`).
|
||||
- Inference Endpoints OpenAPI 2.0.0 `https://api.endpoints.huggingface.cloud/openapi.json`
|
||||
(48 operations). The old `/api-doc/openapi.json` answers 404; the swagger page
|
||||
at `/` points at `/openapi.json`.
|
||||
- TGI `docs/openapi.json` (3.3.6-dev0, 12 operations) and TEI `docs/openapi.json`
|
||||
(1.9.4, 12 operations) from the `huggingface/text-generation-inference` and
|
||||
`huggingface/text-embeddings-inference` repositories, branch `main`.
|
||||
- `huggingface_hub` `src/huggingface_hub/hf_api.py`, `constants.py`,
|
||||
`inference/_providers/__init__.py` and `hf_inference.py`: endpoints and
|
||||
parameters of the routes the OpenAPI omits, the router URL template
|
||||
`https://router.huggingface.co/{provider}`, the 25-task hf-inference map, the
|
||||
feature-extraction / sentence-similarity `/pipeline/{task}` path, binary-input
|
||||
tasks, base64 form with parameters.
|
||||
- `huggingface.js` `packages/tasks/src/tasks/<task>/spec/input.json` for the
|
||||
chat-completion body fields (26 tasks fetched; `audio-to-audio` and
|
||||
`tabular-classification` have no schema file).
|
||||
- Unauthenticated probes, section 8.
|
||||
|
||||
## 7. Verified against spec
|
||||
|
||||
`python3 -I src/vendors/huggingface/tools/check_spec.py --probe`, 2026-10-25:
|
||||
|
||||
```
|
||||
132 rows checked; spec operations hub=374 ie=48 tgi=12 tei=12; 0 problems
|
||||
```
|
||||
|
||||
For `ie`, `tgi`, `tei`: every document operation, query parameter and body
|
||||
field is in the table (equality). For `hub`: every row's method, path (expanded
|
||||
over models / datasets / spaces where the document has one path per type),
|
||||
query parameters and body fields exist in the document; the other 314
|
||||
operations are excluded by explicit rules in `EXCLUDED`, counted per reason in
|
||||
the output (jobs 23, buckets / containers / kernels 33, organizations / users /
|
||||
SCIM 77, ...). The chat row is checked against the huggingface.js
|
||||
chat-completion schema (equality of fields). `--probe` GETs the `src` and
|
||||
`none` rows without a credential: all answered 200 (public) or 401 (the Hub's
|
||||
answer for an unknown or private repo id), none 404.
|
||||
|
||||
Local tests: `make ai-test` runs 66 `huggingface_*` tests (table integrity, id
|
||||
and path rules, query and body checks, caps, Link walks, budget window, token
|
||||
header, anonymous header, error mapping, chat wire, SSE transcript, every
|
||||
surface's request shapes). Replies in tests are `RECORDED` (copied from a probe,
|
||||
trimmed) or `SHAPE` (written to the documented schema). Nothing is a scripted
|
||||
claim about vendor behavior. Example route tests: `tests/example/ai/huggingface/`.
|
||||
|
||||
## 8. Verified live
|
||||
|
||||
Run on 2026-10-25 with no credential unless stated. All through the real HTTP
|
||||
transport (`MOLO_LIVE_NET=1 make ai-test`: 4 live tests, 3 run without a key).
|
||||
|
||||
| Request | Result |
|
||||
|---|---|
|
||||
| `GET huggingface.co/api/models?limit=2`, `/api/datasets?limit=2`, `/api/spaces?limit=2` | 200 JSON array; `Link: <...&cursor=...>; rel="next"`; `RateLimit-Policy: "fixed window";"api";q=500;w=300` |
|
||||
| `GET /api/models/gpt2`, `/api/datasets/squad`, `/api/models/gpt2/refs` | 307 to the canonical `openai-community/gpt2` / `rajpurkar/squad` (the reason legacy ids are refused) |
|
||||
| `GET /api/models/openai-community/gpt2`, `/refs`, `/tree/main`, `/discussions`; `POST /paths-info/main`; `GET /openai-community/gpt2/resolve/main/config.json` (665 bytes, `X-Repo-Commit` header) | 200 |
|
||||
| `GET /api/spaces/gradio/hello_world`, `/api/spaces/hardware`, `/api/trending`, `/api/models-tags-by-type`, `/api/tasks`, `/api/daily_papers`, `/api/quicksearch`, `/api/collections?limit=2`, `/api/collections/nvidia/flora-6ad6a8986320a55467825cf5`, `/api/partners/groq/models` | 200 |
|
||||
| `GET /api/whoami-v2`, `/api/settings/webhooks` without a token | 401 `{"error":"Invalid username or password."}` |
|
||||
| `GET router.huggingface.co/v1/models` | 200, 134 models, per model a `providers` array (status, context_length, pricing, supports_tools, latency) |
|
||||
| `POST router.huggingface.co/v1/chat/completions`, `/v1/embeddings`, `/hf-inference/models/gpt2`, any other path | 401 `text/html` without a token, including a made-up path. The router answers 401 before routing, so these probes do NOT prove a path exists |
|
||||
| `GET router.huggingface.co/hf-inference/models/gpt2` | 401 JSON `{"error":"Invalid username or password."}` |
|
||||
| `GET api.endpoints.huggingface.cloud/v2/provider` | 200 (public vendor / region / compute / price list) |
|
||||
| `GET .../v2/endpoint/huggingface`, `/v2/provider/huggingface`, `/v2/search/huggingface`, `/v2/endpoint/huggingface/x/logs` | 401 `{"error":"401 Unauthorized","code":"UNAUTHORIZED"}` |
|
||||
| Invalid token `hf_invalid_molo_probe` on `whoami` and `endpoints list` | 401 with the two envelopes above (live test) |
|
||||
| `api-inference.huggingface.co` | host does not resolve (the classic host is gone; hf-inference lives behind the router) |
|
||||
| `datasets-server.huggingface.co/is-valid?dataset=squad` | 404 `{"error":"The dataset has been renamed..."}` (out of scope, noted) |
|
||||
|
||||
## 9. Unverified
|
||||
|
||||
No token was available, so no authenticated success body was seen:
|
||||
|
||||
- every `auth: token` call: whoami, space runtime, discussions create / comment /
|
||||
status / title / delete, collections create / update / add item / delete,
|
||||
webhooks (all), Endpoints (all except `GET /v2/provider`);
|
||||
- the router chat completion, its routing suffixes, the `X-HF-Bill-To` header
|
||||
(taken from huggingface_hub / docs, unprobed), streaming usage in the last
|
||||
chunk, `reasoning` vs `reasoning_content` per provider;
|
||||
- every hf-inference task call and reply shape: the `/hf-inference/models/{model}`
|
||||
and `/pipeline/{task}` paths and the binary / base64 input forms come from
|
||||
`huggingface_hub` source, not from a response;
|
||||
- the response shapes our code reads (`choices`, `usage`, tool call fields) are
|
||||
the OpenAI-compatible shapes of the chat-completion spec, not seen from the
|
||||
router;
|
||||
- TGI and TEI: no server was reachable; request shapes follow their OpenAPI
|
||||
documents; `ready()` treats 503 as "starting" per TGI behavior and the
|
||||
documents, unobserved;
|
||||
- `Endpoints::validate_spec` checks only the required top-level keys, the name
|
||||
rule from the schema description and the `type` enum;
|
||||
- ZeroGPU / pro quotas and per-token rate limits other than the observed
|
||||
anonymous `q=500;w=300`;
|
||||
- text ranges in `/resolve` (no range support), xet / LFS downloads of files
|
||||
over 64 MiB.
|
||||
|
||||
## 10. Gaps
|
||||
|
||||
- `llm::ApiClient` cannot send an unauthenticated request (workaround: marker
|
||||
header), has no GET stream (the three Endpoints SSE routes `GET .../sse`,
|
||||
`.../logs/sse` and `/v3/.../logs/sse` are in the table with flag
|
||||
`unsupported` and refused), no per-call timeout, and drops response headers on
|
||||
`request()` (paging uses `request_raw`).
|
||||
- `llm::ApiClient::build` refuses `..` in any request path including the query;
|
||||
query values with `..` are dot-encoded as a workaround.
|
||||
- Provider-specific router paths (`router.huggingface.co/{provider}/...`, for
|
||||
example text-to-image on fal-ai or Replicate) have no per-provider schema
|
||||
published; only `hf-inference` tasks and the generic chat route are wrapped.
|
||||
- Collection item update / delete (`.../items/{itemId}`) are not wrapped: the
|
||||
document names the path with the `{slug}-{id}` split.
|
||||
- Repo writes (commit, upload, branches, tags, LFS) are out of scope; `Hub`
|
||||
reads and downloads only. Legacy ids without owner are refused.
|
||||
- `kernels` repo type exists on the Hub and in the discussions document; not
|
||||
wrapped.
|
||||
- Services `Discussions`, `Collections`, `Webhooks` share `Surface::kHub`.
|
||||
- Google services used: none.
|
||||
Vendored
+197
@@ -0,0 +1,197 @@
|
||||
#include "vendors/huggingface/calls.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
const char* surface_name(Surface surface) {
|
||||
switch (surface) {
|
||||
case Surface::kHub: return "hub";
|
||||
case Surface::kRouter: return "router";
|
||||
case Surface::kEndpoints: return "endpoints";
|
||||
case Surface::kTgi: return "tgi";
|
||||
case Surface::kTei: return "tei";
|
||||
}
|
||||
return "?";
|
||||
}
|
||||
|
||||
// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line, strings in order:
|
||||
// http method, path, spec, ref, auth, flags, body fields, query params)
|
||||
static const CallSpec kCalls[] = {
|
||||
{Call::kWhoami, Surface::kHub, "GET", "/api/whoami-v2", "hub", "-", "token", "", "", ""},
|
||||
{Call::kModelsTagsByType, Surface::kHub, "GET", "/api/models-tags-by-type", "hub", "-", "optional", "", "", "type"},
|
||||
{Call::kDatasetsTagsByType, Surface::kHub, "GET", "/api/datasets-tags-by-type", "hub", "-", "optional", "", "", "type"},
|
||||
{Call::kTrending, Surface::kHub, "GET", "/api/trending", "hub", "-", "optional", "", "", "limit,type"},
|
||||
{Call::kQuicksearch, Surface::kHub, "GET", "/api/quicksearch", "hub", "-", "optional", "", "", "discussionCollectionName,discussionId,exclude,includeInvitees,lang,library,limit,namespace,orgsFilter,pipelines,q,repoName,repoType,reposFilter,spacesTags,type"},
|
||||
{Call::kFullTextSearch, Surface::kHub, "GET", "/api/search/full-text", "hub", "-", "optional", "", "", "filter,limit,q,skip,type"},
|
||||
{Call::kRepoLikers, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/likers", "hub", "-", "optional", "", "", "expand"},
|
||||
{Call::kRepoTree, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/tree/{rev}/{path}", "hub", "-", "optional", "optpath", "", "cursor,expand,limit,recursive"},
|
||||
{Call::kRepoTreeSize, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/treesize/{rev}/{path}", "hub", "-", "optional", "optpath", "", ""},
|
||||
{Call::kRepoRefs, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/refs", "hub", "-", "optional", "", "", "include_prs"},
|
||||
{Call::kRepoCommits, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/commits/{rev}", "hub", "-", "optional", "", "", "expand,limit,p"},
|
||||
{Call::kRepoCompare, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/compare/{compare}", "hub", "-", "optional", "", "", "raw"},
|
||||
{Call::kRepoPathsInfo, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/paths-info/{rev}", "hub", "-", "optional", "", "expand,paths", ""},
|
||||
{Call::kModelSafetensors, Surface::kHub, "GET", "/api/models/{namespace}/{repo}/safetensors-metadata/{path}", "hub", "-", "optional", "", "", ""},
|
||||
{Call::kFileResolve, Surface::kHub, "GET", "{typePrefix}/{namespace}/{repo}/resolve/{rev}/{path}", "hub", "-", "optional", "raw", "", ""},
|
||||
{Call::kDiscussionList, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/discussions", "hub", "-", "optional", "", "", "author,p,search,sort,status,type"},
|
||||
{Call::kDiscussionCreate, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions", "hub", "-", "token", "write", "description,pullRequest,title", ""},
|
||||
{Call::kDiscussionGet, Surface::kHub, "GET", "/api/{repoType}/{namespace}/{repo}/discussions/{num}", "hub", "-", "optional", "", "", ""},
|
||||
{Call::kDiscussionDelete, Surface::kHub, "DELETE", "/api/{repoType}/{namespace}/{repo}/discussions/{num}", "hub", "-", "token", "write,destructive", "", ""},
|
||||
{Call::kDiscussionComment, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions/{num}/comment", "hub", "-", "token", "write", "comment", ""},
|
||||
{Call::kDiscussionStatus, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions/{num}/status", "hub", "-", "token", "write", "comment,status", ""},
|
||||
{Call::kDiscussionTitle, Surface::kHub, "POST", "/api/{repoType}/{namespace}/{repo}/discussions/{num}/title", "hub", "-", "token", "write", "title", ""},
|
||||
{Call::kCollectionsList, Surface::kHub, "GET", "/api/collections", "hub", "-", "optional", "", "", "cursor,expand,item,limit,owner,q,sort"},
|
||||
{Call::kCollectionGet, Surface::kHub, "GET", "/api/collections/{namespace}/{slug}", "hub", "-", "optional", "", "", ""},
|
||||
{Call::kCollectionCreate, Surface::kHub, "POST", "/api/collections", "hub", "-", "token", "write", "description,item,namespace,private,resourceGroupId,title", ""},
|
||||
{Call::kCollectionUpdate, Surface::kHub, "PATCH", "/api/collections/{namespace}/{slug}", "hub", "-", "token", "write", "description,gating,position,private,theme,title", ""},
|
||||
{Call::kCollectionDelete, Surface::kHub, "DELETE", "/api/collections/{namespace}/{slug}", "hub", "-", "token", "write,destructive", "", ""},
|
||||
{Call::kCollectionAddItem, Surface::kHub, "POST", "/api/collections/{namespace}/{slug}/items", "hub", "-", "token", "write", "item,note", ""},
|
||||
{Call::kWebhooksList, Surface::kHub, "GET", "/api/settings/webhooks", "hub", "-", "token", "", "", ""},
|
||||
{Call::kWebhookCreate, Surface::kHub, "POST", "/api/settings/webhooks", "hub", "-", "token", "write", "domains,job,jobSourceId,secret,secrets,url,watched", ""},
|
||||
{Call::kWebhookGet, Surface::kHub, "GET", "/api/settings/webhooks/{webhookId}", "hub", "-", "token", "", "", ""},
|
||||
{Call::kWebhookUpdate, Surface::kHub, "POST", "/api/settings/webhooks/{webhookId}", "hub", "-", "token", "write", "domains,job,jobSourceId,secret,secrets,url,watched", ""},
|
||||
{Call::kWebhookDelete, Surface::kHub, "DELETE", "/api/settings/webhooks/{webhookId}", "hub", "-", "token", "write,destructive", "", ""},
|
||||
{Call::kWebhookToggle, Surface::kHub, "POST", "/api/settings/webhooks/{webhookId}/{action}", "hub", "-", "token", "write", "", ""},
|
||||
{Call::kWebhookReplay, Surface::kHub, "POST", "/api/settings/webhooks/{webhookId}/replay/{logId}", "hub", "-", "token", "write", "", ""},
|
||||
{Call::kDailyPapers, Surface::kHub, "GET", "/api/daily_papers", "hub", "-", "optional", "", "", "date,limit,month,p,sort,submitter,week"},
|
||||
{Call::kPapersList, Surface::kHub, "GET", "/api/papers", "hub", "-", "optional", "", "", "cursor,limit"},
|
||||
{Call::kPapersSearch, Surface::kHub, "GET", "/api/papers/search", "hub", "-", "optional", "", "", "limit,q"},
|
||||
{Call::kPaperGet, Surface::kHub, "GET", "/api/papers/{paperId}", "hub", "-", "optional", "", "", "field"},
|
||||
{Call::kSpacesHardware, Surface::kHub, "GET", "/api/spaces/hardware", "hub", "-", "optional", "", "", ""},
|
||||
{Call::kSpacesTemplates, Surface::kHub, "GET", "/api/spaces/templates", "hub", "-", "optional", "", "", ""},
|
||||
{Call::kSpacesSemanticSearch, Surface::kHub, "GET", "/api/spaces/semantic-search", "hub", "-", "optional", "", "", "agents,category,filter,includeNonRunning,q,sdk"},
|
||||
{Call::kSpacesFeatured, Surface::kHub, "GET", "/api/spaces/featured", "hub", "-", "optional", "", "", "date"},
|
||||
{Call::kPartnerModels, Surface::kHub, "GET", "/api/partners/{provider}/models", "hub", "-", "optional", "", "", "status"},
|
||||
{Call::kModelsList, Surface::kHub, "GET", "/api/models", "src", "{self.endpoint}/api/models\"", "optional", "", "", "filter,author,apps,gated,inference,inference_provider,pipeline_tag,num_parameters,search,sort,limit,full,config,cardData,expand,cursor,direction"},
|
||||
{Call::kDatasetsList, Surface::kHub, "GET", "/api/datasets", "src", "{self.endpoint}/api/datasets\"", "optional", "", "", "filter,author,gated,search,sort,limit,full,expand,cursor,direction"},
|
||||
{Call::kSpacesList, Surface::kHub, "GET", "/api/spaces", "src", "{self.endpoint}/api/spaces\"", "optional", "", "", "filter,author,search,sort,limit,full,expand,cursor,direction,datasets,models,linked"},
|
||||
{Call::kModelInfo, Surface::kHub, "GET", "/api/models/{namespace}/{repo}", "src", "/api/models/{repo_id}\"", "optional", "", "", "securityStatus,blobs,expand"},
|
||||
{Call::kModelRevision, Surface::kHub, "GET", "/api/models/{namespace}/{repo}/revision/{rev}", "src", "/api/models/{repo_id}/revision/", "optional", "", "", "securityStatus,blobs,expand"},
|
||||
{Call::kDatasetInfo, Surface::kHub, "GET", "/api/datasets/{namespace}/{repo}", "src", "/api/datasets/{repo_id}\"", "optional", "", "", "blobs,expand"},
|
||||
{Call::kDatasetRevision, Surface::kHub, "GET", "/api/datasets/{namespace}/{repo}/revision/{rev}", "src", "/api/datasets/{repo_id}/revision/", "optional", "", "", "blobs,expand"},
|
||||
{Call::kSpaceInfo, Surface::kHub, "GET", "/api/spaces/{namespace}/{repo}", "src", "/api/spaces/{repo_id}\"", "optional", "", "", "expand"},
|
||||
{Call::kSpaceRevision, Surface::kHub, "GET", "/api/spaces/{namespace}/{repo}/revision/{rev}", "src", "/api/spaces/{repo_id}/revision/", "optional", "", "", "expand"},
|
||||
{Call::kSpaceRuntime, Surface::kHub, "GET", "/api/spaces/{namespace}/{repo}/runtime", "src", "/api/spaces/{repo_id}/runtime", "token", "", "", ""},
|
||||
{Call::kTasks, Surface::kHub, "GET", "/api/tasks", "src", "{constants.ENDPOINT}/api/tasks", "optional", "", "", ""},
|
||||
{Call::kRouterChat, Surface::kRouter, "POST", "/v1/chat/completions", "task", "chat-completion", "token", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,reasoning_effort,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""},
|
||||
{Call::kRouterModels, Surface::kRouter, "GET", "/v1/models", "none", "probe", "none", "", "", ""},
|
||||
{Call::kHfInferenceTask, Surface::kRouter, "POST", "/hf-inference/models/{model}", "src", "models/{mapped_model}", "token", "", "inputs,parameters,stream", ""},
|
||||
{Call::kHfInferencePipeline, Surface::kRouter, "POST", "/hf-inference/models/{model}/pipeline/{task}", "src", "pipeline/{self.task}", "token", "", "inputs,parameters,normalize,prompt_name,truncate,truncation_direction", ""},
|
||||
{Call::kHfInferenceChat, Surface::kRouter, "POST", "/hf-inference/models/{model}/v1/chat/completions", "src", "\"/v1/chat/completions\"", "token", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,reasoning_effort,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""},
|
||||
{Call::kIeGetAuditLogs, Surface::kEndpoints, "GET", "/v2/audit-logs/{namespace}", "ie", "get_audit_logs", "token", "", "", "cursor,endpoint,limit,username"},
|
||||
{Call::kIeListEndpoint, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}", "ie", "list_endpoint", "token", "", "", "cursor,direction,engine,instanceSize,instanceType,limit,search,sortBy,state,tags,task,type"},
|
||||
{Call::kIeCreateEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}", "ie", "create_endpoint", "token", "write", "cacheHttpResponses,compute,customRouter,experimentalFeatures,model,name,network,notifications,privateService,provider,route,tags,type", ""},
|
||||
{Call::kIeGetEndpoint, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}", "ie", "get_endpoint", "token", "", "", ""},
|
||||
{Call::kIeUpdateEndpoint, Surface::kEndpoints, "PUT", "/v2/endpoint/{namespace}/{name}", "ie", "update_endpoint", "token", "write", "compute,customRouter,experimentalFeatures,model,network,notifications,provider,route,tags,type", ""},
|
||||
{Call::kIeDeleteEndpoint, Surface::kEndpoints, "DELETE", "/v2/endpoint/{namespace}/{name}", "ie", "delete_endpoint", "token", "write,destructive", "", ""},
|
||||
{Call::kIeDebugAgent, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/agent/debug", "ie", "debug_agent", "token", "write", "", ""},
|
||||
{Call::kIeGetEndpointLogs, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/logs", "ie", "get_endpoint_logs", "token", "raw", "", "follow,line_max_length,replica,tail"},
|
||||
{Call::kIeGetEndpointLogsSse, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/logs/sse", "ie", "get_endpoint_logs_sse", "token", "unsupported", "", "line_max_length,replica,tail"},
|
||||
{Call::kIeGetEndpointMetrics, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics", "ie", "get_endpoint_metrics", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetHardwareCpu, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/hardware-cpu", "ie", "get_hardware_cpu", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetHardwareGpu, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/hardware-gpu", "ie", "get_hardware_gpu", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetInferenceItl, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/itl", "ie", "get_inference_itl", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetInferenceKvCache, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/kv-cache", "ie", "get_inference_kv_cache", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetInferencePrefixCache, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/prefix-cache", "ie", "get_inference_prefix_cache", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetInferenceRequests, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/requests", "ie", "get_inference_requests", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetInferenceTtft, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/inference/ttft", "ie", "get_inference_ttft", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetPendingRequest, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/pending-request", "ie", "get_pending_request", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetReplicaIds, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/replica-ids", "ie", "get_replica_ids", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetReplicaRunning, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/replica-running", "ie", "get_replica_running", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetReplicaStatus, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/replica-status", "ie", "get_replica_status", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetResponseElapsed, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/response-elapsed", "ie", "get_response_elapsed", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetResponseStatusCode, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/response-status-code", "ie", "get_response_status_code", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetStatusCodeGrouped, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/metrics/response-status-code-grouped", "ie", "get_status_code_grouped", "token", "", "replicaId,start,stop", ""},
|
||||
{Call::kIeGetEndpointOpenMetrics, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/open-metrics", "ie", "get_endpoint_open_metrics", "token", "raw", "", ""},
|
||||
{Call::kIePauseEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/pause", "ie", "pause_endpoint", "token", "write,destructive", "", ""},
|
||||
{Call::kIeGetEndpointReplica, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/replica", "ie", "get_endpoint_replica", "token", "", "", ""},
|
||||
{Call::kIeResumeEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/resume", "ie", "resume_endpoint", "token", "write", "", ""},
|
||||
{Call::kIeScaleToZeroEndpoint, Surface::kEndpoints, "POST", "/v2/endpoint/{namespace}/{name}/scale-to-zero", "ie", "scale_to_zero_endpoint", "token", "write,destructive", "", ""},
|
||||
{Call::kIeGetEndpointSse, Surface::kEndpoints, "GET", "/v2/endpoint/{namespace}/{name}/sse", "ie", "get_endpoint_sse", "token", "unsupported", "", ""},
|
||||
{Call::kIeListCidrsList, Surface::kEndpoints, "GET", "/v2/network/cidr-list/{namespace}", "ie", "list_cidrs_list", "token", "", "", ""},
|
||||
{Call::kIeCreateCidrList, Surface::kEndpoints, "POST", "/v2/network/cidr-list/{namespace}", "ie", "create_cidr_list", "token", "write", "cidrList,id,name", ""},
|
||||
{Call::kIeUpdateCidrList, Surface::kEndpoints, "POST", "/v2/network/cidr-list/{namespace}/{cidr_id}", "ie", "update_cidr_list", "token", "write", "cidrList,id,name", ""},
|
||||
{Call::kIeDeleteCidrList, Surface::kEndpoints, "DELETE", "/v2/network/cidr-list/{namespace}/{cidr_id}", "ie", "delete_cidr_list", "token", "write,destructive", "", ""},
|
||||
{Call::kIeListDomains, Surface::kEndpoints, "GET", "/v2/network/domain/{namespace}", "ie", "list_domains", "token", "", "", ""},
|
||||
{Call::kIeCreateDomain, Surface::kEndpoints, "POST", "/v2/network/domain/{namespace}", "ie", "create_domain", "token", "write", "domain", ""},
|
||||
{Call::kIeDeleteDomain, Surface::kEndpoints, "DELETE", "/v2/network/domain/{namespace}/{domain}", "ie", "delete_domain", "token", "write,destructive", "", ""},
|
||||
{Call::kIeListPrivateServices, Surface::kEndpoints, "GET", "/v2/network/private-service/{namespace}", "ie", "list_private_services", "token", "", "", "account-id,shared"},
|
||||
{Call::kIeDeletePrivateService, Surface::kEndpoints, "DELETE", "/v2/network/private-service/{namespace}/{service-id}", "ie", "delete_private_service", "token", "write,destructive", "", ""},
|
||||
{Call::kIeListVendors, Surface::kEndpoints, "GET", "/v2/provider", "ie", "list_vendors", "none", "", "", ""},
|
||||
{Call::kIeListVendorsQuotasNamespace, Surface::kEndpoints, "GET", "/v2/provider/quotas/{namespace}", "ie", "list_vendors_quotas_namespace", "token", "", "", ""},
|
||||
{Call::kIeListVendorsNamespace, Surface::kEndpoints, "GET", "/v2/provider/{namespace}", "ie", "list_vendors_namespace", "token", "", "", ""},
|
||||
{Call::kIeSearchEndpoint, Surface::kEndpoints, "GET", "/v2/search/{namespace}", "ie", "search_endpoint", "token", "", "", "limit,search,tags"},
|
||||
{Call::kIeCountEndpointStates, Surface::kEndpoints, "GET", "/v2/search/{namespace}/states", "ie", "count_endpoint_states", "token", "", "", ""},
|
||||
{Call::kIeListEndpointTags, Surface::kEndpoints, "GET", "/v2/search/{namespace}/tags", "ie", "list_endpoint_tags", "token", "", "", "limit,prefix"},
|
||||
{Call::kIeGetEndpointLogsV3, Surface::kEndpoints, "GET", "/v3/endpoint/{namespace}/{name}/logs", "ie", "get_endpoint_logs_v3", "token", "", "", "after,before,level,limit,line_max_length,order,replica,search,since,until"},
|
||||
{Call::kIeDownloadEndpointLogsV3, Surface::kEndpoints, "GET", "/v3/endpoint/{namespace}/{name}/logs/download", "ie", "download_endpoint_logs_v3", "token", "raw", "", "level,line_max_length,replica,search,since,until"},
|
||||
{Call::kIeGetEndpointLogsSseV3, Surface::kEndpoints, "GET", "/v3/endpoint/{namespace}/{name}/logs/sse", "ie", "get_endpoint_logs_sse_v3", "token", "unsupported", "", "after,level,line_max_length,replica,search,since,tail"},
|
||||
{Call::kTgiCompatGenerate, Surface::kTgi, "POST", "/", "tgi", "compat_generate", "optional", "", "inputs,parameters,stream", ""},
|
||||
{Call::kTgiGetChatTokenize, Surface::kTgi, "POST", "/chat_tokenize", "tgi", "get_chat_tokenize", "optional", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""},
|
||||
{Call::kTgiGenerate, Surface::kTgi, "POST", "/generate", "tgi", "generate", "optional", "", "inputs,parameters", ""},
|
||||
{Call::kTgiGenerateStream, Surface::kTgi, "POST", "/generate_stream", "tgi", "generate_stream", "optional", "stream", "inputs,parameters", ""},
|
||||
{Call::kTgiHealth, Surface::kTgi, "GET", "/health", "tgi", "health", "optional", "", "", ""},
|
||||
{Call::kTgiGetModelInfo, Surface::kTgi, "GET", "/info", "tgi", "get_model_info", "optional", "", "", ""},
|
||||
{Call::kTgiSagemakerCompatibility, Surface::kTgi, "POST", "/invocations", "tgi", "sagemaker_compatibility", "optional", "", "frequency_penalty,inputs,logit_bias,logprobs,max_tokens,messages,model,n,parameters,presence_penalty,prompt,repetition_penalty,response_format,seed,stop,stream,stream_options,suffix,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""},
|
||||
{Call::kTgiMetrics, Surface::kTgi, "GET", "/metrics", "tgi", "metrics", "optional", "raw", "", ""},
|
||||
{Call::kTgiTokenize, Surface::kTgi, "POST", "/tokenize", "tgi", "tokenize", "optional", "", "inputs,parameters", ""},
|
||||
{Call::kTgiChatCompletions, Surface::kTgi, "POST", "/v1/chat/completions", "tgi", "chat_completions", "optional", "", "frequency_penalty,logit_bias,logprobs,max_tokens,messages,model,n,presence_penalty,response_format,seed,stop,stream,stream_options,temperature,tool_choice,tool_prompt,tools,top_logprobs,top_p", ""},
|
||||
{Call::kTgiCompletions, Surface::kTgi, "POST", "/v1/completions", "tgi", "completions", "optional", "", "frequency_penalty,max_tokens,model,prompt,repetition_penalty,seed,stop,stream,suffix,temperature,top_p", ""},
|
||||
{Call::kTgiOpenaiGetModelInfo, Surface::kTgi, "GET", "/v1/models", "tgi", "openai_get_model_info", "optional", "", "", ""},
|
||||
{Call::kTeiDecode, Surface::kTei, "POST", "/decode", "tei", "decode", "optional", "", "ids,skip_special_tokens", ""},
|
||||
{Call::kTeiEmbed, Surface::kTei, "POST", "/embed", "tei", "embed", "optional", "", "dimensions,inputs,normalize,prompt_name,truncate,truncation_direction", ""},
|
||||
{Call::kTeiEmbedAll, Surface::kTei, "POST", "/embed_all", "tei", "embed_all", "optional", "", "inputs,prompt_name,truncate,truncation_direction", ""},
|
||||
{Call::kTeiEmbedSparse, Surface::kTei, "POST", "/embed_sparse", "tei", "embed_sparse", "optional", "", "inputs,prompt_name,truncate,truncation_direction", ""},
|
||||
{Call::kTeiHealth, Surface::kTei, "GET", "/health", "tei", "health", "optional", "", "", ""},
|
||||
{Call::kTeiGetModelInfo, Surface::kTei, "GET", "/info", "tei", "get_model_info", "optional", "", "", ""},
|
||||
{Call::kTeiMetrics, Surface::kTei, "GET", "/metrics", "tei", "metrics", "optional", "raw", "", ""},
|
||||
{Call::kTeiPredict, Surface::kTei, "POST", "/predict", "tei", "predict", "optional", "", "inputs,raw_scores,truncate,truncation_direction", ""},
|
||||
{Call::kTeiRerank, Surface::kTei, "POST", "/rerank", "tei", "rerank", "optional", "", "query,raw_scores,return_text,texts,truncate,truncation_direction", ""},
|
||||
{Call::kTeiSimilarity, Surface::kTei, "POST", "/similarity", "tei", "similarity", "optional", "", "inputs,parameters", ""},
|
||||
{Call::kTeiTokenize, Surface::kTei, "POST", "/tokenize", "tei", "tokenize", "optional", "", "add_special_tokens,inputs,prompt_name", ""},
|
||||
{Call::kTeiOpenaiEmbed, Surface::kTei, "POST", "/v1/embeddings", "tei", "openai_embed", "optional", "", "dimensions,encoding_format,input,model,user", ""},
|
||||
};
|
||||
// CALLS-END
|
||||
|
||||
bool CallSpec::has_flag(const char* flag) const { return Calls::in_list(flags, flag); }
|
||||
|
||||
bool Calls::in_list(const char* list, const std::string& name) {
|
||||
const std::size_t n = name.size();
|
||||
const char* p = list;
|
||||
while (*p) {
|
||||
const char* end = std::strchr(p, ',');
|
||||
std::size_t len = end ? static_cast<std::size_t>(end - p) : std::strlen(p);
|
||||
if (len == n && std::strncmp(p, name.c_str(), n) == 0) return true;
|
||||
if (!end) break;
|
||||
p = end + 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::size_t Calls::count() { return sizeof(kCalls) / sizeof(kCalls[0]); }
|
||||
|
||||
const CallSpec& Calls::at(std::size_t index) {
|
||||
if (index >= count()) throw llm::ApiError::invalid("huggingface", "call index out of range");
|
||||
return kCalls[index];
|
||||
}
|
||||
|
||||
const CallSpec& Calls::spec_of(Call call) {
|
||||
const std::size_t i = static_cast<std::size_t>(call);
|
||||
if (i >= count() || kCalls[i].call != call)
|
||||
throw llm::ApiError::invalid("huggingface", "call table out of order");
|
||||
return kCalls[i];
|
||||
}
|
||||
|
||||
clients::Method Calls::method_of(Call call) {
|
||||
const char* m = spec_of(call).http_method;
|
||||
if (!std::strcmp(m, "GET")) return clients::Method::kGet;
|
||||
if (!std::strcmp(m, "POST")) return clients::Method::kPost;
|
||||
if (!std::strcmp(m, "PUT")) return clients::Method::kPut;
|
||||
if (!std::strcmp(m, "PATCH")) return clients::Method::kPatch;
|
||||
if (!std::strcmp(m, "DELETE")) return clients::Method::kDelete;
|
||||
throw llm::ApiError::invalid("huggingface", "unknown http method in the call table");
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+189
@@ -0,0 +1,189 @@
|
||||
#pragma once
|
||||
|
||||
// The ONE table of HTTP calls this module makes (every surface). Code builds
|
||||
// every path, query and body check from it, never from a literal.
|
||||
// tools/check_spec.py verifies each row against the published documents:
|
||||
// spec "hub" -> https://huggingface.co/.well-known/openapi.json (method, path, query, body fields)
|
||||
// spec "ie" -> https://api.endpoints.huggingface.cloud/openapi.json
|
||||
// spec "tgi" -> text-generation-inference docs/openapi.json
|
||||
// spec "tei" -> text-embeddings-inference docs/openapi.json
|
||||
// spec "src" -> pinned huggingface_hub source (the Hub OpenAPI omits these): `ref` must occur in it
|
||||
// spec "task" -> huggingface.js task input schema named by `ref`
|
||||
// spec "none" -> not in any published document; confirmed by an unauthenticated probe (see SPEC)
|
||||
// Fields: auth is "none" (public), "optional" (public, a token widens it) or
|
||||
// "token" (needs HF_TOKEN). flags: write, destructive, raw (non JSON reply),
|
||||
// stream, optpath (a trailing {path} may be empty), unsupported (listed for
|
||||
// coverage, refused at call time with the reason in SPEC.md).
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
|
||||
#include "clients/request.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
enum class Surface { kHub, kRouter, kEndpoints, kTgi, kTei };
|
||||
|
||||
const char* surface_name(Surface surface);
|
||||
|
||||
enum class Call {
|
||||
kWhoami,
|
||||
kModelsTagsByType,
|
||||
kDatasetsTagsByType,
|
||||
kTrending,
|
||||
kQuicksearch,
|
||||
kFullTextSearch,
|
||||
kRepoLikers,
|
||||
kRepoTree,
|
||||
kRepoTreeSize,
|
||||
kRepoRefs,
|
||||
kRepoCommits,
|
||||
kRepoCompare,
|
||||
kRepoPathsInfo,
|
||||
kModelSafetensors,
|
||||
kFileResolve,
|
||||
kDiscussionList,
|
||||
kDiscussionCreate,
|
||||
kDiscussionGet,
|
||||
kDiscussionDelete,
|
||||
kDiscussionComment,
|
||||
kDiscussionStatus,
|
||||
kDiscussionTitle,
|
||||
kCollectionsList,
|
||||
kCollectionGet,
|
||||
kCollectionCreate,
|
||||
kCollectionUpdate,
|
||||
kCollectionDelete,
|
||||
kCollectionAddItem,
|
||||
kWebhooksList,
|
||||
kWebhookCreate,
|
||||
kWebhookGet,
|
||||
kWebhookUpdate,
|
||||
kWebhookDelete,
|
||||
kWebhookToggle,
|
||||
kWebhookReplay,
|
||||
kDailyPapers,
|
||||
kPapersList,
|
||||
kPapersSearch,
|
||||
kPaperGet,
|
||||
kSpacesHardware,
|
||||
kSpacesTemplates,
|
||||
kSpacesSemanticSearch,
|
||||
kSpacesFeatured,
|
||||
kPartnerModels,
|
||||
kModelsList,
|
||||
kDatasetsList,
|
||||
kSpacesList,
|
||||
kModelInfo,
|
||||
kModelRevision,
|
||||
kDatasetInfo,
|
||||
kDatasetRevision,
|
||||
kSpaceInfo,
|
||||
kSpaceRevision,
|
||||
kSpaceRuntime,
|
||||
kTasks,
|
||||
kRouterChat,
|
||||
kRouterModels,
|
||||
kHfInferenceTask,
|
||||
kHfInferencePipeline,
|
||||
kHfInferenceChat,
|
||||
kIeGetAuditLogs,
|
||||
kIeListEndpoint,
|
||||
kIeCreateEndpoint,
|
||||
kIeGetEndpoint,
|
||||
kIeUpdateEndpoint,
|
||||
kIeDeleteEndpoint,
|
||||
kIeDebugAgent,
|
||||
kIeGetEndpointLogs,
|
||||
kIeGetEndpointLogsSse,
|
||||
kIeGetEndpointMetrics,
|
||||
kIeGetHardwareCpu,
|
||||
kIeGetHardwareGpu,
|
||||
kIeGetInferenceItl,
|
||||
kIeGetInferenceKvCache,
|
||||
kIeGetInferencePrefixCache,
|
||||
kIeGetInferenceRequests,
|
||||
kIeGetInferenceTtft,
|
||||
kIeGetPendingRequest,
|
||||
kIeGetReplicaIds,
|
||||
kIeGetReplicaRunning,
|
||||
kIeGetReplicaStatus,
|
||||
kIeGetResponseElapsed,
|
||||
kIeGetResponseStatusCode,
|
||||
kIeGetStatusCodeGrouped,
|
||||
kIeGetEndpointOpenMetrics,
|
||||
kIePauseEndpoint,
|
||||
kIeGetEndpointReplica,
|
||||
kIeResumeEndpoint,
|
||||
kIeScaleToZeroEndpoint,
|
||||
kIeGetEndpointSse,
|
||||
kIeListCidrsList,
|
||||
kIeCreateCidrList,
|
||||
kIeUpdateCidrList,
|
||||
kIeDeleteCidrList,
|
||||
kIeListDomains,
|
||||
kIeCreateDomain,
|
||||
kIeDeleteDomain,
|
||||
kIeListPrivateServices,
|
||||
kIeDeletePrivateService,
|
||||
kIeListVendors,
|
||||
kIeListVendorsQuotasNamespace,
|
||||
kIeListVendorsNamespace,
|
||||
kIeSearchEndpoint,
|
||||
kIeCountEndpointStates,
|
||||
kIeListEndpointTags,
|
||||
kIeGetEndpointLogsV3,
|
||||
kIeDownloadEndpointLogsV3,
|
||||
kIeGetEndpointLogsSseV3,
|
||||
kTgiCompatGenerate,
|
||||
kTgiGetChatTokenize,
|
||||
kTgiGenerate,
|
||||
kTgiGenerateStream,
|
||||
kTgiHealth,
|
||||
kTgiGetModelInfo,
|
||||
kTgiSagemakerCompatibility,
|
||||
kTgiMetrics,
|
||||
kTgiTokenize,
|
||||
kTgiChatCompletions,
|
||||
kTgiCompletions,
|
||||
kTgiOpenaiGetModelInfo,
|
||||
kTeiDecode,
|
||||
kTeiEmbed,
|
||||
kTeiEmbedAll,
|
||||
kTeiEmbedSparse,
|
||||
kTeiHealth,
|
||||
kTeiGetModelInfo,
|
||||
kTeiMetrics,
|
||||
kTeiPredict,
|
||||
kTeiRerank,
|
||||
kTeiSimilarity,
|
||||
kTeiTokenize,
|
||||
kTeiOpenaiEmbed,
|
||||
};
|
||||
|
||||
struct CallSpec {
|
||||
Call call;
|
||||
Surface surface;
|
||||
const char* http_method; // GET POST PUT PATCH DELETE
|
||||
const char* path; // template: {name} placeholders
|
||||
const char* spec; // none, hub, ie, tgi, tei, src, task
|
||||
const char* ref; // operationId, source needle, or task name
|
||||
const char* auth; // none, optional, token
|
||||
const char* flags; // comma list
|
||||
const char* body_fields; // comma list: accepted top-level body fields (empty: no body)
|
||||
const char* query_params; // comma list: accepted query parameters
|
||||
|
||||
bool has_flag(const char* flag) const;
|
||||
bool has_body() const { return body_fields[0] != '\0'; }
|
||||
};
|
||||
|
||||
struct Calls {
|
||||
static const CallSpec& spec_of(Call call);
|
||||
static clients::Method method_of(Call call);
|
||||
static std::size_t count();
|
||||
static const CallSpec& at(std::size_t index);
|
||||
// True when `name` is in the comma list.
|
||||
static bool in_list(const char* list, const std::string& name);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+46
@@ -0,0 +1,46 @@
|
||||
#pragma once
|
||||
|
||||
// Static ceilings for the Hugging Face module. Never autotuned. The numbers
|
||||
// marked "observed" come from response headers seen on 2026-10-25 (see
|
||||
// SPEC.md ## Verified live); the rest are our own hard caps.
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
struct HfCaps {
|
||||
// Path and query rendering.
|
||||
static constexpr std::size_t kMaxSegmentChars = 256; // one path segment (id, rev, tag)
|
||||
static constexpr std::size_t kMaxFilePathChars = 1024; // a path inside a repo
|
||||
static constexpr std::size_t kMaxPathSegments = 32;
|
||||
static constexpr std::size_t kMaxQueryParams = 32;
|
||||
static constexpr std::size_t kMaxQueryValueChars = 2048;
|
||||
static constexpr std::size_t kMaxModelChars = 256;
|
||||
static constexpr std::size_t kMaxServerUrlChars = 2048;
|
||||
|
||||
// Request budget. observed: RateLimit-Policy "fixed window";"api";q=500;w=300
|
||||
// on huggingface.co. We allow twice that per process so a single process
|
||||
// never trips the server limit by accident, and stop locally beyond it.
|
||||
static constexpr int kBudgetWindowSeconds = 300;
|
||||
static constexpr int kBudgetMaxCalls = 1000;
|
||||
|
||||
// Page walks (Link: rel="next" cursors). Truncation is reported, never silent.
|
||||
static constexpr std::size_t kMaxPages = 20;
|
||||
static constexpr std::size_t kMaxItems = 5000;
|
||||
static constexpr int kMaxPageLimit = 1000;
|
||||
static constexpr std::size_t kMaxPathsInfo = 256;
|
||||
|
||||
// Payloads.
|
||||
static constexpr std::size_t kMaxDownloadBytes = 64u * 1024u * 1024u; // matches llm::Caps response cap
|
||||
static constexpr std::size_t kMaxInferenceInputBytes = 16u * 1024u * 1024u;
|
||||
static constexpr std::size_t kMaxBatchInputs = 256; // TEI embed / rerank batches
|
||||
static constexpr std::size_t kMaxStreamEvents = 200000;
|
||||
static constexpr std::size_t kMaxStopSequences = 16;
|
||||
static constexpr std::size_t kMaxMessages = 4096;
|
||||
static constexpr std::size_t kMaxTools = 128;
|
||||
static constexpr int kMaxTokensCeiling = 1048576;
|
||||
static constexpr std::size_t kMaxWebhookDomains = 8;
|
||||
static constexpr std::size_t kMaxTextChars = 4u * 1024u * 1024u;
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
#include "vendors/huggingface/chat_model.h"
|
||||
|
||||
#include "vendors/huggingface/providers.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
HuggingfaceChatModel::HuggingfaceChatModel(std::shared_ptr<ChatEndpoint> endpoint, std::string default_model)
|
||||
: endpoint_(std::move(endpoint)), default_model_(std::move(default_model)) {
|
||||
if (!endpoint_) throw llm::ApiError::invalid("huggingface", "chat model needs an endpoint");
|
||||
}
|
||||
|
||||
std::shared_ptr<HuggingfaceChatModel> HuggingfaceChatModel::from_env(const std::string& default_model) {
|
||||
return over_router(Platform::from_env(), default_model);
|
||||
}
|
||||
|
||||
std::shared_ptr<HuggingfaceChatModel> HuggingfaceChatModel::over_router(const Platform& platform,
|
||||
const std::string& default_model) {
|
||||
return std::make_shared<HuggingfaceChatModel>(std::make_shared<RouterChat>(RouterChat::from_platform(platform)),
|
||||
default_model);
|
||||
}
|
||||
|
||||
clients::body::Json HuggingfaceChatModel::to_body(const llm::ChatRequest& request, bool stream) const {
|
||||
return ChatWire::to_body(request, default_model_, stream);
|
||||
}
|
||||
|
||||
llm::ChatResponse HuggingfaceChatModel::generate(const llm::ChatRequest& request) {
|
||||
return ChatWire::to_response(endpoint_->create(to_body(request, false)));
|
||||
}
|
||||
|
||||
void HuggingfaceChatModel::stream(const llm::ChatRequest& request,
|
||||
const std::function<bool(const llm::ChatChunk&)>& on_chunk) {
|
||||
ChatWire::StreamState state;
|
||||
bool stopped = false;
|
||||
endpoint_->stream(to_body(request, true), [&](const llm::SseEvent& e) {
|
||||
if (stopped) return false;
|
||||
if (!ChatWire::on_event(e.data, state, on_chunk)) {
|
||||
stopped = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+40
@@ -0,0 +1,40 @@
|
||||
#pragma once
|
||||
|
||||
// HuggingfaceChatModel: the llm::ChatModel seam over any Hugging Face chat
|
||||
// endpoint that speaks the OpenAI-compatible chat wire: the Inference
|
||||
// Providers router (default), hf-inference, or a TGI server. The mapping rules
|
||||
// are in chat_wire.h. The default model must be given by the caller: the
|
||||
// router has no default (model ids change; GET /v1/models lists them).
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "vendors/huggingface/chat_wire.h"
|
||||
#include "vendors/huggingface/platform.h"
|
||||
#include "llm/chat_types.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
class HuggingfaceChatModel : public llm::ChatModel {
|
||||
public:
|
||||
HuggingfaceChatModel(std::shared_ptr<ChatEndpoint> endpoint, std::string default_model);
|
||||
// Router with HF_TOKEN from the environment.
|
||||
static std::shared_ptr<HuggingfaceChatModel> from_env(const std::string& default_model);
|
||||
// Router over an explicit platform (tests).
|
||||
static std::shared_ptr<HuggingfaceChatModel> over_router(const Platform& platform,
|
||||
const std::string& default_model);
|
||||
|
||||
std::string vendor() const override { return "huggingface"; }
|
||||
llm::ChatResponse generate(const llm::ChatRequest& request) override;
|
||||
void stream(const llm::ChatRequest& request,
|
||||
const std::function<bool(const llm::ChatChunk&)>& on_chunk) override;
|
||||
|
||||
// The wire body a neutral request maps to (tests, dry runs).
|
||||
clients::body::Json to_body(const llm::ChatRequest& request, bool stream) const;
|
||||
|
||||
private:
|
||||
std::shared_ptr<ChatEndpoint> endpoint_;
|
||||
std::string default_model_;
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+328
@@ -0,0 +1,328 @@
|
||||
#include "vendors/huggingface/chat_wire.h"
|
||||
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/error.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
using clients::body::Json;
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
constexpr std::size_t kMaxStop = 4;
|
||||
|
||||
Json image_url_part(const llm::ImageData& img) {
|
||||
std::string url = img.url;
|
||||
if (url.empty()) {
|
||||
if (img.bytes.empty()) bad("image has neither url nor bytes");
|
||||
if (img.mime.empty()) bad("image bytes need a mime type");
|
||||
url = "data:" + img.mime + ";base64," + security::crypto::Base64::encode(img.bytes);
|
||||
}
|
||||
Json u = Json::object();
|
||||
u.set("url", Json(url));
|
||||
Json p = Json::object();
|
||||
p.set("type", Json("image_url"));
|
||||
p.set("image_url", u);
|
||||
return p;
|
||||
}
|
||||
|
||||
Json text_part(const std::string& t) {
|
||||
Json p = Json::object();
|
||||
p.set("type", Json("text"));
|
||||
p.set("text", Json(t));
|
||||
return p;
|
||||
}
|
||||
|
||||
Json tool_call_json(const llm::ToolCall& c) {
|
||||
Json fn = Json::object();
|
||||
fn.set("name", Json(c.name));
|
||||
fn.set("arguments", Json(c.arguments.is_null() ? std::string("{}") : c.arguments.dump()));
|
||||
Json o = Json::object();
|
||||
o.set("id", Json(c.id));
|
||||
o.set("type", Json("function"));
|
||||
o.set("function", fn);
|
||||
return o;
|
||||
}
|
||||
|
||||
void append_message(Json& messages, const llm::Message& m) {
|
||||
switch (m.role) {
|
||||
case llm::Role::kSystem: {
|
||||
Json o = Json::object();
|
||||
o.set("role", Json("system"));
|
||||
o.set("content", Json(m.text()));
|
||||
if (!m.name.empty()) o.set("name", Json(m.name));
|
||||
messages.push(o);
|
||||
return;
|
||||
}
|
||||
case llm::Role::kUser: {
|
||||
Json o = Json::object();
|
||||
o.set("role", Json("user"));
|
||||
bool has_image = false;
|
||||
for (const llm::ContentPart& p : m.parts) has_image = has_image || p.image.has_value();
|
||||
if (!has_image) {
|
||||
o.set("content", Json(m.text()));
|
||||
} else {
|
||||
Json parts = Json::array();
|
||||
for (const llm::ContentPart& p : m.parts) {
|
||||
if (!p.text.empty()) parts.push(text_part(p.text));
|
||||
if (p.image) parts.push(image_url_part(*p.image));
|
||||
}
|
||||
o.set("content", parts);
|
||||
}
|
||||
if (!m.name.empty()) o.set("name", Json(m.name));
|
||||
messages.push(o);
|
||||
return;
|
||||
}
|
||||
case llm::Role::kAssistant: {
|
||||
Json o = Json::object();
|
||||
o.set("role", Json("assistant"));
|
||||
Json calls = Json::array();
|
||||
for (const llm::ContentPart& p : m.parts)
|
||||
if (p.tool_call) calls.push(tool_call_json(*p.tool_call));
|
||||
const std::string text = m.text();
|
||||
if (!text.empty() || calls.as_array().empty()) o.set("content", Json(text));
|
||||
if (!calls.as_array().empty()) o.set("tool_calls", calls);
|
||||
messages.push(o);
|
||||
return;
|
||||
}
|
||||
case llm::Role::kTool: {
|
||||
bool any = false;
|
||||
for (const llm::ContentPart& p : m.parts) {
|
||||
if (!p.tool_result) continue;
|
||||
Json o = Json::object();
|
||||
o.set("role", Json("tool"));
|
||||
o.set("tool_call_id", Json(p.tool_result->call_id));
|
||||
o.set("content", Json(p.tool_result->content));
|
||||
messages.push(o);
|
||||
any = true;
|
||||
}
|
||||
if (!any) bad("a tool message needs a tool result part");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
llm::Usage usage_of(const Json* u) {
|
||||
llm::Usage out;
|
||||
if (!u || !u->is_object()) return out;
|
||||
auto num = [&](const Json& o, const char* k) {
|
||||
const Json* v = o.find(k);
|
||||
return v && v->is_number() ? static_cast<int>(v->as_int()) : 0;
|
||||
};
|
||||
out.input_tokens = num(*u, "prompt_tokens");
|
||||
out.output_tokens = num(*u, "completion_tokens");
|
||||
out.total_tokens = num(*u, "total_tokens");
|
||||
if (const Json* d = u->find("completion_tokens_details"); d && d->is_object())
|
||||
out.reasoning_tokens = num(*d, "reasoning_tokens");
|
||||
if (const Json* d = u->find("prompt_tokens_details"); d && d->is_object())
|
||||
out.cached_tokens = num(*d, "cached_tokens");
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string str(const Json& o, const char* k) {
|
||||
const Json* v = o.is_object() ? o.find(k) : nullptr;
|
||||
return v && v->is_string() ? v->as_string() : std::string();
|
||||
}
|
||||
|
||||
std::string reasoning_of(const Json& holder) {
|
||||
std::string r = str(holder, "reasoning_content");
|
||||
if (r.empty()) r = str(holder, "reasoning");
|
||||
return r;
|
||||
}
|
||||
|
||||
Json parse_arguments(const std::string& raw) {
|
||||
if (raw.empty()) return Json::object();
|
||||
try {
|
||||
Json j = Json::parse(raw);
|
||||
if (j.is_object()) return j;
|
||||
} catch (const std::exception&) {
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("_invalid_json", Json(raw));
|
||||
return o;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Json ChatWire::to_body(const llm::ChatRequest& r, const std::string& default_model, bool stream) {
|
||||
const std::string model = r.model.empty() ? default_model : r.model;
|
||||
if (model.empty() || model.size() > HfCaps::kMaxModelChars) bad("model is empty or too long");
|
||||
if (r.messages.empty()) bad("messages must not be empty");
|
||||
if (r.messages.size() > HfCaps::kMaxMessages) bad("too many messages");
|
||||
if (r.tools.size() > HfCaps::kMaxTools) bad("too many tools");
|
||||
if (r.stop.size() > kMaxStop) bad("at most 4 stop sequences");
|
||||
if (r.json_mode && !r.response_schema.is_null()) bad("json_mode and response_schema are exclusive");
|
||||
if (r.max_tokens && (*r.max_tokens < 1 || *r.max_tokens > HfCaps::kMaxTokensCeiling)) bad("max_tokens out of range");
|
||||
if (r.temperature && (*r.temperature < 0.0 || *r.temperature > 2.0)) bad("temperature must be 0..2");
|
||||
if (r.top_p && (*r.top_p <= 0.0 || *r.top_p > 1.0)) bad("top_p must be in (0, 1]");
|
||||
|
||||
Json body = Json::object();
|
||||
body.set("model", Json(model));
|
||||
Json messages = Json::array();
|
||||
if (!r.system.empty()) {
|
||||
Json s = Json::object();
|
||||
s.set("role", Json("system"));
|
||||
s.set("content", Json(r.system));
|
||||
messages.push(s);
|
||||
}
|
||||
for (const llm::Message& m : r.messages) append_message(messages, m);
|
||||
body.set("messages", messages);
|
||||
if (!r.tools.empty()) {
|
||||
Json tools = Json::array();
|
||||
for (const llm::ToolSpec& t : r.tools) {
|
||||
if (t.name.empty() || t.name.size() > 128) bad("tool name is empty or too long");
|
||||
Json fn = Json::object();
|
||||
fn.set("name", Json(t.name));
|
||||
if (!t.description.empty()) fn.set("description", Json(t.description));
|
||||
fn.set("parameters", t.parameters);
|
||||
Json o = Json::object();
|
||||
o.set("type", Json("function"));
|
||||
o.set("function", fn);
|
||||
tools.push(o);
|
||||
}
|
||||
body.set("tools", tools);
|
||||
}
|
||||
if (!r.tool_choice.empty()) {
|
||||
if (r.tool_choice == "auto" || r.tool_choice == "none" || r.tool_choice == "required") {
|
||||
body.set("tool_choice", Json(r.tool_choice));
|
||||
} else {
|
||||
Json n = Json::object();
|
||||
n.set("name", Json(r.tool_choice));
|
||||
Json c = Json::object();
|
||||
c.set("type", Json("function"));
|
||||
c.set("function", n);
|
||||
body.set("tool_choice", c);
|
||||
}
|
||||
}
|
||||
if (r.max_tokens) body.set("max_tokens", Json(*r.max_tokens));
|
||||
if (r.temperature) body.set("temperature", Json(*r.temperature));
|
||||
if (r.top_p) body.set("top_p", Json(*r.top_p));
|
||||
if (!r.stop.empty()) {
|
||||
Json s = Json::array();
|
||||
for (const std::string& x : r.stop) s.push(Json(x));
|
||||
body.set("stop", s);
|
||||
}
|
||||
if (r.json_mode) {
|
||||
Json f = Json::object();
|
||||
f.set("type", Json("json_object"));
|
||||
body.set("response_format", f);
|
||||
}
|
||||
if (!r.response_schema.is_null()) {
|
||||
Json cfg = Json::object();
|
||||
cfg.set("name", Json("response"));
|
||||
cfg.set("schema", r.response_schema);
|
||||
cfg.set("strict", Json(true));
|
||||
Json f = Json::object();
|
||||
f.set("type", Json("json_schema"));
|
||||
f.set("json_schema", cfg);
|
||||
body.set("response_format", f);
|
||||
}
|
||||
if (!r.reasoning_effort.empty()) body.set("reasoning_effort", Json(r.reasoning_effort));
|
||||
if (r.extra.is_object())
|
||||
for (const auto& [k, v] : r.extra.as_object()) body.set(k, v);
|
||||
if (stream) {
|
||||
body.set("stream", Json(true));
|
||||
Json so = Json::object();
|
||||
so.set("include_usage", Json(true));
|
||||
body.set("stream_options", so);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
llm::ChatResponse ChatWire::to_response(const Json& doc) {
|
||||
llm::ChatResponse out;
|
||||
out.raw = doc;
|
||||
out.id = str(doc, "id");
|
||||
out.model = str(doc, "model");
|
||||
out.message.role = llm::Role::kAssistant;
|
||||
const Json* choices = doc.is_object() ? doc.find("choices") : nullptr;
|
||||
if (!choices || !choices->is_array() || choices->as_array().empty())
|
||||
throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "reply has no choices");
|
||||
const Json& choice = choices->as_array().front();
|
||||
const Json* msg = choice.find("message");
|
||||
out.finish_reason = str(choice, "finish_reason");
|
||||
if (msg && msg->is_object()) {
|
||||
std::string reasoning = reasoning_of(*msg);
|
||||
if (!reasoning.empty()) {
|
||||
llm::ContentPart r;
|
||||
r.reasoning = reasoning;
|
||||
out.message.parts.push_back(r);
|
||||
}
|
||||
std::string content = str(*msg, "content");
|
||||
const Json* calls = msg->find("tool_calls");
|
||||
const bool has_calls = calls && calls->is_array() && !calls->as_array().empty();
|
||||
if (!content.empty() || !has_calls) out.message.parts.push_back(llm::ContentPart::of_text(content));
|
||||
if (has_calls) {
|
||||
for (const Json& c : calls->as_array()) {
|
||||
llm::ToolCall tc;
|
||||
tc.id = str(c, "id");
|
||||
const Json* fn = c.find("function");
|
||||
if (fn && fn->is_object()) {
|
||||
tc.name = str(*fn, "name");
|
||||
const Json* args = fn->find("arguments");
|
||||
if (args && args->is_string()) tc.arguments = parse_arguments(args->as_string());
|
||||
else if (args && args->is_object()) tc.arguments = *args; // TGI may send an object
|
||||
}
|
||||
out.message.parts.push_back(llm::ContentPart::of_call(tc));
|
||||
}
|
||||
}
|
||||
}
|
||||
out.usage = usage_of(doc.find("usage"));
|
||||
return out;
|
||||
}
|
||||
|
||||
bool ChatWire::on_event(const Json& event, StreamState& state,
|
||||
const std::function<bool(const llm::ChatChunk&)>& sink) {
|
||||
if (const Json* err = event.find("error")) {
|
||||
std::string msg = err->is_string() ? err->as_string() : str(*err, "message");
|
||||
throw llm::ApiError("huggingface", 502, "STREAM_ERROR", msg.empty() ? "error event in stream" : msg);
|
||||
}
|
||||
const Json* choices = event.find("choices");
|
||||
if (choices && choices->is_array() && !choices->as_array().empty()) {
|
||||
const Json& choice = choices->as_array().front();
|
||||
if (const Json* delta = choice.find("delta"); delta && delta->is_object()) {
|
||||
llm::ChatChunk c;
|
||||
c.text_delta = str(*delta, "content");
|
||||
c.reasoning_delta = reasoning_of(*delta);
|
||||
if (!c.text_delta.empty() || !c.reasoning_delta.empty())
|
||||
if (!sink(c)) return false;
|
||||
const Json* calls = delta->find("tool_calls");
|
||||
if (calls && calls->is_array()) {
|
||||
for (const Json& f : calls->as_array()) {
|
||||
int index = 0;
|
||||
if (const Json* i = f.find("index"); i && i->is_number()) index = static_cast<int>(i->as_int());
|
||||
std::string id = str(f, "id");
|
||||
if (!id.empty()) state.ids[index] = id;
|
||||
const Json* fn = f.find("function");
|
||||
std::string name = fn ? str(*fn, "name") : "";
|
||||
if (!name.empty()) state.names[index] = name;
|
||||
llm::ChatChunk t;
|
||||
t.tool_call_id = state.ids[index];
|
||||
t.tool_call_name = state.names[index];
|
||||
if (fn) {
|
||||
const Json* a = fn->find("arguments");
|
||||
if (a && a->is_string()) t.tool_args_delta = a->as_string();
|
||||
}
|
||||
if (!sink(t)) return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::string fin = str(choice, "finish_reason");
|
||||
if (!fin.empty()) {
|
||||
llm::ChatChunk c;
|
||||
c.finish_reason = fin;
|
||||
if (!sink(c)) return false;
|
||||
}
|
||||
}
|
||||
if (const Json* u = event.find("usage"); u && u->is_object()) {
|
||||
llm::ChatChunk c;
|
||||
c.usage = usage_of(u);
|
||||
if (!sink(c)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+51
@@ -0,0 +1,51 @@
|
||||
#pragma once
|
||||
|
||||
// OpenAI-compatible chat wire used by every Hugging Face chat surface: the
|
||||
// Inference Providers router, hf-inference and a TGI server share the shape
|
||||
// (huggingface.js tasks/chat-completion/spec/input.json). ChatWire converts
|
||||
// the vendor-neutral llm::ChatRequest into that body and the replies back.
|
||||
//
|
||||
// Decisions:
|
||||
// - tool call arguments that are not valid JSON arrive as {"_invalid_json": raw}
|
||||
// - reasoning text is read from "reasoning_content", else "reasoning" (providers differ)
|
||||
// - response_schema maps to response_format json_schema (strict); json_mode to json_object;
|
||||
// both together are refused
|
||||
// - at most 4 stop sequences (the task spec's documented limit)
|
||||
// - ChatRequest.extra is merged into the body; keys the call table does not list are refused
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
#include "vendors/huggingface/calls.h"
|
||||
#include "llm/chat_types.h"
|
||||
#include "llm/sse.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
// Where a chat body goes: the router, hf-inference, or a TGI server.
|
||||
class ChatEndpoint {
|
||||
public:
|
||||
virtual ~ChatEndpoint() = default;
|
||||
virtual clients::body::Json create(const clients::body::Json& body) = 0;
|
||||
virtual void stream(const clients::body::Json& body, const llm::SseDecoder::Sink& sink) = 0;
|
||||
};
|
||||
|
||||
struct ChatWire {
|
||||
static clients::body::Json to_body(const llm::ChatRequest& request, const std::string& default_model,
|
||||
bool stream);
|
||||
static llm::ChatResponse to_response(const clients::body::Json& doc);
|
||||
|
||||
// Tracks tool call fragments across stream events (index -> id, name).
|
||||
struct StreamState {
|
||||
std::map<int, std::string> ids;
|
||||
std::map<int, std::string> names;
|
||||
};
|
||||
// Converts one SSE event into chunks; the sink returns false to stop.
|
||||
// Returns false when the sink asked to stop. Throws ApiError for an error event.
|
||||
static bool on_event(const clients::body::Json& event, StreamState& state,
|
||||
const std::function<bool(const llm::ChatChunk&)>& sink);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
#include "vendors/huggingface/community.h"
|
||||
|
||||
#include <set>
|
||||
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
void add_if(Query& q, const char* name, const std::string& value) {
|
||||
if (!value.empty()) q.add(name, value);
|
||||
}
|
||||
|
||||
PathArgs discussion_args(RepoType type, const RepoRef& repo, long num) {
|
||||
if (num < 1 || num > 999999999) bad("discussion number must be 1..999999999");
|
||||
PathArgs a;
|
||||
a.repo(repo).repo_type(type).set("num", std::to_string(num));
|
||||
return a;
|
||||
}
|
||||
|
||||
PathArgs repo_args(RepoType type, const RepoRef& repo) {
|
||||
PathArgs a;
|
||||
a.repo(repo).repo_type(type);
|
||||
return a;
|
||||
}
|
||||
|
||||
void text_in(const std::string& what, const std::string& v, std::size_t min, std::size_t max) {
|
||||
if (v.size() < min || v.size() > max)
|
||||
bad(what + " must be " + std::to_string(min) + ".." + std::to_string(max) + " characters");
|
||||
}
|
||||
|
||||
PathArgs collection_args(const std::string& owner, const std::string& slug) {
|
||||
PathArgs a;
|
||||
a.set("namespace", owner).set("slug", slug);
|
||||
return a;
|
||||
}
|
||||
|
||||
const std::set<std::string>& item_types() {
|
||||
static const std::set<std::string> s = {"paper", "collection", "space", "model", "dataset", "bucket"};
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Json Discussions::list(RepoType type, const RepoRef& repo, const DiscussionFilter& f) {
|
||||
if (f.page < 0 || f.page > 100000) bad("page must be 0..100000");
|
||||
Query q;
|
||||
if (f.page > 0) q.add("p", static_cast<long>(f.page));
|
||||
add_if(q, "type", f.kind);
|
||||
add_if(q, "status", f.status);
|
||||
add_if(q, "author", f.author);
|
||||
add_if(q, "search", f.search);
|
||||
add_if(q, "sort", f.sort);
|
||||
return call(Call::kDiscussionList, repo_args(type, repo), q);
|
||||
}
|
||||
|
||||
Json Discussions::get(RepoType type, const RepoRef& repo, long num) {
|
||||
return call(Call::kDiscussionGet, discussion_args(type, repo, num));
|
||||
}
|
||||
|
||||
Json Discussions::create(RepoType type, const RepoRef& repo, const std::string& title,
|
||||
const std::string& description, bool pull_request) {
|
||||
text_in("title", title, 3, 200);
|
||||
text_in("description", description, 0, 65536);
|
||||
Json body = Json::object();
|
||||
body.set("title", Json(title));
|
||||
body.set("description", Json(description));
|
||||
if (pull_request) body.set("pullRequest", Json(true));
|
||||
return call(Call::kDiscussionCreate, repo_args(type, repo), Query(), &body);
|
||||
}
|
||||
|
||||
Json Discussions::comment(RepoType type, const RepoRef& repo, long num, const std::string& text) {
|
||||
text_in("comment", text, 1, 65536);
|
||||
Json body = Json::object();
|
||||
body.set("comment", Json(text));
|
||||
return call(Call::kDiscussionComment, discussion_args(type, repo, num), Query(), &body);
|
||||
}
|
||||
|
||||
Json Discussions::change_status(RepoType type, const RepoRef& repo, long num, const std::string& status,
|
||||
const std::string& comment) {
|
||||
if (status != "open" && status != "closed") bad("status must be open or closed");
|
||||
text_in("comment", comment, 0, 65536);
|
||||
Json body = Json::object();
|
||||
body.set("status", Json(status));
|
||||
if (!comment.empty()) body.set("comment", Json(comment));
|
||||
return call(Call::kDiscussionStatus, discussion_args(type, repo, num), Query(), &body);
|
||||
}
|
||||
|
||||
Json Discussions::rename(RepoType type, const RepoRef& repo, long num, const std::string& title) {
|
||||
text_in("title", title, 3, 200);
|
||||
Json body = Json::object();
|
||||
body.set("title", Json(title));
|
||||
return call(Call::kDiscussionTitle, discussion_args(type, repo, num), Query(), &body);
|
||||
}
|
||||
|
||||
Json Discussions::remove_discussion(RepoType type, const RepoRef& repo, long num) {
|
||||
return call(Call::kDiscussionDelete, discussion_args(type, repo, num));
|
||||
}
|
||||
|
||||
Json CollectionItem::to_json() const {
|
||||
if (!item_types().count(type)) bad("collection item type must be paper, collection, space, model, dataset or bucket");
|
||||
text_in("collection item id", id, 1, HfCaps::kMaxSegmentChars);
|
||||
Json o = Json::object();
|
||||
o.set("type", Json(type));
|
||||
o.set("id", Json(id));
|
||||
return o;
|
||||
}
|
||||
|
||||
Query CollectionListParams::to_query() const {
|
||||
if (limit < 0 || limit > HfCaps::kMaxPageLimit) bad("limit out of range");
|
||||
Query query;
|
||||
add_if(query, "owner", owner);
|
||||
add_if(query, "q", q);
|
||||
add_if(query, "sort", sort);
|
||||
if (limit > 0) query.add("limit", static_cast<long>(limit));
|
||||
if (!item.id.empty()) {
|
||||
item.to_json();
|
||||
query.add("item", item.type + "/" + item.id);
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
Page Collections::list(const CollectionListParams& params, std::size_t max_items, std::size_t max_pages) {
|
||||
return walk(Call::kCollectionsList, PathArgs(), params.to_query(), max_items, max_pages);
|
||||
}
|
||||
|
||||
Json Collections::get(const std::string& owner, const std::string& slug) {
|
||||
return call(Call::kCollectionGet, collection_args(owner, slug));
|
||||
}
|
||||
|
||||
Json Collections::create(const std::string& owner, const std::string& title, const std::string& description,
|
||||
bool is_private) {
|
||||
if (!RequestPlan::valid_name(owner)) bad("bad owner");
|
||||
text_in("title", title, 1, 60);
|
||||
text_in("description", description, 0, 150);
|
||||
Json body = Json::object();
|
||||
body.set("title", Json(title));
|
||||
body.set("namespace", Json(owner));
|
||||
if (!description.empty()) body.set("description", Json(description));
|
||||
if (is_private) body.set("private", Json(true));
|
||||
return call(Call::kCollectionCreate, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json Collections::update(const std::string& owner, const std::string& slug, const Json& fields) {
|
||||
if (!fields.is_object() || fields.as_object().empty()) bad("fields must be a non-empty object");
|
||||
return call(Call::kCollectionUpdate, collection_args(owner, slug), Query(), &fields);
|
||||
}
|
||||
|
||||
Json Collections::add_item(const std::string& owner, const std::string& slug, const CollectionItem& item,
|
||||
const std::string& note) {
|
||||
text_in("note", note, 0, 500);
|
||||
Json body = Json::object();
|
||||
body.set("item", item.to_json());
|
||||
if (!note.empty()) body.set("note", Json(note));
|
||||
return call(Call::kCollectionAddItem, collection_args(owner, slug), Query(), &body);
|
||||
}
|
||||
|
||||
Json Collections::remove_collection(const std::string& owner, const std::string& slug) {
|
||||
return call(Call::kCollectionDelete, collection_args(owner, slug));
|
||||
}
|
||||
|
||||
Json WebhookSpec::to_json() const {
|
||||
static const std::set<std::string> kinds = {"dataset", "model", "space", "bucket", "kernel",
|
||||
"container", "user", "org"};
|
||||
if (url.size() < 8 || url.size() > HfCaps::kMaxServerUrlChars || url.rfind("http", 0) != 0)
|
||||
bad("webhook url must be an http(s) URL");
|
||||
if (watched.empty() || watched.size() > 64) bad("watched must hold 1..64 entries");
|
||||
if (domains.empty() || domains.size() > HfCaps::kMaxWebhookDomains) bad("domains must hold 1.. entries");
|
||||
Json w = Json::array();
|
||||
for (const Watched& x : watched) {
|
||||
if (!kinds.count(x.type)) bad("watched type must be dataset, model, space, bucket, kernel, container, user or org");
|
||||
if (x.name.empty() || x.name.size() > 256) bad("watched name is empty or too long");
|
||||
Json e = Json::object();
|
||||
e.set("type", Json(x.type));
|
||||
e.set("name", Json(x.name));
|
||||
w.push(e);
|
||||
}
|
||||
Json d = Json::array();
|
||||
for (const std::string& dom : domains) {
|
||||
if (dom != "repo" && dom != "discussion") bad("domain must be repo or discussion");
|
||||
d.push(Json(dom));
|
||||
}
|
||||
Json o = Json::object();
|
||||
o.set("url", Json(url));
|
||||
o.set("watched", w);
|
||||
o.set("domains", d);
|
||||
if (!secret.empty()) {
|
||||
for (unsigned char c : secret)
|
||||
if (c < 0x20 || c > 0x7f) bad("secret must be printable ASCII");
|
||||
o.set("secret", Json(secret));
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
Json Webhooks::list() { return call(Call::kWebhooksList); }
|
||||
|
||||
Json Webhooks::get(const std::string& id) {
|
||||
PathArgs a;
|
||||
a.set("webhookId", id);
|
||||
return call(Call::kWebhookGet, a);
|
||||
}
|
||||
|
||||
Json Webhooks::create(const WebhookSpec& spec) {
|
||||
Json body = spec.to_json();
|
||||
return call(Call::kWebhookCreate, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json Webhooks::update(const std::string& id, const WebhookSpec& spec) {
|
||||
Json body = spec.to_json();
|
||||
PathArgs a;
|
||||
a.set("webhookId", id);
|
||||
return call(Call::kWebhookUpdate, a, Query(), &body);
|
||||
}
|
||||
|
||||
Json Webhooks::enable(const std::string& id) {
|
||||
PathArgs a;
|
||||
a.set("webhookId", id).set("action", "enable");
|
||||
return call(Call::kWebhookToggle, a);
|
||||
}
|
||||
|
||||
Json Webhooks::disable(const std::string& id) {
|
||||
PathArgs a;
|
||||
a.set("webhookId", id).set("action", "disable");
|
||||
return call(Call::kWebhookToggle, a);
|
||||
}
|
||||
|
||||
Json Webhooks::replay(const std::string& id, const std::string& log_id) {
|
||||
PathArgs a;
|
||||
a.set("webhookId", id).set("logId", log_id);
|
||||
return call(Call::kWebhookReplay, a);
|
||||
}
|
||||
|
||||
Json Webhooks::remove_webhook(const std::string& id) {
|
||||
PathArgs a;
|
||||
a.set("webhookId", id);
|
||||
return call(Call::kWebhookDelete, a);
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+94
@@ -0,0 +1,94 @@
|
||||
#pragma once
|
||||
|
||||
// Discussions (and pull request threads), Collections and Webhooks on the Hub.
|
||||
// Reads are public where the Hub makes them public; every write needs a token
|
||||
// and is an explicit method. Deletes have no defaults and are named remove_*.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/huggingface/hf_service.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
struct DiscussionFilter {
|
||||
int page = 0; // p
|
||||
std::string kind; // all, discussion, pull_request
|
||||
std::string status; // all, open, closed
|
||||
std::string author;
|
||||
std::string search;
|
||||
std::string sort; // recently-created, trending, reactions
|
||||
};
|
||||
|
||||
class Discussions : public HfService {
|
||||
public:
|
||||
explicit Discussions(Access access) : HfService(std::move(access), Surface::kHub) {}
|
||||
Json list(RepoType type, const RepoRef& repo, const DiscussionFilter& filter = {});
|
||||
Json get(RepoType type, const RepoRef& repo, long num);
|
||||
|
||||
// Writes (token required).
|
||||
Json create(RepoType type, const RepoRef& repo, const std::string& title, const std::string& description,
|
||||
bool pull_request);
|
||||
Json comment(RepoType type, const RepoRef& repo, long num, const std::string& text);
|
||||
Json change_status(RepoType type, const RepoRef& repo, long num, const std::string& status,
|
||||
const std::string& comment = "");
|
||||
Json rename(RepoType type, const RepoRef& repo, long num, const std::string& title);
|
||||
Json remove_discussion(RepoType type, const RepoRef& repo, long num);
|
||||
};
|
||||
|
||||
struct CollectionItem {
|
||||
std::string type; // paper, collection, space, model, dataset, bucket
|
||||
std::string id;
|
||||
Json to_json() const; // validates
|
||||
};
|
||||
|
||||
struct CollectionListParams {
|
||||
std::string owner;
|
||||
std::string q;
|
||||
std::string sort;
|
||||
int limit = 0;
|
||||
CollectionItem item; // optional filter: collections containing this item (id empty: none)
|
||||
Query to_query() const;
|
||||
};
|
||||
|
||||
class Collections : public HfService {
|
||||
public:
|
||||
explicit Collections(Access access) : HfService(std::move(access), Surface::kHub) {}
|
||||
Page list(const CollectionListParams& params, std::size_t max_items = 100, std::size_t max_pages = 5);
|
||||
// slug is "name-<24 hex id>" exactly as the Hub prints it after the owner.
|
||||
Json get(const std::string& owner, const std::string& slug);
|
||||
|
||||
Json create(const std::string& owner, const std::string& title, const std::string& description,
|
||||
bool is_private);
|
||||
Json update(const std::string& owner, const std::string& slug, const Json& fields);
|
||||
Json add_item(const std::string& owner, const std::string& slug, const CollectionItem& item,
|
||||
const std::string& note = "");
|
||||
Json remove_collection(const std::string& owner, const std::string& slug);
|
||||
};
|
||||
|
||||
struct WebhookSpec {
|
||||
struct Watched {
|
||||
std::string type; // dataset, model, space, bucket, kernel, container, user, org
|
||||
std::string name;
|
||||
};
|
||||
std::string url;
|
||||
std::vector<Watched> watched;
|
||||
std::vector<std::string> domains; // repo, discussion
|
||||
std::string secret; // printable ASCII, optional
|
||||
Json to_json() const; // validates
|
||||
};
|
||||
|
||||
class Webhooks : public HfService {
|
||||
public:
|
||||
explicit Webhooks(Access access) : HfService(std::move(access), Surface::kHub) {}
|
||||
Json list();
|
||||
Json get(const std::string& id);
|
||||
Json create(const WebhookSpec& spec);
|
||||
Json update(const std::string& id, const WebhookSpec& spec);
|
||||
Json enable(const std::string& id);
|
||||
Json disable(const std::string& id);
|
||||
Json replay(const std::string& id, const std::string& log_id);
|
||||
Json remove_webhook(const std::string& id);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
#include "vendors/huggingface/endpoints.h"
|
||||
|
||||
#include <cctype>
|
||||
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
struct MetricRow {
|
||||
MetricKind kind;
|
||||
const char* name;
|
||||
Call call;
|
||||
};
|
||||
|
||||
const MetricRow kMetrics[] = {
|
||||
{MetricKind::kAll, "all", Call::kIeGetEndpointMetrics},
|
||||
{MetricKind::kHardwareCpu, "hardware-cpu", Call::kIeGetHardwareCpu},
|
||||
{MetricKind::kHardwareGpu, "hardware-gpu", Call::kIeGetHardwareGpu},
|
||||
{MetricKind::kItl, "itl", Call::kIeGetInferenceItl},
|
||||
{MetricKind::kKvCache, "kv-cache", Call::kIeGetInferenceKvCache},
|
||||
{MetricKind::kPrefixCache, "prefix-cache", Call::kIeGetInferencePrefixCache},
|
||||
{MetricKind::kRequests, "requests", Call::kIeGetInferenceRequests},
|
||||
{MetricKind::kTtft, "ttft", Call::kIeGetInferenceTtft},
|
||||
{MetricKind::kPendingRequest, "pending-request", Call::kIeGetPendingRequest},
|
||||
{MetricKind::kReplicaIds, "replica-ids", Call::kIeGetReplicaIds},
|
||||
{MetricKind::kReplicaRunning, "replica-running", Call::kIeGetReplicaRunning},
|
||||
{MetricKind::kReplicaStatus, "replica-status", Call::kIeGetReplicaStatus},
|
||||
{MetricKind::kResponseElapsed, "response-elapsed", Call::kIeGetResponseElapsed},
|
||||
{MetricKind::kResponseStatusCode, "response-status-code", Call::kIeGetResponseStatusCode},
|
||||
{MetricKind::kResponseStatusCodeGrouped, "response-status-code-grouped", Call::kIeGetStatusCodeGrouped},
|
||||
};
|
||||
|
||||
PathArgs ns_args(const std::string& ns) {
|
||||
PathArgs a;
|
||||
a.set("namespace", ns);
|
||||
return a;
|
||||
}
|
||||
|
||||
PathArgs ep_args(const std::string& ns, const std::string& name) {
|
||||
PathArgs a = ns_args(ns);
|
||||
a.set("name", name);
|
||||
return a;
|
||||
}
|
||||
|
||||
void check_limit(int limit, int max) {
|
||||
if (limit < 0 || limit > max) bad("limit must be 0.." + std::to_string(max));
|
||||
}
|
||||
|
||||
bool rfc3339_like(const std::string& s) {
|
||||
if (s.size() < 20 || s.size() > 40) return false;
|
||||
return std::isdigit(static_cast<unsigned char>(s[0])) && s[4] == '-' && s[7] == '-' && s[10] == 'T';
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const char* metric_kind_name(MetricKind kind) {
|
||||
for (const MetricRow& r : kMetrics)
|
||||
if (r.kind == kind) return r.name;
|
||||
return "all";
|
||||
}
|
||||
|
||||
bool parse_metric_kind(const std::string& text, MetricKind& out) {
|
||||
for (const MetricRow& r : kMetrics)
|
||||
if (text == r.name) {
|
||||
out = r.kind;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void Endpoints::validate_name(const std::string& name) {
|
||||
if (name.empty() || name.size() > 32) bad("endpoint name must be 1..32 characters");
|
||||
for (char c : name)
|
||||
if (!(std::islower(static_cast<unsigned char>(c)) || std::isdigit(static_cast<unsigned char>(c)) || c == '-'))
|
||||
bad("endpoint name: lowercase letters, digits and - only");
|
||||
}
|
||||
|
||||
void Endpoints::validate_spec(const Json& spec, bool creating) {
|
||||
if (!spec.is_object()) bad("endpoint spec must be a JSON object");
|
||||
if (creating) {
|
||||
for (const char* k : {"name", "type", "provider", "compute", "model"})
|
||||
if (!spec.find(k)) bad(std::string("endpoint spec needs ") + k);
|
||||
const Json* n = spec.find("name");
|
||||
if (!n->is_string()) bad("name must be a string");
|
||||
validate_name(n->as_string());
|
||||
}
|
||||
if (const Json* t = spec.find("type")) {
|
||||
if (!t->is_string() || (t->as_string() != "public" && t->as_string() != "authenticated" &&
|
||||
t->as_string() != "private"))
|
||||
bad("type must be public, authenticated or private");
|
||||
}
|
||||
}
|
||||
|
||||
Json Endpoints::list(const std::string& ns, const EndpointListParams& p) {
|
||||
check_limit(p.limit, 100);
|
||||
Query q;
|
||||
if (!p.search.empty()) q.add("search", p.search);
|
||||
if (!p.tags.empty()) q.add("tags", p.tags);
|
||||
if (!p.type.empty()) q.add("type", p.type);
|
||||
if (!p.task.empty()) q.add("task", p.task);
|
||||
if (!p.state.empty()) q.add("state", p.state);
|
||||
if (!p.cursor.empty()) q.add("cursor", p.cursor);
|
||||
if (p.limit > 0) q.add("limit", static_cast<long>(p.limit));
|
||||
return call(Call::kIeListEndpoint, ns_args(ns), q);
|
||||
}
|
||||
|
||||
Json Endpoints::get(const std::string& ns, const std::string& name) {
|
||||
return call(Call::kIeGetEndpoint, ep_args(ns, name));
|
||||
}
|
||||
|
||||
Json Endpoints::replicas(const std::string& ns, const std::string& name) {
|
||||
return call(Call::kIeGetEndpointReplica, ep_args(ns, name));
|
||||
}
|
||||
|
||||
std::string Endpoints::logs(const std::string& ns, const std::string& name, int tail, const std::string& replica) {
|
||||
check_limit(tail, 1000);
|
||||
Query q;
|
||||
if (tail > 0) q.add("tail", static_cast<long>(tail));
|
||||
if (!replica.empty()) q.add("replica", replica);
|
||||
return call_raw(Call::kIeGetEndpointLogs, ep_args(ns, name), q).body;
|
||||
}
|
||||
|
||||
Json Endpoints::logs_v3(const std::string& ns, const std::string& name, int limit) {
|
||||
check_limit(limit, 1000);
|
||||
Query q;
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kIeGetEndpointLogsV3, ep_args(ns, name), q);
|
||||
}
|
||||
|
||||
Json Endpoints::metrics(MetricKind kind, const std::string& ns, const std::string& name,
|
||||
const std::string& start, const std::string& stop, const std::string& replica_id) {
|
||||
if (!rfc3339_like(start) || !rfc3339_like(stop)) bad("start and stop must be RFC 3339 (2021-01-01T00:00:00Z)");
|
||||
Json body = Json::object();
|
||||
body.set("start", Json(start));
|
||||
body.set("stop", Json(stop));
|
||||
if (!replica_id.empty()) body.set("replicaId", Json(replica_id));
|
||||
for (const MetricRow& r : kMetrics)
|
||||
if (r.kind == kind) return call(r.call, ep_args(ns, name), Query(), &body);
|
||||
bad("unknown metric kind");
|
||||
}
|
||||
|
||||
std::string Endpoints::open_metrics(const std::string& ns, const std::string& name) {
|
||||
return call_raw(Call::kIeGetEndpointOpenMetrics, ep_args(ns, name)).body;
|
||||
}
|
||||
|
||||
Json Endpoints::search(const std::string& ns, const std::string& text, int limit) {
|
||||
check_limit(limit, 100);
|
||||
Query q;
|
||||
if (!text.empty()) q.add("search", text);
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kIeSearchEndpoint, ns_args(ns), q);
|
||||
}
|
||||
|
||||
Json Endpoints::state_counts(const std::string& ns) { return call(Call::kIeCountEndpointStates, ns_args(ns)); }
|
||||
|
||||
Json Endpoints::tags(const std::string& ns, const std::string& prefix, int limit) {
|
||||
check_limit(limit, 100);
|
||||
Query q;
|
||||
if (!prefix.empty()) q.add("prefix", prefix);
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kIeListEndpointTags, ns_args(ns), q);
|
||||
}
|
||||
|
||||
Json Endpoints::vendors() { return call(Call::kIeListVendors); }
|
||||
Json Endpoints::vendors_for(const std::string& ns) { return call(Call::kIeListVendorsNamespace, ns_args(ns)); }
|
||||
Json Endpoints::quotas(const std::string& ns) { return call(Call::kIeListVendorsQuotasNamespace, ns_args(ns)); }
|
||||
|
||||
Json Endpoints::audit_logs(const std::string& ns, int limit) {
|
||||
check_limit(limit, 100);
|
||||
Query q;
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kIeGetAuditLogs, ns_args(ns), q);
|
||||
}
|
||||
|
||||
Json Endpoints::cidr_lists(const std::string& ns) { return call(Call::kIeListCidrsList, ns_args(ns)); }
|
||||
Json Endpoints::domains(const std::string& ns) { return call(Call::kIeListDomains, ns_args(ns)); }
|
||||
Json Endpoints::private_services(const std::string& ns) { return call(Call::kIeListPrivateServices, ns_args(ns)); }
|
||||
|
||||
Json Endpoints::create(const std::string& ns, const Json& spec) {
|
||||
validate_spec(spec, true);
|
||||
return call(Call::kIeCreateEndpoint, ns_args(ns), Query(), &spec);
|
||||
}
|
||||
|
||||
Json Endpoints::update(const std::string& ns, const std::string& name, const Json& spec) {
|
||||
validate_spec(spec, false);
|
||||
return call(Call::kIeUpdateEndpoint, ep_args(ns, name), Query(), &spec);
|
||||
}
|
||||
|
||||
Json Endpoints::pause(const std::string& ns, const std::string& name) {
|
||||
return call(Call::kIePauseEndpoint, ep_args(ns, name));
|
||||
}
|
||||
Json Endpoints::resume(const std::string& ns, const std::string& name) {
|
||||
return call(Call::kIeResumeEndpoint, ep_args(ns, name));
|
||||
}
|
||||
Json Endpoints::scale_to_zero(const std::string& ns, const std::string& name) {
|
||||
return call(Call::kIeScaleToZeroEndpoint, ep_args(ns, name));
|
||||
}
|
||||
Json Endpoints::remove_endpoint(const std::string& ns, const std::string& name) {
|
||||
return call(Call::kIeDeleteEndpoint, ep_args(ns, name));
|
||||
}
|
||||
|
||||
Json Endpoints::create_cidr_list(const std::string& ns, const Json& spec) {
|
||||
return call(Call::kIeCreateCidrList, ns_args(ns), Query(), &spec);
|
||||
}
|
||||
|
||||
Json Endpoints::remove_cidr_list(const std::string& ns, const std::string& id) {
|
||||
PathArgs a = ns_args(ns);
|
||||
a.set("cidr_id", id);
|
||||
return call(Call::kIeDeleteCidrList, a);
|
||||
}
|
||||
|
||||
Json Endpoints::create_domain(const std::string& ns, const std::string& domain) {
|
||||
if (domain.empty() || domain.size() > 253) bad("domain is empty or too long");
|
||||
Json body = Json::object();
|
||||
body.set("domain", Json(domain));
|
||||
return call(Call::kIeCreateDomain, ns_args(ns), Query(), &body);
|
||||
}
|
||||
|
||||
Json Endpoints::remove_domain(const std::string& ns, const std::string& domain) {
|
||||
PathArgs a = ns_args(ns);
|
||||
a.set("domain", domain);
|
||||
return call(Call::kIeDeleteDomain, a);
|
||||
}
|
||||
|
||||
Json Endpoints::remove_private_service(const std::string& ns, const std::string& id) {
|
||||
PathArgs a = ns_args(ns);
|
||||
a.set("service-id", id);
|
||||
return call(Call::kIeDeletePrivateService, a);
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+79
@@ -0,0 +1,79 @@
|
||||
#pragma once
|
||||
|
||||
// Inference Endpoints management API (https://api.endpoints.huggingface.cloud,
|
||||
// OpenAPI 2.0.0): endpoints (list, get, create, update, pause, resume, scale
|
||||
// to zero, delete), logs, metrics, replicas, providers and quotas, search,
|
||||
// network (CIDR lists, domains, private services), audit logs.
|
||||
// Creating an endpoint costs money; deleting, pausing and scaling to zero
|
||||
// disrupt service. All of those are explicit methods and the demo only runs
|
||||
// them with "confirm":true. The two GET SSE streams are listed in the call
|
||||
// table as unsupported (ApiClient.stream is POST only).
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "vendors/huggingface/hf_service.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
enum class MetricKind {
|
||||
kAll, kHardwareCpu, kHardwareGpu, kItl, kKvCache, kPrefixCache, kRequests, kTtft,
|
||||
kPendingRequest, kReplicaIds, kReplicaRunning, kReplicaStatus, kResponseElapsed,
|
||||
kResponseStatusCode, kResponseStatusCodeGrouped,
|
||||
};
|
||||
|
||||
const char* metric_kind_name(MetricKind kind);
|
||||
bool parse_metric_kind(const std::string& text, MetricKind& out);
|
||||
|
||||
struct EndpointListParams {
|
||||
std::string search;
|
||||
std::string tags;
|
||||
std::string type; // public, authenticated, private
|
||||
std::string task;
|
||||
std::string state;
|
||||
std::string cursor;
|
||||
int limit = 0; // 1..100
|
||||
};
|
||||
|
||||
class Endpoints : public HfService {
|
||||
public:
|
||||
explicit Endpoints(Access access) : HfService(std::move(access), Surface::kEndpoints) {}
|
||||
static Endpoints from_platform(const Platform& p) { return Endpoints(p.endpoints); }
|
||||
|
||||
// Reads.
|
||||
Json list(const std::string& ns, const EndpointListParams& params = {});
|
||||
Json get(const std::string& ns, const std::string& name);
|
||||
Json replicas(const std::string& ns, const std::string& name);
|
||||
std::string logs(const std::string& ns, const std::string& name, int tail = 100, const std::string& replica = "");
|
||||
Json logs_v3(const std::string& ns, const std::string& name, int limit = 100);
|
||||
Json metrics(MetricKind kind, const std::string& ns, const std::string& name, const std::string& start,
|
||||
const std::string& stop, const std::string& replica_id = "");
|
||||
std::string open_metrics(const std::string& ns, const std::string& name);
|
||||
Json search(const std::string& ns, const std::string& text, int limit = 20);
|
||||
Json state_counts(const std::string& ns);
|
||||
Json tags(const std::string& ns, const std::string& prefix = "", int limit = 50);
|
||||
Json vendors(); // public, no token
|
||||
Json vendors_for(const std::string& ns);
|
||||
Json quotas(const std::string& ns);
|
||||
Json audit_logs(const std::string& ns, int limit = 50);
|
||||
Json cidr_lists(const std::string& ns);
|
||||
Json domains(const std::string& ns);
|
||||
Json private_services(const std::string& ns);
|
||||
|
||||
// Writes. create() costs money while the endpoint runs.
|
||||
Json create(const std::string& ns, const Json& spec);
|
||||
Json update(const std::string& ns, const std::string& name, const Json& spec);
|
||||
Json pause(const std::string& ns, const std::string& name);
|
||||
Json resume(const std::string& ns, const std::string& name);
|
||||
Json scale_to_zero(const std::string& ns, const std::string& name);
|
||||
Json remove_endpoint(const std::string& ns, const std::string& name);
|
||||
Json create_cidr_list(const std::string& ns, const Json& spec);
|
||||
Json remove_cidr_list(const std::string& ns, const std::string& id);
|
||||
Json create_domain(const std::string& ns, const std::string& domain);
|
||||
Json remove_domain(const std::string& ns, const std::string& domain);
|
||||
Json remove_private_service(const std::string& ns, const std::string& id);
|
||||
|
||||
static void validate_name(const std::string& name); // lowercase alnum and '-', 1..32
|
||||
static void validate_spec(const Json& spec, bool creating);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
#include "vendors/huggingface/hf_service.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "clients/url.h"
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/caps.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
HfService::HfService(Access access, Surface surface)
|
||||
: llm::Service(access.client), access_(std::move(access)), surface_(surface) {}
|
||||
|
||||
const CallSpec& HfService::checked(Call call, bool) const {
|
||||
const CallSpec& spec = Calls::spec_of(call);
|
||||
if (spec.surface != surface_)
|
||||
throw llm::ApiError::invalid("huggingface", std::string("call ") + spec.path + " belongs to the " +
|
||||
surface_name(spec.surface) + " surface");
|
||||
if (!std::strcmp(spec.auth, "token") && !access_.has_token)
|
||||
throw llm::NotConfiguredError("huggingface", Platform::kKeyEnv);
|
||||
if (access_.budget) access_.budget->spend();
|
||||
return spec;
|
||||
}
|
||||
|
||||
Json HfService::call(Call c, const PathArgs& args, const Query& query, const Json* body,
|
||||
const llm::Headers& extra) {
|
||||
checked(c);
|
||||
Plan plan = RequestPlan::make(c, args, query, body);
|
||||
return client().request(plan.method, plan.target, body, extra);
|
||||
}
|
||||
|
||||
llm::RawResponse HfService::call_raw(Call c, const PathArgs& args, const Query& query, const Json* body,
|
||||
const llm::Headers& extra) {
|
||||
checked(c);
|
||||
Plan plan = RequestPlan::make(c, args, query, body);
|
||||
std::string text;
|
||||
if (body) text = body->dump();
|
||||
return client().request_raw(plan.method, plan.target, body ? &text : nullptr,
|
||||
body ? "application/json" : "", extra);
|
||||
}
|
||||
|
||||
llm::RawResponse HfService::call_bytes(Call c, const PathArgs& args, const std::string& body,
|
||||
const std::string& content_type, const llm::Headers& extra) {
|
||||
const CallSpec& spec = checked(c);
|
||||
if (!spec.has_body()) throw llm::ApiError::invalid("huggingface", "this call takes no body");
|
||||
if (body.size() > HfCaps::kMaxInferenceInputBytes)
|
||||
throw llm::ApiError::invalid("huggingface", "input is over the byte cap");
|
||||
Plan plan = RequestPlan::make(c, args, Query());
|
||||
return client().request_raw(plan.method, plan.target, &body, content_type, extra);
|
||||
}
|
||||
|
||||
std::string next_cursor_of(const std::string& link, const std::string& expected_host) {
|
||||
// <https://huggingface.co/api/models?limit=2&cursor=...>; rel="next"
|
||||
std::size_t pos = 0;
|
||||
while (pos < link.size()) {
|
||||
std::size_t lt = link.find('<', pos);
|
||||
if (lt == std::string::npos) return "";
|
||||
std::size_t gt = link.find('>', lt);
|
||||
if (gt == std::string::npos) return "";
|
||||
std::size_t next_entry = link.find('<', gt);
|
||||
std::string params = link.substr(gt + 1, (next_entry == std::string::npos ? link.size() : next_entry) - gt - 1);
|
||||
if (params.find("rel=\"next\"") != std::string::npos) {
|
||||
clients::Url url = clients::Url::parse(link.substr(lt + 1, gt - lt - 1));
|
||||
if (url.host() != expected_host)
|
||||
throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "Link header points at another host");
|
||||
auto cur = url.query_value("cursor");
|
||||
return cur ? *cur : "";
|
||||
}
|
||||
if (next_entry == std::string::npos) break;
|
||||
pos = next_entry;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
Page HfService::walk(Call c, const PathArgs& args, const Query& query, std::size_t max_items,
|
||||
std::size_t max_pages) {
|
||||
const CallSpec& spec = Calls::spec_of(c);
|
||||
if (std::strcmp(spec.http_method, "GET") != 0)
|
||||
throw llm::ApiError::invalid("huggingface", "only GET lists can be walked");
|
||||
if (max_items == 0 || max_items > HfCaps::kMaxItems) max_items = HfCaps::kMaxItems;
|
||||
if (max_pages == 0 || max_pages > HfCaps::kMaxPages) max_pages = HfCaps::kMaxPages;
|
||||
clients::Url base = clients::Url::parse(client().config().base_url);
|
||||
Page page;
|
||||
Query q = query;
|
||||
while (true) {
|
||||
checked(c);
|
||||
Plan plan = RequestPlan::make(c, args, q);
|
||||
llm::RawResponse raw = client().request_raw(plan.method, plan.target);
|
||||
Json doc;
|
||||
try {
|
||||
doc = Json::parse(raw.body);
|
||||
} catch (const std::exception& e) {
|
||||
throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", e.what());
|
||||
}
|
||||
if (!doc.is_array()) throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "list reply is not an array");
|
||||
++page.pages;
|
||||
std::string cursor = next_cursor_of(raw.headers.get_or("link", ""), base.host());
|
||||
for (const Json& item : doc.as_array()) {
|
||||
if (page.items.as_array().size() >= max_items) {
|
||||
page.truncated = true;
|
||||
page.truncated_by = "items";
|
||||
page.next_cursor = "";
|
||||
return page;
|
||||
}
|
||||
page.items.push(item);
|
||||
}
|
||||
if (cursor.empty()) return page;
|
||||
if (page.pages >= max_pages) {
|
||||
page.truncated = true;
|
||||
page.truncated_by = "pages";
|
||||
page.next_cursor = cursor;
|
||||
return page;
|
||||
}
|
||||
if (!Calls::in_list(spec.query_params, "cursor"))
|
||||
throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "server paged a call that has no cursor");
|
||||
q.replace("cursor", cursor);
|
||||
}
|
||||
}
|
||||
|
||||
void HfService::stream(Call c, const PathArgs& args, const Json& body, const llm::SseDecoder::Sink& sink,
|
||||
const llm::Headers& extra) {
|
||||
const CallSpec& spec = checked(c);
|
||||
if (std::strcmp(spec.http_method, "POST") != 0)
|
||||
throw llm::ApiError::invalid("huggingface", "streaming is POST only (llm gap)");
|
||||
Plan plan = RequestPlan::make(c, args, Query(), &body);
|
||||
std::size_t events = 0;
|
||||
client().stream(plan.target, body,
|
||||
[&](const llm::SseEvent& e) {
|
||||
if (++events > HfCaps::kMaxStreamEvents) return false;
|
||||
return sink(e);
|
||||
},
|
||||
extra);
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
#pragma once
|
||||
|
||||
// HfService: base of every service in this package. One method per kind of
|
||||
// call (JSON, raw bytes, JSON body to raw reply, page walk, SSE), all driven by
|
||||
// the call table through RequestPlan. Enforces: the call belongs to this
|
||||
// service's surface, a token exists when the row demands one, the shared
|
||||
// request budget, destructive/write rows only through the explicit methods
|
||||
// that pass `allow_write`.
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
#include "vendors/huggingface/calls.h"
|
||||
#include "vendors/huggingface/platform.h"
|
||||
#include "vendors/huggingface/request_plan.h"
|
||||
#include "llm/api_client.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
using Json = clients::body::Json;
|
||||
|
||||
// One walked list: items plus the loud truncation report.
|
||||
struct Page {
|
||||
Json items = Json::array();
|
||||
bool truncated = false; // stopped by a cap while the server had more
|
||||
std::size_t pages = 0;
|
||||
std::string next_cursor; // set when truncated: resume point
|
||||
std::string truncated_by; // "items", "pages" or ""
|
||||
};
|
||||
|
||||
class HfService : public llm::Service {
|
||||
public:
|
||||
HfService(Access access, Surface surface);
|
||||
Surface surface() const { return surface_; }
|
||||
bool has_token() const { return access_.has_token; }
|
||||
|
||||
protected:
|
||||
Json call(Call call, const PathArgs& args = {}, const Query& query = {}, const Json* body = nullptr,
|
||||
const llm::Headers& extra = {});
|
||||
llm::RawResponse call_raw(Call call, const PathArgs& args = {}, const Query& query = {},
|
||||
const Json* body = nullptr, const llm::Headers& extra = {});
|
||||
// Raw request body (binary inference inputs).
|
||||
llm::RawResponse call_bytes(Call call, const PathArgs& args, const std::string& body,
|
||||
const std::string& content_type, const llm::Headers& extra = {});
|
||||
// GET list that follows Link: rel="next" cursors up to the caps.
|
||||
Page walk(Call call, const PathArgs& args, const Query& query, std::size_t max_items,
|
||||
std::size_t max_pages);
|
||||
void stream(Call call, const PathArgs& args, const Json& body, const llm::SseDecoder::Sink& sink,
|
||||
const llm::Headers& extra = {});
|
||||
|
||||
const CallSpec& checked(Call call, bool body_expected = false) const;
|
||||
|
||||
Access access_;
|
||||
Surface surface_;
|
||||
};
|
||||
|
||||
// The cursor in a Link header's rel="next" target, or empty.
|
||||
std::string next_cursor_of(const std::string& link_header, const std::string& expected_host);
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+244
@@ -0,0 +1,244 @@
|
||||
#include "vendors/huggingface/hub.h"
|
||||
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
void check_limit(int limit) {
|
||||
if (limit < 0 || limit > HfCaps::kMaxPageLimit) bad("limit must be 0.." + std::to_string(HfCaps::kMaxPageLimit));
|
||||
}
|
||||
|
||||
void add_if(Query& q, const char* name, const std::string& value) {
|
||||
if (!value.empty()) q.add(name, value);
|
||||
}
|
||||
|
||||
Call info_call(RepoType t, bool revision) {
|
||||
switch (t) {
|
||||
case RepoType::kModel: return revision ? Call::kModelRevision : Call::kModelInfo;
|
||||
case RepoType::kDataset: return revision ? Call::kDatasetRevision : Call::kDatasetInfo;
|
||||
case RepoType::kSpace: return revision ? Call::kSpaceRevision : Call::kSpaceInfo;
|
||||
}
|
||||
return Call::kModelInfo;
|
||||
}
|
||||
|
||||
PathArgs repo_args(RepoType type, const RepoRef& ref) {
|
||||
PathArgs a;
|
||||
a.repo(ref).repo_type(type);
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Query RepoListParams::to_query() const {
|
||||
check_limit(limit);
|
||||
Query q;
|
||||
add_if(q, "search", search);
|
||||
add_if(q, "author", author);
|
||||
add_if(q, "sort", sort);
|
||||
add_if(q, "direction", direction);
|
||||
add_if(q, "pipeline_tag", pipeline_tag);
|
||||
add_if(q, "inference_provider", inference_provider);
|
||||
for (const std::string& f : filter) q.add("filter", f);
|
||||
for (const std::string& e : expand) q.add("expand", e);
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
if (full) q.flag("full", true);
|
||||
return q;
|
||||
}
|
||||
|
||||
Call Hub::list_call(RepoType t) {
|
||||
switch (t) {
|
||||
case RepoType::kModel: return Call::kModelsList;
|
||||
case RepoType::kDataset: return Call::kDatasetsList;
|
||||
case RepoType::kSpace: return Call::kSpacesList;
|
||||
}
|
||||
return Call::kModelsList;
|
||||
}
|
||||
|
||||
Json Hub::whoami() { return call(Call::kWhoami); }
|
||||
|
||||
Page Hub::list_repos(RepoType type, const RepoListParams& params, std::size_t max_items,
|
||||
std::size_t max_pages) {
|
||||
return walk(list_call(type), PathArgs(), params.to_query(), max_items, max_pages);
|
||||
}
|
||||
|
||||
Json Hub::repo_info(RepoType type, const RepoRef& ref, const std::string& revision,
|
||||
const std::vector<std::string>& expand) {
|
||||
PathArgs a;
|
||||
a.set("namespace", ref.owner()).set("repo", ref.name());
|
||||
if (!revision.empty()) a.set("rev", revision);
|
||||
Query q;
|
||||
for (const std::string& e : expand) q.add("expand", e);
|
||||
return call(info_call(type, !revision.empty()), a, q);
|
||||
}
|
||||
|
||||
Json Hub::inference_provider_mapping(const RepoRef& model) {
|
||||
return repo_info(RepoType::kModel, model, "", {"inferenceProviderMapping"});
|
||||
}
|
||||
|
||||
Json Hub::tree(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path,
|
||||
bool recursive, int limit) {
|
||||
check_limit(limit);
|
||||
PathArgs a = repo_args(type, ref);
|
||||
a.set("rev", rev).set("path", path);
|
||||
Query q;
|
||||
if (recursive) q.flag("recursive", true);
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kRepoTree, a, q);
|
||||
}
|
||||
|
||||
Page Hub::tree_all(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path,
|
||||
bool recursive, std::size_t max_items, std::size_t max_pages) {
|
||||
PathArgs a = repo_args(type, ref);
|
||||
a.set("rev", rev).set("path", path);
|
||||
Query q;
|
||||
if (recursive) q.flag("recursive", true);
|
||||
return walk(Call::kRepoTree, a, q, max_items, max_pages);
|
||||
}
|
||||
|
||||
Json Hub::refs(RepoType type, const RepoRef& ref, bool include_prs) {
|
||||
Query q;
|
||||
if (include_prs) q.flag("include_prs", true);
|
||||
return call(Call::kRepoRefs, repo_args(type, ref), q);
|
||||
}
|
||||
|
||||
Json Hub::commits(RepoType type, const RepoRef& ref, const std::string& rev, int limit) {
|
||||
check_limit(limit);
|
||||
PathArgs a = repo_args(type, ref);
|
||||
a.set("rev", rev);
|
||||
Query q;
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kRepoCommits, a, q);
|
||||
}
|
||||
|
||||
Json Hub::compare(RepoType type, const RepoRef& ref, const std::string& cmp) {
|
||||
PathArgs a = repo_args(type, ref);
|
||||
a.set("compare", cmp);
|
||||
return call(Call::kRepoCompare, a);
|
||||
}
|
||||
|
||||
Json Hub::paths_info(RepoType type, const RepoRef& ref, const std::string& rev,
|
||||
const std::vector<std::string>& paths) {
|
||||
if (paths.empty() || paths.size() > HfCaps::kMaxPathsInfo)
|
||||
bad("paths must hold 1.." + std::to_string(HfCaps::kMaxPathsInfo) + " entries");
|
||||
Json arr = Json::array();
|
||||
for (const std::string& p : paths) {
|
||||
RequestPlan::encode_file_path(p); // validates
|
||||
arr.push(Json(p));
|
||||
}
|
||||
Json body = Json::object();
|
||||
body.set("paths", arr);
|
||||
PathArgs a = repo_args(type, ref);
|
||||
a.set("rev", rev);
|
||||
return call(Call::kRepoPathsInfo, a, Query(), &body);
|
||||
}
|
||||
|
||||
Json Hub::likers(RepoType type, const RepoRef& ref) { return call(Call::kRepoLikers, repo_args(type, ref)); }
|
||||
|
||||
Json Hub::safetensors_metadata(const RepoRef& model, const std::string& path) {
|
||||
PathArgs a;
|
||||
a.repo(model).set("path", path);
|
||||
return call(Call::kModelSafetensors, a);
|
||||
}
|
||||
|
||||
Download Hub::download(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path) {
|
||||
PathArgs a = repo_args(type, ref);
|
||||
a.set("rev", rev).set("path", path);
|
||||
llm::RawResponse raw = call_raw(Call::kFileResolve, a);
|
||||
Download d;
|
||||
d.bytes = std::move(raw.body);
|
||||
d.content_type = raw.content_type;
|
||||
d.commit = raw.headers.get_or("x-repo-commit", "");
|
||||
d.etag = raw.headers.get_or("etag", "");
|
||||
return d;
|
||||
}
|
||||
|
||||
Json Hub::quicksearch(const std::string& q, int limit, const std::string& type) {
|
||||
check_limit(limit);
|
||||
if (q.empty()) bad("q must not be empty");
|
||||
Query query;
|
||||
query.add("q", q);
|
||||
if (limit > 0) query.add("limit", static_cast<long>(limit));
|
||||
add_if(query, "type", type);
|
||||
return call(Call::kQuicksearch, PathArgs(), query);
|
||||
}
|
||||
|
||||
Json Hub::full_text_search(const std::string& q, int limit, const std::string& type) {
|
||||
check_limit(limit);
|
||||
if (q.empty()) bad("q must not be empty");
|
||||
Query query;
|
||||
query.add("q", q);
|
||||
if (limit > 0) query.add("limit", static_cast<long>(limit));
|
||||
add_if(query, "type", type);
|
||||
return call(Call::kFullTextSearch, PathArgs(), query);
|
||||
}
|
||||
|
||||
Json Hub::trending(const std::string& type, int limit) {
|
||||
check_limit(limit);
|
||||
Query q;
|
||||
add_if(q, "type", type);
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kTrending, PathArgs(), q);
|
||||
}
|
||||
|
||||
Json Hub::tags_by_type(RepoType type, const std::string& tag_type) {
|
||||
Query q;
|
||||
add_if(q, "type", tag_type);
|
||||
if (type == RepoType::kModel) return call(Call::kModelsTagsByType, PathArgs(), q);
|
||||
if (type == RepoType::kDataset) return call(Call::kDatasetsTagsByType, PathArgs(), q);
|
||||
bad("tags exist for models and datasets only");
|
||||
}
|
||||
|
||||
Json Hub::tasks() { return call(Call::kTasks); }
|
||||
Json Hub::spaces_hardware() { return call(Call::kSpacesHardware); }
|
||||
Json Hub::spaces_featured() { return call(Call::kSpacesFeatured); }
|
||||
|
||||
Json Hub::spaces_semantic_search(const std::string& q) {
|
||||
if (q.empty()) bad("q must not be empty");
|
||||
Query query;
|
||||
query.add("q", q);
|
||||
return call(Call::kSpacesSemanticSearch, PathArgs(), query);
|
||||
}
|
||||
|
||||
Json Hub::space_runtime(const RepoRef& space) {
|
||||
PathArgs a;
|
||||
a.repo(space);
|
||||
return call(Call::kSpaceRuntime, a);
|
||||
}
|
||||
|
||||
Json Hub::daily_papers(int limit, const std::string& date) {
|
||||
check_limit(limit);
|
||||
Query q;
|
||||
if (limit > 0) q.add("limit", static_cast<long>(limit));
|
||||
add_if(q, "date", date);
|
||||
return call(Call::kDailyPapers, PathArgs(), q);
|
||||
}
|
||||
|
||||
Json Hub::papers_search(const std::string& q, int limit) {
|
||||
check_limit(limit);
|
||||
if (q.empty()) bad("q must not be empty");
|
||||
Query query;
|
||||
query.add("q", q);
|
||||
if (limit > 0) query.add("limit", static_cast<long>(limit));
|
||||
return call(Call::kPapersSearch, PathArgs(), query);
|
||||
}
|
||||
|
||||
Json Hub::paper(const std::string& paper_id) {
|
||||
PathArgs a;
|
||||
a.set("paperId", paper_id);
|
||||
return call(Call::kPaperGet, a);
|
||||
}
|
||||
|
||||
Json Hub::partner_models(const std::string& provider, const std::string& status) {
|
||||
PathArgs a;
|
||||
a.set("provider", provider);
|
||||
Query q;
|
||||
add_if(q, "status", status);
|
||||
return call(Call::kPartnerModels, a, q);
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+93
@@ -0,0 +1,93 @@
|
||||
#pragma once
|
||||
|
||||
// Hub reads over https://huggingface.co/api: models, datasets, spaces, repo
|
||||
// trees / refs / commits / files, search, trending, tags, papers, Spaces
|
||||
// catalogs, Inference Provider mappings, whoami, and file download.
|
||||
// Public reads work without a token (Platform builds an anonymous client).
|
||||
// Writes to repositories (commit, upload, delete, settings, branches, LFS) are
|
||||
// outside this module; see SPEC.md ## Scope.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/huggingface/hf_service.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
// Query for GET /api/{models,datasets,spaces}. Fields a repo type does not
|
||||
// accept are refused by the call table, loudly.
|
||||
struct RepoListParams {
|
||||
std::string search;
|
||||
std::string author;
|
||||
std::string sort; // downloads, likes, trendingScore, createdAt, lastModified, ...
|
||||
std::string direction; // "-1" descending
|
||||
std::string pipeline_tag; // models only
|
||||
std::string inference_provider; // models only (provider name or "all")
|
||||
std::vector<std::string> filter;
|
||||
std::vector<std::string> expand; // repeated expand=
|
||||
int limit = 0; // page size, 0: server default; max HfCaps::kMaxPageLimit
|
||||
bool full = false;
|
||||
|
||||
Query to_query() const;
|
||||
};
|
||||
|
||||
struct Download {
|
||||
std::string bytes;
|
||||
std::string content_type;
|
||||
std::string commit; // X-Repo-Commit when sent
|
||||
std::string etag;
|
||||
};
|
||||
|
||||
class Hub : public HfService {
|
||||
public:
|
||||
explicit Hub(Access access) : HfService(std::move(access), Surface::kHub) {}
|
||||
static Hub from_platform(const Platform& p) { return Hub(p.hub); }
|
||||
|
||||
// The call table row that lists this repo type.
|
||||
static Call list_call(RepoType type);
|
||||
|
||||
Json whoami(); // needs a token
|
||||
|
||||
// Lists (page walks, truncation reported).
|
||||
Page list_repos(RepoType type, const RepoListParams& params, std::size_t max_items = 100,
|
||||
std::size_t max_pages = 5);
|
||||
Json repo_info(RepoType type, const RepoRef& ref, const std::string& revision = "",
|
||||
const std::vector<std::string>& expand = {});
|
||||
Json inference_provider_mapping(const RepoRef& model); // model info expanded to the mapping
|
||||
|
||||
// Repo contents.
|
||||
Json tree(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path = "",
|
||||
bool recursive = false, int limit = 0);
|
||||
Page tree_all(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path,
|
||||
bool recursive, std::size_t max_items = 1000, std::size_t max_pages = 10);
|
||||
Json refs(RepoType type, const RepoRef& ref, bool include_prs = false);
|
||||
Json commits(RepoType type, const RepoRef& ref, const std::string& rev, int limit = 0);
|
||||
Json compare(RepoType type, const RepoRef& ref, const std::string& compare);
|
||||
Json paths_info(RepoType type, const RepoRef& ref, const std::string& rev,
|
||||
const std::vector<std::string>& paths);
|
||||
Json likers(RepoType type, const RepoRef& ref);
|
||||
Json safetensors_metadata(const RepoRef& model, const std::string& path);
|
||||
// File bytes through /{repo}/resolve/{rev}/{path}; over the byte cap fails loudly.
|
||||
Download download(RepoType type, const RepoRef& ref, const std::string& rev, const std::string& path);
|
||||
|
||||
// Search and catalogs.
|
||||
Json quicksearch(const std::string& q, int limit = 10, const std::string& type = "");
|
||||
Json full_text_search(const std::string& q, int limit = 10, const std::string& type = "");
|
||||
Json trending(const std::string& type = "", int limit = 0);
|
||||
Json tags_by_type(RepoType type, const std::string& tag_type = ""); // models or datasets
|
||||
Json tasks();
|
||||
Json spaces_hardware();
|
||||
Json spaces_featured();
|
||||
Json spaces_semantic_search(const std::string& q);
|
||||
Json space_runtime(const RepoRef& space); // needs a token
|
||||
|
||||
// Papers.
|
||||
Json daily_papers(int limit = 0, const std::string& date = "");
|
||||
Json papers_search(const std::string& q, int limit = 0);
|
||||
Json paper(const std::string& paper_id);
|
||||
|
||||
// Inference Providers catalog: which models a provider serves.
|
||||
Json partner_models(const std::string& provider, const std::string& status = "");
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+130
@@ -0,0 +1,130 @@
|
||||
#include "vendors/huggingface/platform.h"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "clients/url.h"
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/config.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
RequestBudget::RequestBudget(int max_calls, int window_seconds, Clock clock)
|
||||
: max_calls_(max_calls), window_(window_seconds), clock_(std::move(clock)) {
|
||||
if (!clock_) {
|
||||
clock_ = [] {
|
||||
return static_cast<long>(std::chrono::duration_cast<std::chrono::seconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch())
|
||||
.count());
|
||||
};
|
||||
}
|
||||
window_start_ = clock_();
|
||||
}
|
||||
|
||||
void RequestBudget::spend() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
const long now = clock_();
|
||||
if (now - window_start_ >= window_) {
|
||||
window_start_ = now;
|
||||
used_ = 0;
|
||||
}
|
||||
if (used_ >= max_calls_)
|
||||
throw llm::ApiError("huggingface", 429, "LOCAL_BUDGET",
|
||||
"local request budget spent (" + std::to_string(max_calls_) + " calls per " +
|
||||
std::to_string(window_) + " s); wait for the next window");
|
||||
++used_;
|
||||
}
|
||||
|
||||
int RequestBudget::used() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return used_;
|
||||
}
|
||||
|
||||
std::string Platform::token_from_env() {
|
||||
std::string t = llm::Config::get(kKeyEnv);
|
||||
if (t.empty()) t = llm::Config::get(kLegacyKeyEnv);
|
||||
return t;
|
||||
}
|
||||
|
||||
llm::ApiClientConfig Platform::config(const std::string& base_url, bool has_token) {
|
||||
llm::ApiClientConfig c;
|
||||
c.vendor = "huggingface";
|
||||
c.base_url = base_url;
|
||||
c.key_env = kKeyEnv;
|
||||
if (has_token) {
|
||||
c.auth = llm::AuthStyle::kBearer;
|
||||
} else {
|
||||
c.auth = llm::AuthStyle::kHeader;
|
||||
c.auth_header = "X-Molo-Anonymous"; // marker, not a credential
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
Access make_access(std::unique_ptr<clients::Transport> transport, const std::string& base_url,
|
||||
const std::string& token, std::shared_ptr<RequestBudget> budget) {
|
||||
Access a;
|
||||
a.has_token = !token.empty();
|
||||
a.client = std::make_shared<llm::ApiClient>(std::move(transport), Platform::config(base_url, a.has_token),
|
||||
a.has_token ? token : std::string("1"));
|
||||
a.budget = std::move(budget);
|
||||
return a;
|
||||
}
|
||||
|
||||
std::shared_ptr<RequestBudget> default_budget() {
|
||||
return std::make_shared<RequestBudget>(HfCaps::kBudgetMaxCalls, HfCaps::kBudgetWindowSeconds);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Platform Platform::from_env() {
|
||||
return with(std::make_unique<clients::transports::HttpTransport>(),
|
||||
std::make_unique<clients::transports::HttpTransport>(),
|
||||
std::make_unique<clients::transports::HttpTransport>(), token_from_env());
|
||||
}
|
||||
|
||||
Platform Platform::with(std::unique_ptr<clients::Transport> hub_transport,
|
||||
std::unique_ptr<clients::Transport> router_transport,
|
||||
std::unique_ptr<clients::Transport> endpoints_transport,
|
||||
const std::string& token, std::shared_ptr<RequestBudget> budget) {
|
||||
if (!budget) budget = default_budget();
|
||||
Platform p;
|
||||
p.hub = make_access(std::move(hub_transport), kHubHost, token, budget);
|
||||
p.router = make_access(std::move(router_transport), kRouterHost, token, budget);
|
||||
p.endpoints = make_access(std::move(endpoints_transport), kEndpointsHost, token, budget);
|
||||
return p;
|
||||
}
|
||||
|
||||
std::string Platform::check_server_url(const std::string& url) {
|
||||
if (url.empty() || url.size() > HfCaps::kMaxServerUrlChars)
|
||||
throw llm::ApiError::invalid("huggingface", "server url is empty or too long");
|
||||
clients::Url parsed;
|
||||
try {
|
||||
parsed = clients::Url::parse(url);
|
||||
} catch (const std::exception& e) {
|
||||
throw llm::ApiError::invalid("huggingface", std::string("bad server url: ") + e.what());
|
||||
}
|
||||
if ((parsed.scheme() != "http" && parsed.scheme() != "https") || !parsed.has_host())
|
||||
throw llm::ApiError::invalid("huggingface", "server url must be http(s)://host[:port]");
|
||||
if (url.find('@') != std::string::npos || url.find('?') != std::string::npos ||
|
||||
url.find('#') != std::string::npos)
|
||||
throw llm::ApiError::invalid("huggingface", "server url must not carry credentials, a query or a fragment");
|
||||
std::string out = url;
|
||||
while (!out.empty() && out.back() == '/') out.pop_back();
|
||||
return out;
|
||||
}
|
||||
|
||||
Access Platform::server(const std::string& base_url, const std::string& token,
|
||||
std::shared_ptr<RequestBudget> budget) {
|
||||
return server_with(std::make_unique<clients::transports::HttpTransport>(), base_url, token, std::move(budget));
|
||||
}
|
||||
|
||||
Access Platform::server_with(std::unique_ptr<clients::Transport> transport, const std::string& base_url,
|
||||
const std::string& token, std::shared_ptr<RequestBudget> budget) {
|
||||
if (!budget) budget = default_budget();
|
||||
return make_access(std::move(transport), check_server_url(base_url), token, std::move(budget));
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+83
@@ -0,0 +1,83 @@
|
||||
#pragma once
|
||||
|
||||
// Wiring for the Hugging Face hosts and one shared request budget.
|
||||
//
|
||||
// One credential, HF_TOKEN (the legacy name HUGGING_FACE_HUB_TOKEN is also
|
||||
// read), through llm::Config. The Hub serves public reads without any token,
|
||||
// but llm::ApiClient cannot send an unauthenticated request. Workaround kept
|
||||
// inside this package: without a token the clients are built with a harmless
|
||||
// marker header instead of Authorization (reported in SPEC.md ## Gaps).
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#include "clients/transport.h"
|
||||
#include "llm/api_client.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
// Fixed-window call counter. Local ceiling so one process never trips the
|
||||
// server limit; the clock is injectable for tests.
|
||||
class RequestBudget {
|
||||
public:
|
||||
using Clock = std::function<long()>; // monotonic seconds
|
||||
RequestBudget(int max_calls, int window_seconds, Clock clock = nullptr);
|
||||
// Throws ApiError 429 LOCAL_BUDGET when the window is spent.
|
||||
void spend();
|
||||
int used() const;
|
||||
int max_calls() const { return max_calls_; }
|
||||
|
||||
private:
|
||||
mutable std::mutex mutex_;
|
||||
int max_calls_;
|
||||
int window_;
|
||||
Clock clock_;
|
||||
long window_start_ = 0;
|
||||
int used_ = 0;
|
||||
};
|
||||
|
||||
// What a service needs: a client, whether a token backs it, the shared budget.
|
||||
struct Access {
|
||||
std::shared_ptr<llm::ApiClient> client;
|
||||
bool has_token = false;
|
||||
std::shared_ptr<RequestBudget> budget;
|
||||
};
|
||||
|
||||
struct Platform {
|
||||
static constexpr const char* kHubHost = "https://huggingface.co";
|
||||
static constexpr const char* kRouterHost = "https://router.huggingface.co";
|
||||
static constexpr const char* kEndpointsHost = "https://api.endpoints.huggingface.cloud";
|
||||
static constexpr const char* kKeyEnv = "HF_TOKEN";
|
||||
static constexpr const char* kLegacyKeyEnv = "HUGGING_FACE_HUB_TOKEN";
|
||||
|
||||
Access hub;
|
||||
Access router;
|
||||
Access endpoints;
|
||||
|
||||
// HF_TOKEN, else HUGGING_FACE_HUB_TOKEN, else empty.
|
||||
static std::string token_from_env();
|
||||
// Real HTTPS transports. A missing token still builds working anonymous
|
||||
// clients (public Hub reads); calls that need a token fail NOT_CONFIGURED.
|
||||
static Platform from_env();
|
||||
static Platform with(std::unique_ptr<clients::Transport> hub_transport,
|
||||
std::unique_ptr<clients::Transport> router_transport,
|
||||
std::unique_ptr<clients::Transport> endpoints_transport,
|
||||
const std::string& token,
|
||||
std::shared_ptr<RequestBudget> budget = nullptr);
|
||||
bool configured() const { return hub.has_token; }
|
||||
|
||||
// A TGI / TEI server (Inference Endpoint URL, localhost, any host). The token
|
||||
// is optional: self-hosted servers usually have none.
|
||||
static Access server(const std::string& base_url, const std::string& token,
|
||||
std::shared_ptr<RequestBudget> budget = nullptr);
|
||||
static Access server_with(std::unique_ptr<clients::Transport> transport, const std::string& base_url,
|
||||
const std::string& token, std::shared_ptr<RequestBudget> budget = nullptr);
|
||||
// http(s) URL with a host, no userinfo, query or fragment; trailing / removed.
|
||||
static std::string check_server_url(const std::string& url);
|
||||
|
||||
static llm::ApiClientConfig config(const std::string& base_url, bool has_token);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+188
@@ -0,0 +1,188 @@
|
||||
#include "vendors/huggingface/providers.h"
|
||||
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/error.h"
|
||||
#include "security/crypto/Base64.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
// hf-inference task map, from huggingface_hub inference/_providers/__init__.py
|
||||
// ("hf-inference" entry) and hf_inference.py (binary input classes).
|
||||
const TaskInfo kTasks[] = {
|
||||
{"text-to-image", false, false, "binary image reply"},
|
||||
{"text-generation", false, false, ""},
|
||||
{"text-classification", false, false, ""},
|
||||
{"question-answering", false, false, ""},
|
||||
{"audio-classification", true, false, ""},
|
||||
{"automatic-speech-recognition", true, false, ""},
|
||||
{"fill-mask", false, false, ""},
|
||||
{"feature-extraction", false, true, "parameters at the top level"},
|
||||
{"image-classification", true, false, ""},
|
||||
{"image-segmentation", true, false, ""},
|
||||
{"document-question-answering", false, false, ""},
|
||||
{"image-to-text", true, false, ""},
|
||||
{"object-detection", true, false, ""},
|
||||
{"audio-to-audio", true, false, ""},
|
||||
{"zero-shot-image-classification", true, false, ""},
|
||||
{"zero-shot-classification", false, false, ""},
|
||||
{"image-to-image", true, false, "binary image reply"},
|
||||
{"sentence-similarity", false, true, ""},
|
||||
{"table-question-answering", false, false, ""},
|
||||
{"tabular-classification", false, false, ""},
|
||||
{"text-to-speech", false, false, "binary audio reply"},
|
||||
{"token-classification", false, false, ""},
|
||||
{"translation", false, false, ""},
|
||||
{"summarization", false, false, ""},
|
||||
{"visual-question-answering", true, false, ""},
|
||||
};
|
||||
|
||||
bool is_json_type(const std::string& ct) {
|
||||
return ct.find("json") != std::string::npos;
|
||||
}
|
||||
|
||||
InferenceResult to_result(const llm::RawResponse& raw) {
|
||||
InferenceResult r;
|
||||
r.content_type = raw.content_type;
|
||||
if (is_json_type(raw.content_type) || raw.content_type.empty()) {
|
||||
try {
|
||||
r.json = Json::parse(raw.body);
|
||||
r.is_json = true;
|
||||
return r;
|
||||
} catch (const std::exception&) {
|
||||
if (!raw.content_type.empty())
|
||||
throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "JSON reply does not parse");
|
||||
}
|
||||
}
|
||||
r.bytes = raw.body;
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
RouterModel RouterModel::parse(const std::string& text) {
|
||||
if (text.empty() || text.size() > HfCaps::kMaxModelChars) bad("model is empty or too long");
|
||||
RouterModel m;
|
||||
const std::size_t colon = text.find(':');
|
||||
m.id = text.substr(0, colon);
|
||||
if (colon != std::string::npos) {
|
||||
m.selector = text.substr(colon + 1);
|
||||
if (!llm::Service::valid_id(m.selector) || m.selector.find(':') != std::string::npos)
|
||||
bad("bad routing selector after ':' (fastest, cheapest, preferred or a provider id)");
|
||||
}
|
||||
RequestPlan::check_model(m.id);
|
||||
return m;
|
||||
}
|
||||
|
||||
void RouterChat::set_bill_to(const std::string& organization) {
|
||||
if (!organization.empty() && !RequestPlan::valid_name(organization)) bad("bad organization name");
|
||||
bill_to_ = organization;
|
||||
}
|
||||
|
||||
llm::Headers RouterChat::headers() const {
|
||||
llm::Headers h;
|
||||
if (!bill_to_.empty()) h["X-HF-Bill-To"] = bill_to_;
|
||||
return h;
|
||||
}
|
||||
|
||||
Json RouterChat::create(const Json& body) {
|
||||
const Json* model = body.is_object() ? body.find("model") : nullptr;
|
||||
if (!model || !model->is_string()) bad("model is required");
|
||||
RouterModel::parse(model->as_string());
|
||||
if (const Json* s = body.find("stream"); s && s->is_bool() && s->as_bool()) bad("use stream() for streaming");
|
||||
return call(Call::kRouterChat, PathArgs(), Query(), &body, headers());
|
||||
}
|
||||
|
||||
void RouterChat::stream(const Json& body, const llm::SseDecoder::Sink& sink) {
|
||||
const Json* model = body.is_object() ? body.find("model") : nullptr;
|
||||
if (!model || !model->is_string()) bad("model is required");
|
||||
RouterModel::parse(model->as_string());
|
||||
HfService::stream(Call::kRouterChat, PathArgs(), body, sink, headers());
|
||||
}
|
||||
|
||||
Json RouterChat::models() { return call(Call::kRouterModels); }
|
||||
|
||||
Json RouterChat::create_hf_inference(const std::string& model, const Json& body) {
|
||||
PathArgs a;
|
||||
a.set("model", model);
|
||||
return call(Call::kHfInferenceChat, a, Query(), &body, headers());
|
||||
}
|
||||
|
||||
std::size_t TaskCatalog::count() { return sizeof(kTasks) / sizeof(kTasks[0]); }
|
||||
const TaskInfo& TaskCatalog::at(std::size_t i) {
|
||||
if (i >= count()) bad("task index out of range");
|
||||
return kTasks[i];
|
||||
}
|
||||
const TaskInfo* TaskCatalog::find(const std::string& name) {
|
||||
for (const TaskInfo& t : kTasks)
|
||||
if (name == t.name) return &t;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
InferenceResult InferenceTasks::run_json(const std::string& task, const std::string& model, const Json& inputs,
|
||||
const Json& parameters) {
|
||||
const TaskInfo* info = TaskCatalog::find(task);
|
||||
if (!info) bad("unknown hf-inference task " + task);
|
||||
if (info->binary_input) bad("task " + task + " takes a file input; use run_binary");
|
||||
if (inputs.is_null()) bad("inputs is required");
|
||||
RequestPlan::check_model(model);
|
||||
Json body = Json::object();
|
||||
body.set("inputs", inputs);
|
||||
if (info->pipeline_path) {
|
||||
if (parameters.is_object())
|
||||
for (const auto& [k, v] : parameters.as_object()) body.set(k, v);
|
||||
} else if (!parameters.is_null()) {
|
||||
if (!parameters.is_object()) bad("parameters must be an object");
|
||||
if (!parameters.as_object().empty()) body.set("parameters", parameters);
|
||||
}
|
||||
PathArgs a;
|
||||
a.set("model", model);
|
||||
if (info->pipeline_path) {
|
||||
a.set("task", task);
|
||||
return to_result(call_raw(Call::kHfInferencePipeline, a, Query(), &body));
|
||||
}
|
||||
return to_result(call_raw(Call::kHfInferenceTask, a, Query(), &body));
|
||||
}
|
||||
|
||||
InferenceResult InferenceTasks::run_binary(const std::string& task, const std::string& model,
|
||||
const std::string& bytes, const std::string& mime,
|
||||
const Json& parameters) {
|
||||
const TaskInfo* info = TaskCatalog::find(task);
|
||||
if (!info) bad("unknown hf-inference task " + task);
|
||||
if (!info->binary_input) bad("task " + task + " takes JSON input; use run_json");
|
||||
if (bytes.empty()) bad("input bytes are empty");
|
||||
if (bytes.size() > HfCaps::kMaxInferenceInputBytes) bad("input is over the byte cap");
|
||||
RequestPlan::check_model(model);
|
||||
PathArgs a;
|
||||
a.set("model", model);
|
||||
const bool has_params = parameters.is_object() && !parameters.as_object().empty();
|
||||
if (!has_params) {
|
||||
if (mime.empty() || mime.find('\n') != std::string::npos) bad("mime type required");
|
||||
return to_result(call_bytes(Call::kHfInferenceTask, a, bytes, mime));
|
||||
}
|
||||
Json body = Json::object();
|
||||
body.set("inputs", Json(security::crypto::Base64::encode(bytes)));
|
||||
body.set("parameters", parameters);
|
||||
return to_result(call_raw(Call::kHfInferenceTask, a, Query(), &body));
|
||||
}
|
||||
|
||||
Json InferenceTasks::embed(const std::string& model, const std::vector<std::string>& texts, bool normalize) {
|
||||
if (texts.empty() || texts.size() > HfCaps::kMaxBatchInputs) bad("texts must hold 1..256 entries");
|
||||
Json arr = Json::array();
|
||||
std::size_t total = 0;
|
||||
for (const std::string& t : texts) {
|
||||
total += t.size();
|
||||
if (total > HfCaps::kMaxTextChars) bad("texts are over the size cap");
|
||||
arr.push(Json(t));
|
||||
}
|
||||
Json params = Json::object();
|
||||
if (normalize) params.set("normalize", Json(true));
|
||||
InferenceResult r = run_json("feature-extraction", model, texts.size() == 1 ? Json(texts[0]) : arr, params);
|
||||
if (!r.is_json) throw llm::ApiError("huggingface", 502, "BAD_RESPONSE", "embedding reply is not JSON");
|
||||
return r.json;
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+85
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
// Inference Providers (https://router.huggingface.co): the OpenAI-compatible
|
||||
// chat router plus the hf-inference task APIs (the classic Inference API).
|
||||
// The classic host api-inference.huggingface.co no longer resolves (probe
|
||||
// 2026-10-25); the same tasks live under /hf-inference/models/{model}.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/huggingface/chat_wire.h"
|
||||
#include "vendors/huggingface/hf_service.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
// "owner/name" or "owner/name:selector" where selector is fastest, cheapest,
|
||||
// preferred or a provider id (the router's routing suffix).
|
||||
struct RouterModel {
|
||||
std::string id;
|
||||
std::string selector;
|
||||
static RouterModel parse(const std::string& text); // throws ApiError::invalid
|
||||
std::string str() const { return selector.empty() ? id : id + ":" + selector; }
|
||||
};
|
||||
|
||||
class RouterChat : public HfService, public ChatEndpoint {
|
||||
public:
|
||||
explicit RouterChat(Access access) : HfService(std::move(access), Surface::kRouter) {}
|
||||
static RouterChat from_platform(const Platform& p) { return RouterChat(p.router); }
|
||||
|
||||
// Chat completions through the router. `bill_to` sets X-HF-Bill-To (an
|
||||
// organization that pays); empty: the token owner pays.
|
||||
Json create(const Json& body) override;
|
||||
void stream(const Json& body, const llm::SseDecoder::Sink& sink) override;
|
||||
void set_bill_to(const std::string& organization);
|
||||
|
||||
// GET /v1/models: public, no token needed.
|
||||
Json models();
|
||||
|
||||
// Chat straight to hf-inference for one model (no provider routing).
|
||||
Json create_hf_inference(const std::string& model, const Json& body);
|
||||
|
||||
private:
|
||||
llm::Headers headers() const;
|
||||
std::string bill_to_;
|
||||
};
|
||||
|
||||
// One hf-inference task.
|
||||
struct TaskInfo {
|
||||
const char* name; // pipeline tag: text-classification, ...
|
||||
bool binary_input; // the input is a file (image, audio), not JSON
|
||||
bool pipeline_path; // /pipeline/{task} (feature-extraction, sentence-similarity)
|
||||
const char* note;
|
||||
};
|
||||
|
||||
struct TaskCatalog {
|
||||
static std::size_t count();
|
||||
static const TaskInfo& at(std::size_t index);
|
||||
static const TaskInfo* find(const std::string& name); // nullptr when unknown
|
||||
};
|
||||
|
||||
struct InferenceResult {
|
||||
bool is_json = false;
|
||||
Json json; // when is_json
|
||||
std::string bytes; // images, audio
|
||||
std::string content_type;
|
||||
};
|
||||
|
||||
class InferenceTasks : public HfService {
|
||||
public:
|
||||
explicit InferenceTasks(Access access) : HfService(std::move(access), Surface::kRouter) {}
|
||||
static InferenceTasks from_platform(const Platform& p) { return InferenceTasks(p.router); }
|
||||
|
||||
// JSON-input tasks: {"inputs": inputs, "parameters": parameters}. For the
|
||||
// pipeline tasks the parameters go at the top level, as huggingface_hub does.
|
||||
InferenceResult run_json(const std::string& task, const std::string& model, const Json& inputs,
|
||||
const Json& parameters = Json());
|
||||
// File-input tasks: raw bytes when there are no parameters, else the
|
||||
// base64 JSON form huggingface_hub sends.
|
||||
InferenceResult run_binary(const std::string& task, const std::string& model, const std::string& bytes,
|
||||
const std::string& mime, const Json& parameters = Json());
|
||||
// feature-extraction convenience: embeddings for 1..HfCaps::kMaxBatchInputs texts.
|
||||
Json embed(const std::string& model, const std::vector<std::string>& texts, bool normalize = false);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
#include "vendors/huggingface/request_plan.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <set>
|
||||
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/api_client.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
bool has_control(const std::string& s) {
|
||||
for (unsigned char c : s)
|
||||
if (c < 0x20 || c == 0x7f) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool all_digits(const std::string& s) {
|
||||
if (s.empty()) return false;
|
||||
for (char c : s)
|
||||
if (!std::isdigit(static_cast<unsigned char>(c))) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string segment(const std::string& name, const std::string& value) {
|
||||
if (value.empty()) bad(name + " must not be empty");
|
||||
if (value.size() > HfCaps::kMaxSegmentChars) bad(name + " is too long");
|
||||
if (has_control(value)) bad(name + " has control characters");
|
||||
if (value == "." || value == "..") bad(name + " must not be . or ..");
|
||||
return RequestPlan::encode(value);
|
||||
}
|
||||
|
||||
std::string render_value(const std::string& name, const std::string& value) {
|
||||
if (name == "namespace" || name == "repo" || name == "name") {
|
||||
if (!RequestPlan::valid_name(value)) bad("bad " + name + " (letters, digits, - _ . only)");
|
||||
return value;
|
||||
}
|
||||
if (name == "repoType") {
|
||||
RepoType t;
|
||||
if (!parse_repo_type(value, t)) bad("repoType must be models, datasets or spaces");
|
||||
return repo_type_plural(t);
|
||||
}
|
||||
if (name == "typePrefix") {
|
||||
RepoType t;
|
||||
if (!parse_repo_type(value, t)) bad("repoType must be models, datasets or spaces");
|
||||
return t == RepoType::kModel ? "" : std::string("/") + repo_type_plural(t);
|
||||
}
|
||||
if (name == "path") return RequestPlan::encode_file_path(value);
|
||||
if (name == "model") return RequestPlan::check_model(value);
|
||||
if (name == "num") {
|
||||
if (!all_digits(value) || value.size() > 9) bad("num must be a discussion number");
|
||||
return value;
|
||||
}
|
||||
if (name == "action") {
|
||||
if (value != "enable" && value != "disable") bad("action must be enable or disable");
|
||||
return value;
|
||||
}
|
||||
if (name == "task" || name == "provider") {
|
||||
if (!llm::Service::valid_id(value)) bad("bad " + name);
|
||||
return value;
|
||||
}
|
||||
return segment(name, value);
|
||||
}
|
||||
|
||||
std::string query_encode_value(const std::string& v) {
|
||||
std::string out = RequestPlan::encode(v);
|
||||
// ApiClient refuses ".." anywhere in the request path, query included.
|
||||
if (v.find("..") != std::string::npos) {
|
||||
std::string fixed;
|
||||
for (char c : out) {
|
||||
if (c == '.') fixed += "%2E";
|
||||
else fixed += c;
|
||||
}
|
||||
return fixed;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
const char* repo_type_plural(RepoType type) {
|
||||
switch (type) {
|
||||
case RepoType::kModel: return "models";
|
||||
case RepoType::kDataset: return "datasets";
|
||||
case RepoType::kSpace: return "spaces";
|
||||
}
|
||||
return "models";
|
||||
}
|
||||
|
||||
bool parse_repo_type(const std::string& text, RepoType& out) {
|
||||
if (text == "models" || text == "model") out = RepoType::kModel;
|
||||
else if (text == "datasets" || text == "dataset") out = RepoType::kDataset;
|
||||
else if (text == "spaces" || text == "space") out = RepoType::kSpace;
|
||||
else return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
RepoRef RepoRef::parse(const std::string& id) {
|
||||
const std::size_t slash = id.find('/');
|
||||
if (slash == std::string::npos)
|
||||
bad("repo id must be owner/name (for example openai-community/gpt2); legacy ids without an owner redirect");
|
||||
RepoRef r;
|
||||
r.owner_ = id.substr(0, slash);
|
||||
r.name_ = id.substr(slash + 1);
|
||||
if (!RequestPlan::valid_name(r.owner_) || !RequestPlan::valid_name(r.name_))
|
||||
bad("bad repo id (owner/name, letters, digits, - _ . only)");
|
||||
return r;
|
||||
}
|
||||
|
||||
PathArgs& PathArgs::set(const std::string& name, std::string value) {
|
||||
values_[name] = std::move(value);
|
||||
return *this;
|
||||
}
|
||||
|
||||
PathArgs& PathArgs::repo(const RepoRef& ref) {
|
||||
values_["namespace"] = ref.owner();
|
||||
values_["repo"] = ref.name();
|
||||
return *this;
|
||||
}
|
||||
|
||||
PathArgs& PathArgs::repo_type(RepoType type) {
|
||||
values_["repoType"] = repo_type_plural(type);
|
||||
values_["typePrefix"] = repo_type_plural(type);
|
||||
return *this;
|
||||
}
|
||||
|
||||
const std::string* PathArgs::find(const std::string& name) const {
|
||||
auto it = values_.find(name);
|
||||
return it == values_.end() ? nullptr : &it->second;
|
||||
}
|
||||
|
||||
Query& Query::add(const std::string& name, const std::string& value) {
|
||||
if (items_.size() >= HfCaps::kMaxQueryParams) bad("too many query parameters");
|
||||
if (value.size() > HfCaps::kMaxQueryValueChars) bad("query parameter " + name + " is too long");
|
||||
if (has_control(value)) bad("query parameter " + name + " has control characters");
|
||||
items_.emplace_back(name, value);
|
||||
return *this;
|
||||
}
|
||||
|
||||
Query& Query::add(const std::string& name, long value) { return add(name, std::to_string(value)); }
|
||||
|
||||
Query& Query::flag(const std::string& name, bool value) { return add(name, value ? "true" : "false"); }
|
||||
|
||||
bool Query::has(const std::string& name) const {
|
||||
for (const auto& kv : items_)
|
||||
if (kv.first == name) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
Query& Query::replace(const std::string& name, const std::string& value) {
|
||||
std::vector<std::pair<std::string, std::string>> kept;
|
||||
for (auto& kv : items_)
|
||||
if (kv.first != name) kept.push_back(kv);
|
||||
items_ = std::move(kept);
|
||||
return add(name, value);
|
||||
}
|
||||
|
||||
std::string RequestPlan::encode(const std::string& text) {
|
||||
static const char* hex = "0123456789ABCDEF";
|
||||
std::string out;
|
||||
for (unsigned char c : text) {
|
||||
if (std::isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~') {
|
||||
out += static_cast<char>(c);
|
||||
} else {
|
||||
out += '%';
|
||||
out += hex[c >> 4];
|
||||
out += hex[c & 15];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
bool RequestPlan::valid_name(const std::string& text) {
|
||||
if (text.empty() || text.size() > HfCaps::kMaxSegmentChars || text == "." || text == "..") return false;
|
||||
for (char c : text)
|
||||
if (!(std::isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '_' || c == '.')) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string RequestPlan::encode_file_path(const std::string& path) {
|
||||
if (path.empty()) bad("path must not be empty");
|
||||
if (path.size() > HfCaps::kMaxFilePathChars) bad("path is too long");
|
||||
if (path.front() == '/' || path.back() == '/') bad("path must not start or end with /");
|
||||
std::string out;
|
||||
std::size_t segments = 0, start = 0;
|
||||
while (start <= path.size()) {
|
||||
std::size_t end = path.find('/', start);
|
||||
if (end == std::string::npos) end = path.size();
|
||||
std::string seg = path.substr(start, end - start);
|
||||
if (seg.empty()) bad("path has an empty segment");
|
||||
if (++segments > HfCaps::kMaxPathSegments) bad("path has too many segments");
|
||||
if (!out.empty()) out += '/';
|
||||
out += segment("path segment", seg);
|
||||
start = end + 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string RequestPlan::check_model(const std::string& model) {
|
||||
if (model.empty() || model.size() > HfCaps::kMaxModelChars) bad("model id is empty or too long");
|
||||
const std::size_t slash = model.find('/');
|
||||
if (slash == std::string::npos) {
|
||||
if (!valid_name(model)) bad("bad model id");
|
||||
return model;
|
||||
}
|
||||
if (model.find('/', slash + 1) != std::string::npos) bad("model id must be owner/name");
|
||||
if (!valid_name(model.substr(0, slash)) || !valid_name(model.substr(slash + 1))) bad("bad model id");
|
||||
return model;
|
||||
}
|
||||
|
||||
Plan RequestPlan::make(Call call, const PathArgs& args, const Query& query, const clients::body::Json* body) {
|
||||
const CallSpec& spec = Calls::spec_of(call);
|
||||
if (spec.has_flag("unsupported"))
|
||||
bad(std::string("call is listed but not supported: ") + spec.path + " (see SPEC.md ## Gaps)");
|
||||
std::string out;
|
||||
std::set<std::string> used;
|
||||
const std::string tpl = spec.path;
|
||||
for (std::size_t i = 0; i < tpl.size();) {
|
||||
if (tpl[i] != '{') {
|
||||
out += tpl[i++];
|
||||
continue;
|
||||
}
|
||||
const std::size_t close = tpl.find('}', i);
|
||||
if (close == std::string::npos) bad("malformed call table path");
|
||||
const std::string name = tpl.substr(i + 1, close - i - 1);
|
||||
i = close + 1;
|
||||
const std::string* v = args.find(name);
|
||||
const bool last_optional = name == "path" && spec.has_flag("optpath") && i == tpl.size();
|
||||
if (!v || (v->empty() && last_optional)) {
|
||||
if (last_optional) {
|
||||
if (!out.empty() && out.back() == '/') out.pop_back();
|
||||
used.insert(name);
|
||||
continue;
|
||||
}
|
||||
bad("missing path argument " + name);
|
||||
}
|
||||
used.insert(name);
|
||||
out += render_value(name, *v);
|
||||
}
|
||||
for (const auto& kv : args.all()) {
|
||||
// repo_type() sets both repoType and typePrefix; only the ones the path uses count.
|
||||
if (!used.count(kv.first) && kv.first != "repoType" && kv.first != "typePrefix")
|
||||
bad("path argument " + kv.first + " is not used by " + spec.path);
|
||||
}
|
||||
if (spec.path[0] == '{' && out.empty()) out = "/";
|
||||
std::string qs;
|
||||
for (const auto& kv : query.items()) {
|
||||
if (!Calls::in_list(spec.query_params, kv.first))
|
||||
bad("query parameter " + kv.first + " is not accepted by " + spec.http_method + " " + spec.path);
|
||||
if (!qs.empty()) qs += '&';
|
||||
qs += encode(kv.first) + "=" + query_encode_value(kv.second);
|
||||
}
|
||||
if (body) {
|
||||
if (!spec.has_body()) bad(std::string(spec.http_method) + " " + spec.path + " takes no body");
|
||||
if (!body->is_object()) bad("body must be a JSON object");
|
||||
for (const auto& kv : body->as_object())
|
||||
if (!Calls::in_list(spec.body_fields, kv.first))
|
||||
bad("body field " + kv.first + " is not accepted by " + spec.http_method + " " + spec.path);
|
||||
}
|
||||
Plan plan;
|
||||
plan.call = call;
|
||||
plan.method = Calls::method_of(call);
|
||||
plan.target = out + (qs.empty() ? "" : "?" + qs);
|
||||
return plan;
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
// RequestPlan: turns one row of the call table plus caller arguments into a
|
||||
// validated request target (path and query). Every id is checked before it is
|
||||
// placed in a path, every query parameter and body field must be one the table
|
||||
// lists, every size is capped (caps.h). Nothing here touches the network.
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/body/json.h"
|
||||
#include "clients/request.h"
|
||||
#include "vendors/huggingface/calls.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
// Hub repository kinds the module reads. Kernels exist on the Hub but are
|
||||
// outside this module (SPEC.md ## Gaps).
|
||||
enum class RepoType { kModel, kDataset, kSpace };
|
||||
|
||||
const char* repo_type_plural(RepoType type); // models, datasets, spaces
|
||||
bool parse_repo_type(const std::string& text, RepoType& out); // accepts singular or plural
|
||||
|
||||
// "owner/name" validated once. Legacy ids without an owner (gpt2) are refused
|
||||
// loudly: the Hub answers them with a redirect to the canonical id.
|
||||
class RepoRef {
|
||||
public:
|
||||
static RepoRef parse(const std::string& id); // throws ApiError::invalid
|
||||
const std::string& owner() const { return owner_; }
|
||||
const std::string& name() const { return name_; }
|
||||
std::string id() const { return owner_ + "/" + name_; }
|
||||
|
||||
private:
|
||||
std::string owner_;
|
||||
std::string name_;
|
||||
};
|
||||
|
||||
class PathArgs {
|
||||
public:
|
||||
PathArgs& set(const std::string& name, std::string value);
|
||||
PathArgs& repo(const RepoRef& ref); // namespace + repo
|
||||
PathArgs& repo_type(RepoType type); // repoType + typePrefix
|
||||
const std::string* find(const std::string& name) const;
|
||||
const std::map<std::string, std::string>& all() const { return values_; }
|
||||
|
||||
private:
|
||||
std::map<std::string, std::string> values_;
|
||||
};
|
||||
|
||||
class Query {
|
||||
public:
|
||||
Query& add(const std::string& name, const std::string& value); // repeats allowed
|
||||
Query& add(const std::string& name, long value);
|
||||
Query& flag(const std::string& name, bool value);
|
||||
bool has(const std::string& name) const;
|
||||
// Replaces every value of `name` (page walks set the cursor).
|
||||
Query& replace(const std::string& name, const std::string& value);
|
||||
const std::vector<std::pair<std::string, std::string>>& items() const { return items_; }
|
||||
|
||||
private:
|
||||
std::vector<std::pair<std::string, std::string>> items_;
|
||||
};
|
||||
|
||||
struct Plan {
|
||||
Call call;
|
||||
clients::Method method;
|
||||
std::string target; // "/path?query", percent-encoded
|
||||
};
|
||||
|
||||
struct RequestPlan {
|
||||
// Throws ApiError::invalid (400 INVALID_ARGUMENT) naming the offending part.
|
||||
static Plan make(Call call, const PathArgs& args, const Query& query,
|
||||
const clients::body::Json* body = nullptr);
|
||||
|
||||
static std::string encode(const std::string& text); // RFC 3986 unreserved kept
|
||||
// True for a Hub-style name: 1..256 of [A-Za-z0-9._-] and not "." or "..".
|
||||
static bool valid_name(const std::string& text);
|
||||
// Validates a repo-relative file path and returns it percent-encoded per segment.
|
||||
static std::string encode_file_path(const std::string& path);
|
||||
// Validates "owner/name" or a bare "name" model id (router, hf-inference).
|
||||
static std::string check_model(const std::string& model);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+183
@@ -0,0 +1,183 @@
|
||||
#include "vendors/huggingface/servers.h"
|
||||
|
||||
#include "vendors/huggingface/caps.h"
|
||||
#include "llm/error.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
namespace {
|
||||
|
||||
[[noreturn]] void bad(const std::string& why) { throw llm::ApiError::invalid("huggingface", why); }
|
||||
|
||||
Json with_inputs(const Json& inputs, const Json& extra) {
|
||||
Json body = Json::object();
|
||||
body.set("inputs", inputs);
|
||||
if (extra.is_object())
|
||||
for (const auto& [k, v] : extra.as_object()) body.set(k, v);
|
||||
return body;
|
||||
}
|
||||
|
||||
bool health_of(HfService& s, const std::function<llm::RawResponse()>& get) {
|
||||
(void)s;
|
||||
try {
|
||||
get();
|
||||
return true;
|
||||
} catch (const llm::ApiError& e) {
|
||||
if (e.http_status() == 503) return false;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool TgiServer::ready() {
|
||||
return health_of(*this, [&] { return call_raw(Call::kTgiHealth); });
|
||||
}
|
||||
Json TgiServer::info() { return call(Call::kTgiGetModelInfo); }
|
||||
std::string TgiServer::metrics() { return call_raw(Call::kTgiMetrics).body; }
|
||||
Json TgiServer::models() { return call(Call::kTgiOpenaiGetModelInfo); }
|
||||
|
||||
Json TgiServer::generate(const std::string& inputs, const Json& parameters) {
|
||||
if (inputs.empty() || inputs.size() > HfCaps::kMaxTextChars) bad("inputs is empty or over the size cap");
|
||||
Json body = Json::object();
|
||||
body.set("inputs", Json(inputs));
|
||||
if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters);
|
||||
return call(Call::kTgiGenerate, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
void TgiServer::generate_stream(const std::string& inputs, const Json& parameters,
|
||||
const llm::SseDecoder::Sink& sink) {
|
||||
if (inputs.empty() || inputs.size() > HfCaps::kMaxTextChars) bad("inputs is empty or over the size cap");
|
||||
Json body = Json::object();
|
||||
body.set("inputs", Json(inputs));
|
||||
if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters);
|
||||
HfService::stream(Call::kTgiGenerateStream, PathArgs(), body, sink);
|
||||
}
|
||||
|
||||
Json TgiServer::tokenize(const std::string& inputs, const Json& parameters) {
|
||||
if (inputs.empty() || inputs.size() > HfCaps::kMaxTextChars) bad("inputs is empty or over the size cap");
|
||||
Json body = Json::object();
|
||||
body.set("inputs", Json(inputs));
|
||||
if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters);
|
||||
return call(Call::kTgiTokenize, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TgiServer::chat_tokenize(const Json& chat_body) {
|
||||
return call(Call::kTgiGetChatTokenize, PathArgs(), Query(), &chat_body);
|
||||
}
|
||||
|
||||
Json TgiServer::create(const Json& body) {
|
||||
if (const Json* s = body.is_object() ? body.find("stream") : nullptr; s && s->is_bool() && s->as_bool())
|
||||
bad("use stream() for streaming");
|
||||
return call(Call::kTgiChatCompletions, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
void TgiServer::stream(const Json& body, const llm::SseDecoder::Sink& sink) {
|
||||
HfService::stream(Call::kTgiChatCompletions, PathArgs(), body, sink);
|
||||
}
|
||||
|
||||
Json TgiServer::completions(const Json& body) { return call(Call::kTgiCompletions, PathArgs(), Query(), &body); }
|
||||
Json TgiServer::invocations(const Json& body) { return call(Call::kTgiSagemakerCompatibility, PathArgs(), Query(), &body); }
|
||||
|
||||
void TeiServer::check_inputs(const Json& inputs) {
|
||||
std::size_t chars = 0, count = 1;
|
||||
if (inputs.is_string()) {
|
||||
chars = inputs.as_string().size();
|
||||
} else if (inputs.is_array()) {
|
||||
count = inputs.as_array().size();
|
||||
if (count == 0 || count > HfCaps::kMaxBatchInputs) bad("inputs must hold 1..256 entries");
|
||||
for (const Json& x : inputs.as_array()) {
|
||||
if (x.is_string()) chars += x.as_string().size();
|
||||
else if (x.is_array()) {
|
||||
for (const Json& y : x.as_array())
|
||||
if (y.is_string()) chars += y.as_string().size();
|
||||
} else bad("inputs entries must be strings (or pairs of strings for predict)");
|
||||
}
|
||||
} else {
|
||||
bad("inputs must be a string or an array");
|
||||
}
|
||||
if (chars == 0 || chars > HfCaps::kMaxTextChars) bad("inputs are empty or over the size cap");
|
||||
}
|
||||
|
||||
bool TeiServer::ready() { return health_of(*this, [&] { return call_raw(Call::kTeiHealth); }); }
|
||||
Json TeiServer::info() { return call(Call::kTeiGetModelInfo); }
|
||||
std::string TeiServer::metrics() { return call_raw(Call::kTeiMetrics).body; }
|
||||
|
||||
Json TeiServer::embed(const Json& inputs, const Json& options) {
|
||||
check_inputs(inputs);
|
||||
Json body = with_inputs(inputs, options);
|
||||
return call(Call::kTeiEmbed, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::embed_all(const Json& inputs, const Json& options) {
|
||||
check_inputs(inputs);
|
||||
Json body = with_inputs(inputs, options);
|
||||
return call(Call::kTeiEmbedAll, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::embed_sparse(const Json& inputs, const Json& options) {
|
||||
check_inputs(inputs);
|
||||
Json body = with_inputs(inputs, options);
|
||||
return call(Call::kTeiEmbedSparse, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::predict(const Json& inputs, const Json& options) {
|
||||
check_inputs(inputs);
|
||||
Json body = with_inputs(inputs, options);
|
||||
return call(Call::kTeiPredict, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::rerank(const std::string& query, const std::vector<std::string>& texts, bool return_text,
|
||||
bool raw_scores) {
|
||||
if (query.empty() || query.size() > HfCaps::kMaxTextChars) bad("query is empty or over the size cap");
|
||||
if (texts.empty() || texts.size() > HfCaps::kMaxBatchInputs) bad("texts must hold 1..256 entries");
|
||||
Json arr = Json::array();
|
||||
std::size_t chars = query.size();
|
||||
for (const std::string& t : texts) {
|
||||
chars += t.size();
|
||||
arr.push(Json(t));
|
||||
}
|
||||
if (chars > HfCaps::kMaxTextChars) bad("texts are over the size cap");
|
||||
Json body = Json::object();
|
||||
body.set("query", Json(query));
|
||||
body.set("texts", arr);
|
||||
if (return_text) body.set("return_text", Json(true));
|
||||
if (raw_scores) body.set("raw_scores", Json(true));
|
||||
return call(Call::kTeiRerank, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::similarity(const Json& inputs, const Json& parameters) {
|
||||
if (!inputs.is_object()) bad("similarity inputs must be an object {source_sentence, sentences}");
|
||||
Json body = Json::object();
|
||||
body.set("inputs", inputs);
|
||||
if (parameters.is_object() && !parameters.as_object().empty()) body.set("parameters", parameters);
|
||||
return call(Call::kTeiSimilarity, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::tokenize(const Json& inputs, const Json& options) {
|
||||
check_inputs(inputs);
|
||||
Json body = with_inputs(inputs, options);
|
||||
return call(Call::kTeiTokenize, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::decode(const std::vector<long>& ids, bool skip_special_tokens) {
|
||||
if (ids.empty() || ids.size() > 100000) bad("ids must hold 1..100000 entries");
|
||||
Json arr = Json::array();
|
||||
for (long id : ids) arr.push(Json(id));
|
||||
Json body = Json::object();
|
||||
body.set("ids", arr);
|
||||
body.set("skip_special_tokens", Json(skip_special_tokens));
|
||||
return call(Call::kTeiDecode, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
Json TeiServer::openai_embeddings(const Json& input, const std::string& model, int dimensions) {
|
||||
check_inputs(input);
|
||||
if (dimensions < 0 || dimensions > 65536) bad("dimensions out of range");
|
||||
Json body = Json::object();
|
||||
body.set("input", input);
|
||||
if (!model.empty()) body.set("model", Json(model));
|
||||
if (dimensions > 0) body.set("dimensions", Json(dimensions));
|
||||
return call(Call::kTeiOpenaiEmbed, PathArgs(), Query(), &body);
|
||||
}
|
||||
|
||||
} // namespace molo::huggingface
|
||||
Vendored
+64
@@ -0,0 +1,64 @@
|
||||
#pragma once
|
||||
|
||||
// Text Generation Inference (TGI) and Text Embeddings Inference (TEI) server
|
||||
// APIs. Both run anywhere: an Inference Endpoint URL, a container on
|
||||
// localhost, a Space. The base URL is given by the caller (Platform::server
|
||||
// validates it); the token is optional because self-hosted servers usually
|
||||
// have none. Request shapes follow the servers' own OpenAPI documents.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "vendors/huggingface/chat_wire.h"
|
||||
#include "vendors/huggingface/hf_service.h"
|
||||
|
||||
namespace molo::huggingface {
|
||||
|
||||
class TgiServer : public HfService, public ChatEndpoint {
|
||||
public:
|
||||
explicit TgiServer(Access access) : HfService(std::move(access), Surface::kTgi) {}
|
||||
|
||||
bool ready(); // GET /health: true on 2xx, false on 503, throws otherwise
|
||||
Json info(); // GET /info
|
||||
std::string metrics(); // GET /metrics (Prometheus text)
|
||||
Json models(); // GET /v1/models
|
||||
|
||||
// Native API.
|
||||
Json generate(const std::string& inputs, const Json& parameters = Json());
|
||||
// Streams token events; sink returns false to stop.
|
||||
void generate_stream(const std::string& inputs, const Json& parameters, const llm::SseDecoder::Sink& sink);
|
||||
Json tokenize(const std::string& inputs, const Json& parameters = Json());
|
||||
Json chat_tokenize(const Json& chat_body);
|
||||
|
||||
// OpenAI-compatible API.
|
||||
Json create(const Json& body) override; // POST /v1/chat/completions
|
||||
void stream(const Json& body, const llm::SseDecoder::Sink& sink) override;
|
||||
Json completions(const Json& body); // POST /v1/completions
|
||||
Json invocations(const Json& body); // POST /invocations (SageMaker compatibility)
|
||||
};
|
||||
|
||||
class TeiServer : public HfService {
|
||||
public:
|
||||
explicit TeiServer(Access access) : HfService(std::move(access), Surface::kTei) {}
|
||||
|
||||
bool ready();
|
||||
Json info();
|
||||
std::string metrics();
|
||||
|
||||
// inputs: one string or an array of strings (1..HfCaps::kMaxBatchInputs).
|
||||
Json embed(const Json& inputs, const Json& options = Json());
|
||||
Json embed_all(const Json& inputs, const Json& options = Json());
|
||||
Json embed_sparse(const Json& inputs, const Json& options = Json());
|
||||
Json predict(const Json& inputs, const Json& options = Json());
|
||||
Json rerank(const std::string& query, const std::vector<std::string>& texts, bool return_text = false,
|
||||
bool raw_scores = false);
|
||||
Json similarity(const Json& inputs, const Json& parameters = Json());
|
||||
Json tokenize(const Json& inputs, const Json& options = Json());
|
||||
Json decode(const std::vector<long>& ids, bool skip_special_tokens = true);
|
||||
Json openai_embeddings(const Json& input, const std::string& model = "", int dimensions = 0);
|
||||
|
||||
// Counts and size of an `inputs` value: refuses anything over the caps.
|
||||
static void check_inputs(const Json& inputs);
|
||||
};
|
||||
|
||||
} // namespace molo::huggingface
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checks calls.cpp (the module's one call table) against Hugging Face's
|
||||
published documents. Python 3 stdlib only, no credentials.
|
||||
|
||||
python3 -I src/vendors/huggingface/tools/check_spec.py [--cache DIR] [--probe]
|
||||
|
||||
Documents (all fetched unless found in --cache):
|
||||
hub https://huggingface.co/.well-known/openapi.json (Hub OpenAPI 3.1)
|
||||
ie https://api.endpoints.huggingface.cloud/openapi.json (Inference Endpoints 2.0.0)
|
||||
tgi text-generation-inference docs/openapi.json (main)
|
||||
tei text-embeddings-inference docs/openapi.json (main)
|
||||
src huggingface_hub hf_api.py, hf_inference.py, constants.py (main; the Hub OpenAPI
|
||||
omits the repo list / info routes, so those rows cite the SDK source)
|
||||
task huggingface.js packages/tasks/src/tasks/<task>/spec/input.json
|
||||
|
||||
Per row it requires: the HTTP method and path exist (a {repoType} row is
|
||||
expanded to models, datasets, spaces; {typePrefix} to "", /datasets, /spaces),
|
||||
the operationId matches when the document has one, every query parameter and
|
||||
body field in the row is present in the document. Coverage: for ie, tgi and tei
|
||||
every document operation, query parameter and body field must be in the table;
|
||||
for hub every operation must be in the table or match an explicit EXCLUDED rule
|
||||
(counted per reason in the output); for hub the unlisted query parameters and
|
||||
body fields are reported (not failed: the Hub document is large and
|
||||
the table lists what the module sends). With --probe, rows whose spec is
|
||||
"src" or "none" are probed without a credential: a 404 fails.
|
||||
Exit 0: all rows verified. Exit 1: at least one mismatch (printed).
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
TABLE = os.path.join(HERE, "..", "calls.cpp")
|
||||
RAW = "https://raw.githubusercontent.com/huggingface/"
|
||||
URLS = {
|
||||
"hub.json": "https://huggingface.co/.well-known/openapi.json",
|
||||
"ie.json": "https://api.endpoints.huggingface.cloud/openapi.json",
|
||||
"tgi.json": RAW + "text-generation-inference/main/docs/openapi.json",
|
||||
"tei.json": RAW + "text-embeddings-inference/main/docs/openapi.json",
|
||||
"hf_api.py": RAW + "huggingface_hub/main/src/huggingface_hub/hf_api.py",
|
||||
"hf_inference.py": RAW + "huggingface_hub/main/src/huggingface_hub/inference/_providers/hf_inference.py",
|
||||
"constants.py": RAW + "huggingface_hub/main/src/huggingface_hub/constants.py",
|
||||
}
|
||||
TASK_URL = RAW + "huggingface.js/main/packages/tasks/src/tasks/%s/spec/input.json"
|
||||
HOSTS = {
|
||||
"Hub": "https://huggingface.co",
|
||||
"Router": "https://router.huggingface.co",
|
||||
"Endpoints": "https://api.endpoints.huggingface.cloud",
|
||||
}
|
||||
S = r'"((?:[^"\\]|\\.)*)"'
|
||||
ROW = re.compile(r"^\s*\{Call::(\w+), Surface::k(\w+), " + ", ".join([S] * 8) + r"\},?\s*$")
|
||||
METHODS = ("get", "post", "put", "patch", "delete")
|
||||
REPO_TYPES = ["models", "datasets", "spaces"]
|
||||
PREFIXES = ["", "/datasets", "/spaces"]
|
||||
|
||||
# Hub operations deliberately not wrapped: (reason, regex over "METHOD /path").
|
||||
EXCLUDED = [
|
||||
("repository writes (commit, upload, LFS, branches, tags, squash, duplicate, settings, move, create)",
|
||||
r"^(POST|PUT|DELETE) /api/(models|datasets|spaces)/\{namespace\}/\{repo\}/(commit|preupload|lfs-files|tag|branch|super-squash|settings|resource-group|volumes|hardware|sleeptime|dev-mode|custom-domain|secrets|variables)"),
|
||||
("repository writes (commit, upload, LFS, branches, tags, squash, duplicate, settings, move, create)",
|
||||
r"^(POST|PUT|DELETE|GET) /api/(repos|\{repoType\})/.*(create|move|duplicate)"),
|
||||
("repository admin reads (LFS file list, xet tokens, jwt, scan, notebook, leaderboard, resource group)",
|
||||
r"^GET /api/(models|datasets|spaces)/\{namespace\}/\{repo\}/(lfs-files|xet-write-token|xet-read-token|jwt|scan|notebook|leaderboard|resource-group|logs|events|metrics)"),
|
||||
("gated repo access requests", r"(ask-access|user-access|access-request)"),
|
||||
("resolve-cache and viewer helpers", r"^GET /api/resolve-cache/"),
|
||||
("secrets and variables of a Space (credentials)", r"^(GET|POST|DELETE) /api/spaces/\{namespace\}/\{repo\}/(secrets|variables)"),
|
||||
("rarely used discussion moderation (pin, merge, ignore, storage, ref delete, comment edit, hide, reaction)",
|
||||
r"^(GET|POST|DELETE) /api/\{repoType\}/\{namespace\}/\{repo\}/discussions/\{num\}/(pin|merge|ignore|storage|ref|comment/)"),
|
||||
("collection item update and delete (need the slug-id split, see SPEC Gaps) and resource groups",
|
||||
r"^(PATCH|DELETE|POST|GET) /api/collections/.*(items/|items/batch|resource-group|\{slug\}-\{id\})"),
|
||||
("comments on blogs, papers, posts", r"^(POST|DELETE) /api/(blog|papers|posts)/"),
|
||||
("paper curation (index, links, media, claim)", r"^POST /api/(papers/index|papers/\{paperId\}/(links|media)|settings/papers/claim)"),
|
||||
("account settings, billing and notifications", r"^(GET|POST|PUT|PATCH|DELETE) /api/(settings/(?!webhooks)|notifications|credentials)"),
|
||||
("organizations, users, avatars, SCIM, service accounts, resource groups",
|
||||
r"^[A-Z]+ /api/(orgs|organizations|users|avatars|scim|service-accounts|resource-groups)"),
|
||||
("Hub Jobs and scheduled jobs (separate product)", r"^[A-Z]+ /api/(jobs|scheduled-jobs)"),
|
||||
("buckets, containers, kernels (separate repo kinds)", r"^[A-Z]+ /(api/)?(buckets|containers|kernels)"),
|
||||
("Spaces runtime control", r"^[A-Z]+ /api/spaces/(zero-gpu|\{namespace\}/\{repo\}/(runtime|restart|pause))"),
|
||||
("partner provider mapping writes and inference-provider account settings",
|
||||
r"^(POST|PUT) /api/(partners|settings/inference-providers)"),
|
||||
("Inference Endpoints auth-check helper (internal)", r"/api/inference-endpoints/"),
|
||||
("OAuth, docs search, agents, sql console, registry token, repo search POST variant",
|
||||
r"^[A-Z]+ /(oauth|api/(docs|agent-harnesses|registry|sql-console|\{repoType\}/\{namespace\}/\{repo\}/sql-console|quicksearch$))"),
|
||||
("blog, MCP settings, likes", r"^[A-Z]+ /api/(blog|settings|trending-.*)"),
|
||||
]
|
||||
|
||||
|
||||
def load(cache, name):
|
||||
if cache:
|
||||
p = os.path.join(cache, name)
|
||||
if os.path.exists(p):
|
||||
return open(p, "rb").read()
|
||||
url = URLS.get(name) or name
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
data = r.read()
|
||||
if cache:
|
||||
os.makedirs(os.path.dirname(os.path.join(cache, name)), exist_ok=True)
|
||||
open(os.path.join(cache, name), "wb").write(data)
|
||||
return data
|
||||
|
||||
|
||||
def resolve(doc, node):
|
||||
seen = 0
|
||||
while isinstance(node, dict) and "$ref" in node and seen < 20:
|
||||
cur = doc
|
||||
for p in node["$ref"].lstrip("#/").split("/"):
|
||||
cur = cur[p]
|
||||
node = cur
|
||||
seen += 1
|
||||
return node
|
||||
|
||||
|
||||
def props(doc, schema):
|
||||
schema = resolve(doc, schema)
|
||||
if not isinstance(schema, dict):
|
||||
return {}
|
||||
out = dict(schema.get("properties", {}))
|
||||
for key in ("allOf", "anyOf", "oneOf"):
|
||||
for sub in schema.get(key, []):
|
||||
out.update(props(doc, sub))
|
||||
return out
|
||||
|
||||
|
||||
def op_fields(doc, op):
|
||||
q = set()
|
||||
for p in op.get("parameters", []):
|
||||
p = resolve(doc, p)
|
||||
if p.get("in") == "query":
|
||||
q.add(p["name"])
|
||||
body = set()
|
||||
rb = resolve(doc, op.get("requestBody", {}))
|
||||
if rb:
|
||||
content = rb.get("content", {})
|
||||
ct = next(iter(content), None)
|
||||
if ct:
|
||||
body = set(props(doc, content[ct].get("schema")).keys())
|
||||
return q, body
|
||||
|
||||
|
||||
def unesc(s):
|
||||
return s.replace('\\"', '"').replace("\\\\", "\\")
|
||||
|
||||
|
||||
def variants(path, doc_paths=()):
|
||||
# The document may carry {repoType} literally (discussions) or one path per
|
||||
# repo type (everything else); a literal match wins.
|
||||
if path in doc_paths:
|
||||
return [path]
|
||||
if "{repoType}" in path:
|
||||
return [path.replace("{repoType}", t) for t in REPO_TYPES]
|
||||
if "{typePrefix}" in path:
|
||||
return [path.replace("{typePrefix}", p) for p in PREFIXES]
|
||||
return [path]
|
||||
|
||||
|
||||
def csv(s):
|
||||
return [x for x in s.split(",") if x]
|
||||
|
||||
|
||||
def probe_url(surface, path):
|
||||
subs = {"{namespace}": "openai-community", "{repo}": "gpt2", "{rev}": "main", "{path}": "config.json",
|
||||
"{model}": "openai-community/gpt2", "{task}": "feature-extraction", "{repoType}": "models"}
|
||||
for k, v in subs.items():
|
||||
path = path.replace(k, v)
|
||||
return HOSTS[surface] + path
|
||||
|
||||
|
||||
def main():
|
||||
cache = None
|
||||
if "--cache" in sys.argv:
|
||||
cache = sys.argv[sys.argv.index("--cache") + 1]
|
||||
text = open(TABLE).read()
|
||||
begin, end = text.index("// CALLS-BEGIN"), text.index("// CALLS-END")
|
||||
rows = []
|
||||
for line in text[begin:end].splitlines():
|
||||
m = ROW.match(line)
|
||||
if m:
|
||||
g = m.groups()
|
||||
rows.append(dict(name=g[0], surface=g[1], method=unesc(g[2]), path=unesc(g[3]), spec=unesc(g[4]),
|
||||
ref=unesc(g[5]), auth=unesc(g[6]), flags=unesc(g[7]), body=csv(unesc(g[8])),
|
||||
query=csv(unesc(g[9]))))
|
||||
if not rows:
|
||||
print("no rows parsed from calls.cpp")
|
||||
return 1
|
||||
docs = {}
|
||||
shas = {}
|
||||
for key in ("hub", "ie", "tgi", "tei"):
|
||||
raw = load(cache, key + ".json")
|
||||
shas[key] = hashlib.sha256(raw).hexdigest()[:12]
|
||||
docs[key] = json.loads(raw)
|
||||
sources = ""
|
||||
for name in ("hf_api.py", "hf_inference.py", "constants.py"):
|
||||
raw = load(cache, name)
|
||||
shas[name] = hashlib.sha256(raw).hexdigest()[:12]
|
||||
sources += raw.decode("utf-8", "replace")
|
||||
ops = {}
|
||||
for key, doc in docs.items():
|
||||
ops[key] = {}
|
||||
for path, item in doc["paths"].items():
|
||||
for method, op in item.items():
|
||||
if method in METHODS:
|
||||
ops[key][(method.upper(), path)] = op
|
||||
bad = []
|
||||
names = set()
|
||||
covered = {k: set() for k in docs}
|
||||
notes = {"hub_unlisted_query": 0, "hub_unlisted_body": 0}
|
||||
for r in rows:
|
||||
if r["name"] in names:
|
||||
bad.append("duplicate row %s" % r["name"])
|
||||
names.add(r["name"])
|
||||
spec = r["spec"]
|
||||
if spec in docs:
|
||||
union_q, union_b, found = set(), set(), False
|
||||
for v in variants(r["path"], docs[spec]["paths"]):
|
||||
op = ops[spec].get((r["method"], v))
|
||||
if op is None:
|
||||
bad.append("%s: %s %s not in the %s document" % (r["name"], r["method"], v, spec))
|
||||
continue
|
||||
found = True
|
||||
covered[spec].add((r["method"], v))
|
||||
if spec != "hub" and r["ref"] != "-" and op.get("operationId") != r["ref"]:
|
||||
bad.append("%s: operationId %s, table says %s" % (r["name"], op.get("operationId"), r["ref"]))
|
||||
q, b = op_fields(docs[spec], op)
|
||||
union_q |= q
|
||||
union_b |= b
|
||||
if not found:
|
||||
continue
|
||||
for f in r["query"]:
|
||||
if f not in union_q:
|
||||
bad.append("%s: query parameter %s absent from the document" % (r["name"], f))
|
||||
for f in r["body"]:
|
||||
if f not in union_b:
|
||||
bad.append("%s: body field %s absent from the document" % (r["name"], f))
|
||||
if spec == "hub":
|
||||
notes["hub_unlisted_query"] += len(union_q - set(r["query"]))
|
||||
notes["hub_unlisted_body"] += len(union_b - set(r["body"]))
|
||||
else:
|
||||
for f in union_q - set(r["query"]):
|
||||
bad.append("%s: document query parameter %s not in the table" % (r["name"], f))
|
||||
for f in union_b - set(r["body"]):
|
||||
bad.append("%s: document body field %s not in the table" % (r["name"], f))
|
||||
elif spec == "src":
|
||||
if r["ref"] not in sources:
|
||||
bad.append("%s: source needle %r not found in huggingface_hub" % (r["name"], r["ref"]))
|
||||
elif spec == "task":
|
||||
try:
|
||||
schema = json.loads(load(cache, "tasks/%s.json" % r["ref"]) if cache and os.path.exists(os.path.join(cache, "tasks/%s.json" % r["ref"])) else load(None, TASK_URL % r["ref"]))
|
||||
except Exception as e: # noqa
|
||||
bad.append("%s: cannot load the %s task schema (%s)" % (r["name"], r["ref"], e))
|
||||
continue
|
||||
allowed = set(schema.get("properties", {}).keys())
|
||||
for f in r["body"]:
|
||||
if f not in allowed:
|
||||
bad.append("%s: body field %s absent from the %s task schema" % (r["name"], f, r["ref"]))
|
||||
for f in allowed - set(r["body"]):
|
||||
bad.append("%s: task schema field %s not in the table" % (r["name"], f))
|
||||
elif spec != "none":
|
||||
bad.append("%s: unknown spec kind %s" % (r["name"], spec))
|
||||
# Coverage.
|
||||
for key in ("ie", "tgi", "tei"):
|
||||
for k in ops[key]:
|
||||
if k not in covered[key]:
|
||||
bad.append("unwrapped %s operation %s %s" % (key, k[0], k[1]))
|
||||
excluded_count = {}
|
||||
unwrapped_hub = 0
|
||||
for (method, path), op in sorted(ops["hub"].items()):
|
||||
if (method, path) in covered["hub"]:
|
||||
continue
|
||||
label = "%s %s" % (method, path)
|
||||
reason = None
|
||||
for why, rx in EXCLUDED:
|
||||
if re.search(rx, label):
|
||||
reason = why
|
||||
break
|
||||
if reason is None:
|
||||
bad.append("unwrapped hub operation %s (tags %s)" % (label, op.get("tags")))
|
||||
else:
|
||||
excluded_count[reason] = excluded_count.get(reason, 0) + 1
|
||||
unwrapped_hub += 1
|
||||
if "--probe" in sys.argv:
|
||||
for r in rows:
|
||||
if r["spec"] not in ("src", "none") or r["surface"] not in HOSTS or r["method"] != "GET":
|
||||
continue
|
||||
if "{" in r["path"].replace("{namespace}", "").replace("{repo}", "").replace("{rev}", "").replace("{path}", "").replace("{model}", ""):
|
||||
continue
|
||||
url = probe_url(r["surface"], r["path"])
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "molo-check-spec/1"})
|
||||
try:
|
||||
status = urllib.request.urlopen(req, timeout=60).status
|
||||
except urllib.error.HTTPError as e:
|
||||
status = e.code
|
||||
print("probe GET %s -> %d" % (url, status))
|
||||
if status == 404:
|
||||
bad.append("%s: probe answered 404" % r["name"])
|
||||
for b in bad:
|
||||
print("FAIL " + b)
|
||||
print("sources: " + ", ".join("%s=%s" % kv for kv in sorted(shas.items())))
|
||||
print("hub operations excluded (%d):" % unwrapped_hub)
|
||||
for why, n in sorted(excluded_count.items()):
|
||||
print(" %3d %s" % (n, why))
|
||||
print("hub columns the module does not send: %d query parameters, %d body fields (information)"
|
||||
% (notes["hub_unlisted_query"], notes["hub_unlisted_body"]))
|
||||
counts = {k: len(v) for k, v in ops.items()}
|
||||
print("%d rows checked; spec operations hub=%d ie=%d tgi=%d tei=%d; %d problems"
|
||||
% (len(rows), counts["hub"], counts["ie"], counts["tgi"], counts["tei"], len(bad)))
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
// Routes: GET /ai/huggingface (page), /status, /calls, /tasks (no credential needed)
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support/Assert.hpp"
|
||||
|
||||
using namespace molo::example::test;
|
||||
|
||||
int main() {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("HF_TOKEN");
|
||||
unsetenv("HUGGING_FACE_HUB_TOKEN");
|
||||
Fixture f;
|
||||
auto page = f.client->get("/ai/huggingface");
|
||||
check(page.status == 200, "page 200");
|
||||
check(contains(page.headers.get("Content-Type"), "text/html"), "html");
|
||||
check(contains(page.body, "Hugging Face") && contains(page.body, "viewport"), "title and mobile viewport");
|
||||
check(contains(page.body, "/embedded/ai_huggingface.js"), "script wired");
|
||||
check(f.client->post("/ai/huggingface", "{}", "application/json").status == 405, "post page is 405");
|
||||
|
||||
auto st = f.client->get("/ai/huggingface/status");
|
||||
check(st.status == 200 && contains(st.body, "\"configured\":false"), "status without a token");
|
||||
check(contains(st.body, "HF_TOKEN") && contains(st.body, "router.huggingface.co") &&
|
||||
contains(st.body, "api.endpoints.huggingface.cloud"),
|
||||
"status names credential env and hosts");
|
||||
check(contains(st.body, "\"calls\":132") && contains(st.body, "\"tasks\":25"), "status counts calls and tasks");
|
||||
check(f.client->post("/ai/huggingface/status", "{}", "application/json").status == 405, "post status is 405");
|
||||
|
||||
auto calls = f.client->get("/ai/huggingface/calls");
|
||||
check(calls.status == 200 && contains(calls.body, "/v1/chat/completions") &&
|
||||
contains(calls.body, "/v2/endpoint/{namespace}/{name}/pause") && contains(calls.body, "/embed") &&
|
||||
contains(calls.body, "/api/whoami-v2"),
|
||||
"call table listed across surfaces");
|
||||
check(f.client->post("/ai/huggingface/calls", "{}", "application/json").status == 405, "post calls is 405");
|
||||
|
||||
auto tasks = f.client->get("/ai/huggingface/tasks");
|
||||
check(tasks.status == 200 && contains(tasks.body, "feature-extraction") &&
|
||||
contains(tasks.body, "\"binary_input\":true"),
|
||||
"task catalog listed");
|
||||
return finish("ai/huggingface/page");
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Routes: POST /ai/huggingface/* (405, 422, dry runs, 503 without a credential)
|
||||
#include <cstdlib>
|
||||
|
||||
#include "support/Assert.hpp"
|
||||
|
||||
using namespace molo::example::test;
|
||||
|
||||
int main() {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("HF_TOKEN");
|
||||
unsetenv("HUGGING_FACE_HUB_TOKEN");
|
||||
Fixture f;
|
||||
auto post = [&](const std::string& route, const std::string& body) {
|
||||
return f.client->post("/ai/huggingface/" + route, body, "application/json");
|
||||
};
|
||||
const char* all[] = {"hub/list", "hub/repo", "hub/search", "hub/whoami", "hub/discussions", "hub/collections",
|
||||
"hub/webhooks", "chat", "chat/stream", "task", "endpoints", "tgi", "tei"};
|
||||
for (const char* r : all) {
|
||||
std::string route = std::string("/ai/huggingface/") + r;
|
||||
check(f.client->get(route).status == 405, route + " get is 405");
|
||||
check(post(r, "not json").status == 422, route + " bad json is 422");
|
||||
check(post(r, "[1]").status == 422, route + " non-object is 422");
|
||||
}
|
||||
|
||||
// 422: input rules run before anything else
|
||||
check(post("hub/list", "{\"kind\":\"kernels\"}").status == 422, "unknown kind");
|
||||
check(post("hub/list", "{\"limit\":101}").status == 422, "limit over 100");
|
||||
check(post("hub/list", "{\"pipeline_tag\":\"x\",\"kind\":\"datasets\"}").status == 422, "pipeline_tag is models only");
|
||||
check(post("hub/repo", "{}").status == 422, "repo needs action");
|
||||
check(post("hub/repo", "{\"action\":\"info\",\"id\":\"gpt2\"}").status == 422, "legacy id without owner");
|
||||
check(post("hub/repo", "{\"action\":\"info\",\"id\":\"../x/y\"}").status == 422, "path traversal id");
|
||||
check(post("hub/repo", "{\"action\":\"fly\",\"id\":\"a/b\"}").status == 422, "unknown repo action");
|
||||
check(post("hub/repo", "{\"action\":\"file\",\"id\":\"a/b\",\"path\":\"../etc\"}").status == 422, "file path traversal");
|
||||
check(post("hub/search", "{\"action\":\"quicksearch\"}").status == 422, "quicksearch needs q");
|
||||
check(post("hub/search", "{\"action\":\"nope\"}").status == 422, "unknown search action");
|
||||
check(post("hub/discussions", "{\"action\":\"get\",\"id\":\"a/b\"}").status == 422, "discussion get needs num");
|
||||
check(post("hub/discussions", "{\"action\":\"create\",\"id\":\"a/b\"}").status == 422, "discussion create needs title");
|
||||
check(post("hub/collections", "{\"action\":\"add_item\",\"owner\":\"me\",\"slug\":\"x-1\",\"item_type\":\"video\",\"item_id\":\"x\"}").status == 422,
|
||||
"bad collection item type");
|
||||
check(post("hub/webhooks", "{\"action\":\"create\",\"url\":\"https://example.com/hook\"}").status == 422, "webhook needs watched");
|
||||
check(post("chat", "{\"model\":\"a/b\"}").status == 422, "chat needs prompt");
|
||||
check(post("chat", "{\"prompt\":\"hi\"}").status == 422, "chat needs model");
|
||||
check(post("chat", "{\"model\":\"a b\",\"prompt\":\"hi\"}").status == 422, "bad model id");
|
||||
check(post("chat", "{\"model\":\"a/b\",\"prompt\":\"hi\",\"max_tokens\":0}").status == 422, "max_tokens 0");
|
||||
check(post("task", "{\"task\":\"nope\",\"model\":\"a/b\",\"inputs\":\"x\"}").status == 422, "unknown task");
|
||||
check(post("task", "{\"task\":\"translation\",\"model\":\"a/b\"}").status == 422, "task needs inputs");
|
||||
check(post("task", "{\"task\":\"image-classification\",\"model\":\"a/b\",\"data_base64\":\"@@\",\"mime\":\"image/png\"}").status == 422,
|
||||
"bad base64");
|
||||
check(post("endpoints", "{\"action\":\"get\",\"namespace\":\"acme\"}").status == 422, "endpoint get needs name");
|
||||
check(post("endpoints", "{\"action\":\"get\",\"namespace\":\"acme\",\"name\":\"Bad_Name\"}").status == 422, "bad endpoint name");
|
||||
check(post("endpoints", "{\"action\":\"create\",\"namespace\":\"acme\",\"spec\":{\"name\":\"x\"}}").status == 422, "incomplete endpoint spec");
|
||||
check(post("tgi", "{\"action\":\"health\",\"url\":\"ftp://x\"}").status == 422, "tgi url must be http(s)");
|
||||
check(post("tei", "{\"action\":\"embed\",\"url\":\"http://127.0.0.1:1\"}").status == 422, "tei embed needs inputs");
|
||||
|
||||
// dry runs: paid or resource-changing routes show the exact request
|
||||
auto dry = post("chat", "{\"model\":\"openai/gpt-oss-20b:fastest\",\"prompt\":\"hi\"}");
|
||||
check(dry.status == 200 && contains(dry.body, "\"dry_run\":true") && contains(dry.body, "/v1/chat/completions") &&
|
||||
contains(dry.body, "openai/gpt-oss-20b:fastest"),
|
||||
"chat is a dry run without confirm");
|
||||
check(contains(post("chat/stream", "{\"model\":\"a/b\",\"prompt\":\"hi\"}").body, "\"stream\":true"), "stream preview");
|
||||
check(contains(post("task", "{\"task\":\"translation\",\"model\":\"t5/x\",\"inputs\":\"hi\"}").body, "\"dry_run\":true"), "task is a dry run");
|
||||
check(contains(post("hub/discussions", "{\"action\":\"create\",\"id\":\"a/b\",\"title\":\"Typo\"}").body, "\"dry_run\":true"),
|
||||
"discussion create is a dry run");
|
||||
check(contains(post("hub/discussions", "{\"action\":\"remove\",\"id\":\"a/b\",\"num\":3}").body, "DELETE"), "discussion remove is a dry run");
|
||||
check(contains(post("hub/collections", "{\"action\":\"create\",\"owner\":\"me\",\"title\":\"T\"}").body, "\"dry_run\":true"),
|
||||
"collection create is a dry run");
|
||||
check(contains(post("hub/webhooks", "{\"action\":\"remove\",\"id\":\"w1\"}").body, "\"dry_run\":true"), "webhook remove is a dry run");
|
||||
check(contains(post("endpoints", "{\"action\":\"pause\",\"namespace\":\"acme\",\"name\":\"ep\"}").body, "\"dry_run\":true"),
|
||||
"endpoint pause is a dry run");
|
||||
check(contains(post("endpoints", "{\"action\":\"remove\",\"namespace\":\"acme\",\"name\":\"ep\"}").body, "DELETE"),
|
||||
"endpoint remove is a dry run");
|
||||
|
||||
// 503: reads and confirmed calls need the credential, before any network traffic
|
||||
for (const char* r : {"hub/whoami"}) {
|
||||
auto res = post(r, "{}");
|
||||
check(res.status == 503 && contains(res.body, "huggingface_not_configured"), std::string(r) + " without a token is 503");
|
||||
}
|
||||
check(post("hub/list", "{}").status == 503, "list without a token is 503");
|
||||
check(post("hub/repo", "{\"action\":\"info\",\"id\":\"openai-community/gpt2\"}").status == 503, "repo info without a token is 503");
|
||||
check(post("hub/search", "{\"action\":\"tasks\"}").status == 503, "search without a token is 503");
|
||||
check(post("endpoints", "{\"action\":\"vendors\"}").status == 503, "endpoints without a token is 503");
|
||||
check(post("tgi", "{\"action\":\"health\",\"url\":\"http://127.0.0.1:1\"}").status == 503, "tgi without a token is 503");
|
||||
check(post("chat", "{\"model\":\"a/b\",\"prompt\":\"hi\",\"confirm\":true}").status == 503, "confirmed chat without a token is 503");
|
||||
check(post("endpoints", "{\"action\":\"remove\",\"namespace\":\"acme\",\"name\":\"ep\",\"confirm\":true}").status == 503,
|
||||
"confirmed endpoint remove without a token is 503");
|
||||
return finish("ai/huggingface/routes");
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
// Shared rig for the Hugging Face tests: a Platform over three MockTransports
|
||||
// (hub, router, endpoints). The mock only records the requests the real client
|
||||
// builds and answers with canned bodies. Canned bodies are either RECORDED
|
||||
// (copied from an unauthenticated probe on 2026-10-25, trimmed) or SHAPE
|
||||
// (written by us to the documented schema, not recorded). Nothing here
|
||||
// verifies vendor behavior (see src/vendors/huggingface/SPEC.md ## Unverified).
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "clients/transports/mock_transport.h"
|
||||
#include "framework.h"
|
||||
#include "vendors/huggingface/platform.h"
|
||||
|
||||
namespace hf_test {
|
||||
|
||||
using molo::clients::ClientResponse;
|
||||
using molo::clients::HeaderMap;
|
||||
using molo::clients::Method;
|
||||
using molo::clients::transports::MockExpectation;
|
||||
using molo::clients::transports::MockTransport;
|
||||
using Json = molo::clients::body::Json;
|
||||
|
||||
inline ClientResponse reply(const std::string& body, int status = 200,
|
||||
const std::string& type = "application/json") {
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", type);
|
||||
return ClientResponse(status, h, body);
|
||||
}
|
||||
|
||||
inline ClientResponse sse(const std::string& body) { return reply(body, 200, "text/event-stream"); }
|
||||
|
||||
inline const char* kToken = "hf_TESTTOKEN_DO_NOT_LEAK";
|
||||
|
||||
struct Rig {
|
||||
MockTransport* hub = nullptr;
|
||||
MockTransport* router = nullptr;
|
||||
MockTransport* endpoints = nullptr;
|
||||
molo::huggingface::Platform platform;
|
||||
|
||||
explicit Rig(const std::string& token = kToken,
|
||||
std::shared_ptr<molo::huggingface::RequestBudget> budget = nullptr) {
|
||||
auto a = std::make_unique<MockTransport>();
|
||||
auto b = std::make_unique<MockTransport>();
|
||||
auto c = std::make_unique<MockTransport>();
|
||||
hub = a.get();
|
||||
router = b.get();
|
||||
endpoints = c.get();
|
||||
platform = molo::huggingface::Platform::with(std::move(a), std::move(b), std::move(c), token, budget);
|
||||
}
|
||||
|
||||
static std::string header(const MockTransport::SeenRequest& r, const std::string& name) {
|
||||
return r.headers.get_or(name, "");
|
||||
}
|
||||
};
|
||||
|
||||
inline bool has(const std::string& hay, const std::string& needle) {
|
||||
return hay.find(needle) != std::string::npos;
|
||||
}
|
||||
|
||||
} // namespace hf_test
|
||||
+212
@@ -0,0 +1,212 @@
|
||||
// The call table and request planning: structure of the table, id and path
|
||||
// validation, query and body field checks, caps. Pure local logic.
|
||||
#include <set>
|
||||
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/calls.h"
|
||||
#include "vendors/huggingface/request_plan.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using namespace hf_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
TEST(huggingface_calls, table_is_in_enum_order_and_unique) {
|
||||
std::set<std::string> seen;
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
const CallSpec& s = Calls::at(i);
|
||||
ASSERT_EQ(static_cast<std::size_t>(s.call), i);
|
||||
ASSERT_TRUE(&Calls::spec_of(s.call) == &s);
|
||||
std::string key = std::string(surface_name(s.surface)) + " " + s.http_method + " " + s.path;
|
||||
ASSERT_TRUE(seen.insert(key).second);
|
||||
}
|
||||
ASSERT_EQ(Calls::count(), static_cast<std::size_t>(132));
|
||||
}
|
||||
|
||||
TEST(huggingface_calls, rows_use_known_vocabulary) {
|
||||
const std::set<std::string> specs = {"none", "hub", "ie", "tgi", "tei", "src", "task"};
|
||||
const std::set<std::string> auths = {"none", "optional", "token"};
|
||||
const std::set<std::string> methods = {"GET", "POST", "PUT", "PATCH", "DELETE"};
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
const CallSpec& s = Calls::at(i);
|
||||
ASSERT_TRUE(specs.count(s.spec) == 1);
|
||||
ASSERT_TRUE(auths.count(s.auth) == 1);
|
||||
ASSERT_TRUE(methods.count(s.http_method) == 1);
|
||||
ASSERT_TRUE(s.path[0] == '/' || s.path[0] == '{');
|
||||
ASSERT_TRUE(std::string(s.ref).size() > 0);
|
||||
// braces balance
|
||||
int depth = 0;
|
||||
for (const char* p = s.path; *p; ++p) {
|
||||
if (*p == '{') ++depth;
|
||||
if (*p == '}') --depth;
|
||||
ASSERT_TRUE(depth == 0 || depth == 1);
|
||||
}
|
||||
ASSERT_EQ(depth, 0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(huggingface_calls, writes_need_a_token_and_deletes_are_marked_destructive) {
|
||||
std::size_t deletes = 0;
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
const CallSpec& s = Calls::at(i);
|
||||
if (s.has_flag("write")) ASSERT_EQ(std::string(s.auth), std::string("token"));
|
||||
if (std::string(s.http_method) == "DELETE") {
|
||||
++deletes;
|
||||
ASSERT_TRUE(s.has_flag("destructive"));
|
||||
}
|
||||
if (s.has_flag("destructive")) ASSERT_TRUE(s.has_flag("write"));
|
||||
}
|
||||
ASSERT_EQ(deletes, static_cast<std::size_t>(7)); // 3 Hub (discussion, collection, webhook) + 4 Endpoints
|
||||
}
|
||||
|
||||
TEST(huggingface_calls, per_surface_counts_match_the_published_documents) {
|
||||
std::size_t counts[5] = {0, 0, 0, 0, 0};
|
||||
std::size_t unsupported = 0;
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
++counts[static_cast<int>(Calls::at(i).surface)];
|
||||
if (Calls::at(i).has_flag("unsupported")) ++unsupported;
|
||||
}
|
||||
ASSERT_EQ(counts[static_cast<int>(Surface::kEndpoints)], static_cast<std::size_t>(48)); // IE openapi 2.0.0
|
||||
ASSERT_EQ(counts[static_cast<int>(Surface::kTgi)], static_cast<std::size_t>(12)); // TGI 3.3.6-dev0
|
||||
ASSERT_EQ(counts[static_cast<int>(Surface::kTei)], static_cast<std::size_t>(12)); // TEI 1.9.4
|
||||
ASSERT_EQ(unsupported, static_cast<std::size_t>(3)); // the three GET SSE streams
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, repo_ref_needs_owner_and_name) {
|
||||
RepoRef r = RepoRef::parse("openai-community/gpt2");
|
||||
ASSERT_EQ(r.owner(), std::string("openai-community"));
|
||||
ASSERT_EQ(r.name(), std::string("gpt2"));
|
||||
ASSERT_THROW(RepoRef::parse("gpt2"), ApiError);
|
||||
ASSERT_THROW(RepoRef::parse("a/b/c"), ApiError);
|
||||
ASSERT_THROW(RepoRef::parse("../etc"), ApiError);
|
||||
ASSERT_THROW(RepoRef::parse("a/.."), ApiError);
|
||||
ASSERT_THROW(RepoRef::parse("a/b?x=1"), ApiError);
|
||||
ASSERT_THROW(RepoRef::parse("a/b c"), ApiError);
|
||||
ASSERT_THROW(RepoRef::parse(std::string("a/") + std::string(257, 'x')), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, renders_typed_repo_paths) {
|
||||
PathArgs a;
|
||||
a.repo(RepoRef::parse("squad/x")).repo_type(RepoType::kDataset).set("rev", "main").set("path", "");
|
||||
Plan p = RequestPlan::make(Call::kRepoTree, a, Query());
|
||||
ASSERT_EQ(p.target, std::string("/api/datasets/squad/x/tree/main"));
|
||||
ASSERT_TRUE(p.method == Method::kGet);
|
||||
a.set("path", "data/train 1.parquet");
|
||||
p = RequestPlan::make(Call::kRepoTree, a, Query());
|
||||
ASSERT_EQ(p.target, std::string("/api/datasets/squad/x/tree/main/data/train%201.parquet"));
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, revision_is_one_encoded_segment) {
|
||||
PathArgs a;
|
||||
a.repo(RepoRef::parse("o/n")).repo_type(RepoType::kModel).set("rev", "refs/pr/1");
|
||||
Plan p = RequestPlan::make(Call::kRepoCommits, a, Query());
|
||||
ASSERT_EQ(p.target, std::string("/api/models/o/n/commits/refs%2Fpr%2F1"));
|
||||
a.set("rev", "..");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kRepoCommits, a, Query()), ApiError);
|
||||
a.set("rev", "");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kRepoCommits, a, Query()), ApiError);
|
||||
a.set("rev", "a\nb");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kRepoCommits, a, Query()), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, resolve_uses_the_type_prefix) {
|
||||
PathArgs a;
|
||||
a.repo(RepoRef::parse("o/n")).set("rev", "main").set("path", "config.json");
|
||||
a.repo_type(RepoType::kModel);
|
||||
ASSERT_EQ(RequestPlan::make(Call::kFileResolve, a, Query()).target, std::string("/o/n/resolve/main/config.json"));
|
||||
a.repo_type(RepoType::kDataset);
|
||||
ASSERT_EQ(RequestPlan::make(Call::kFileResolve, a, Query()).target,
|
||||
std::string("/datasets/o/n/resolve/main/config.json"));
|
||||
a.repo_type(RepoType::kSpace);
|
||||
ASSERT_EQ(RequestPlan::make(Call::kFileResolve, a, Query()).target, std::string("/spaces/o/n/resolve/main/config.json"));
|
||||
a.set("path", ""); // a file path is required for resolve
|
||||
ASSERT_THROW(RequestPlan::make(Call::kFileResolve, a, Query()), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, file_path_rules) {
|
||||
ASSERT_EQ(RequestPlan::encode_file_path("a/b.txt"), std::string("a/b.txt"));
|
||||
ASSERT_THROW(RequestPlan::encode_file_path("/abs"), ApiError);
|
||||
ASSERT_THROW(RequestPlan::encode_file_path("a//b"), ApiError);
|
||||
ASSERT_THROW(RequestPlan::encode_file_path("a/../b"), ApiError);
|
||||
ASSERT_THROW(RequestPlan::encode_file_path("a/./b"), ApiError);
|
||||
ASSERT_THROW(RequestPlan::encode_file_path(std::string(1025, 'x')), ApiError);
|
||||
std::string deep;
|
||||
for (int i = 0; i < 33; ++i) deep += "d/";
|
||||
ASSERT_THROW(RequestPlan::encode_file_path(deep + "f"), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, query_and_body_are_checked_against_the_table) {
|
||||
PathArgs a;
|
||||
a.repo(RepoRef::parse("o/n")).repo_type(RepoType::kModel);
|
||||
Query ok;
|
||||
ok.flag("include_prs", true);
|
||||
ASSERT_EQ(RequestPlan::make(Call::kRepoRefs, a, ok).target, std::string("/api/models/o/n/refs?include_prs=true"));
|
||||
Query unknown;
|
||||
unknown.add("evil", "1");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kRepoRefs, a, unknown), ApiError);
|
||||
Json body = Json::object();
|
||||
ASSERT_THROW(RequestPlan::make(Call::kRepoRefs, a, Query(), &body), ApiError); // GET takes no body
|
||||
PathArgs b;
|
||||
b.repo(RepoRef::parse("o/n")).repo_type(RepoType::kModel).set("rev", "main");
|
||||
Json paths = Json::object();
|
||||
paths.set("paths", Json::array());
|
||||
ASSERT_TRUE(!RequestPlan::make(Call::kRepoPathsInfo, b, Query(), &paths).target.empty());
|
||||
paths.set("sneaky", Json(1));
|
||||
ASSERT_THROW(RequestPlan::make(Call::kRepoPathsInfo, b, Query(), &paths), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, query_values_are_encoded_and_dots_survive_the_client_guard) {
|
||||
Query q;
|
||||
q.add("search", "a b&c=d");
|
||||
Plan p = RequestPlan::make(Call::kModelsList, PathArgs(), q);
|
||||
ASSERT_EQ(p.target, std::string("/api/models?search=a%20b%26c%3Dd"));
|
||||
Query dots;
|
||||
dots.add("search", "x..y");
|
||||
p = RequestPlan::make(Call::kModelsList, PathArgs(), dots);
|
||||
ASSERT_EQ(p.target, std::string("/api/models?search=x%2E%2Ey"));
|
||||
ASSERT_TRUE(p.target.find("..") == std::string::npos);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, missing_and_unused_arguments_are_refused) {
|
||||
PathArgs none;
|
||||
ASSERT_THROW(RequestPlan::make(Call::kModelInfo, none, Query()), ApiError);
|
||||
PathArgs extra;
|
||||
extra.repo(RepoRef::parse("o/n")).set("surplus", "1");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kModelInfo, extra, Query()), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, caps_on_queries) {
|
||||
Query q;
|
||||
for (std::size_t i = 0; i < 32; ++i) q.add("filter", "x");
|
||||
ASSERT_THROW(q.add("filter", "x"), ApiError);
|
||||
Query big;
|
||||
ASSERT_THROW(big.add("search", std::string(2049, 'x')), ApiError);
|
||||
ASSERT_THROW(big.add("search", std::string("a\r\nb")), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, unsupported_calls_are_refused_with_a_reason) {
|
||||
PathArgs a;
|
||||
a.set("namespace", "acme").set("name", "ep");
|
||||
try {
|
||||
RequestPlan::make(Call::kIeGetEndpointSse, a, Query());
|
||||
FAIL("expected a refusal");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 400);
|
||||
ASSERT_TRUE(molo::huggingface::Calls::spec_of(Call::kIeGetEndpointSse).has_flag("unsupported"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(huggingface_paths, special_placeholders) {
|
||||
PathArgs a;
|
||||
a.set("webhookId", "abc123").set("action", "enable");
|
||||
ASSERT_EQ(RequestPlan::make(Call::kWebhookToggle, a, Query()).target, std::string("/api/settings/webhooks/abc123/enable"));
|
||||
a.set("action", "explode");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kWebhookToggle, a, Query()), ApiError);
|
||||
PathArgs d;
|
||||
d.repo(RepoRef::parse("o/n")).repo_type(RepoType::kSpace).set("num", "12");
|
||||
ASSERT_EQ(RequestPlan::make(Call::kDiscussionGet, d, Query()).target, std::string("/api/spaces/o/n/discussions/12"));
|
||||
d.set("num", "12x");
|
||||
ASSERT_THROW(RequestPlan::make(Call::kDiscussionGet, d, Query()), ApiError);
|
||||
ASSERT_EQ(RequestPlan::check_model("gpt2"), std::string("gpt2"));
|
||||
ASSERT_THROW(RequestPlan::check_model("a/b/c"), ApiError);
|
||||
ASSERT_THROW(RequestPlan::check_model(""), ApiError);
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
// Discussions, collections, webhooks: validation and the exact requests built.
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/community.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using namespace hf_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// RECORDED 2026-10-25: GET /api/collections/nvidia/flora-6ad6a8986320a55467825cf5 (first fields)
|
||||
const char* kCollection =
|
||||
"{\"slug\":\"nvidia/flora-6ad6a8986320a55467825cf5\",\"title\":\"Flora\",\"gating\":false,"
|
||||
"\"owner\":{\"name\":\"nvidia\",\"type\":\"org\"}}";
|
||||
} // namespace
|
||||
|
||||
TEST(huggingface_community, discussion_reads_are_public_and_writes_need_a_token) {
|
||||
Rig anon("");
|
||||
Discussions d(anon.platform.hub);
|
||||
anon.hub->expect_response(Method::kGet, "/api/models/o/n/discussions", reply("{\"discussions\":[]}"));
|
||||
DiscussionFilter f;
|
||||
f.status = "open";
|
||||
f.page = 2;
|
||||
d.list(RepoType::kModel, RepoRef::parse("o/n"), f);
|
||||
ASSERT_EQ(anon.hub->seen().back().target, std::string("/api/models/o/n/discussions?p=2&status=open"));
|
||||
try {
|
||||
d.create(RepoType::kModel, RepoRef::parse("o/n"), "A title", "body", false);
|
||||
FAIL("expected NOT_CONFIGURED");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
|
||||
}
|
||||
ASSERT_EQ(anon.hub->send_count(), static_cast<std::size_t>(1));
|
||||
}
|
||||
|
||||
TEST(huggingface_community, discussion_writes_build_the_documented_bodies) {
|
||||
Rig rig;
|
||||
Discussions d(rig.platform.hub);
|
||||
RepoRef r = RepoRef::parse("o/n");
|
||||
rig.hub->expect_response(Method::kPost, "/api/datasets/o/n/discussions", reply("{\"num\":7}"));
|
||||
d.create(RepoType::kDataset, r, "Typo in card", "details", true);
|
||||
ASSERT_EQ(rig.hub->seen().back().body, std::string("{\"description\":\"details\",\"pullRequest\":true,\"title\":\"Typo in card\"}"));
|
||||
rig.hub->expect_response(Method::kPost, "/api/datasets/o/n/discussions/7/status", reply("{}"));
|
||||
d.change_status(RepoType::kDataset, r, 7, "closed", "done");
|
||||
ASSERT_EQ(rig.hub->seen().back().body, std::string("{\"comment\":\"done\",\"status\":\"closed\"}"));
|
||||
rig.hub->expect_response(Method::kDelete, "/api/datasets/o/n/discussions/7", reply("{}"));
|
||||
d.remove_discussion(RepoType::kDataset, r, 7);
|
||||
ASSERT_THROW(d.create(RepoType::kModel, r, "ab", "x", false), ApiError); // title 3..200
|
||||
ASSERT_THROW(d.change_status(RepoType::kModel, r, 1, "maybe"), ApiError);
|
||||
ASSERT_THROW(d.comment(RepoType::kModel, r, 0, "x"), ApiError);
|
||||
ASSERT_THROW(d.comment(RepoType::kModel, r, 1, ""), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_community, collections_read_and_write) {
|
||||
Rig rig;
|
||||
Collections c(rig.platform.hub);
|
||||
rig.hub->expect_response(Method::kGet, "/api/collections/nvidia/flora-6ad6a8986320a55467825cf5", reply(kCollection));
|
||||
Json got = c.get("nvidia", "flora-6ad6a8986320a55467825cf5");
|
||||
ASSERT_EQ(got.find("title")->as_string(), std::string("Flora"));
|
||||
rig.hub->expect_response(Method::kPost, "/api/collections", reply("{\"slug\":\"me/x-1\"}"));
|
||||
c.create("me", "My list", "d", true);
|
||||
ASSERT_EQ(rig.hub->seen().back().body,
|
||||
std::string("{\"description\":\"d\",\"namespace\":\"me\",\"private\":true,\"title\":\"My list\"}"));
|
||||
ASSERT_THROW(c.create("me", "", "", false), ApiError);
|
||||
ASSERT_THROW(c.create("me", std::string(61, 'x'), "", false), ApiError);
|
||||
ASSERT_THROW(c.create("me", "t", std::string(151, 'x'), false), ApiError);
|
||||
CollectionItem bad_item;
|
||||
bad_item.type = "video";
|
||||
bad_item.id = "x";
|
||||
ASSERT_THROW(c.add_item("me", "x-1", bad_item), ApiError);
|
||||
CollectionItem item;
|
||||
item.type = "model";
|
||||
item.id = "openai-community/gpt2";
|
||||
rig.hub->expect_response(Method::kPost, "/api/collections/me/x-1/items", reply("{}"));
|
||||
c.add_item("me", "x-1", item, "note");
|
||||
ASSERT_EQ(rig.hub->seen().back().body,
|
||||
std::string("{\"item\":{\"id\":\"openai-community/gpt2\",\"type\":\"model\"},\"note\":\"note\"}"));
|
||||
ASSERT_THROW(c.update("me", "x-1", Json::object()), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_community, collection_list_walks_with_item_filter) {
|
||||
Rig rig("");
|
||||
Collections c(rig.platform.hub);
|
||||
rig.hub->expect_response(Method::kGet, "/api/collections", reply("[{\"slug\":\"a/b-1\"}]"));
|
||||
CollectionListParams p;
|
||||
p.owner = "nvidia";
|
||||
p.limit = 5;
|
||||
p.item.type = "model";
|
||||
p.item.id = "openai-community/gpt2";
|
||||
Page page = c.list(p, 10, 2);
|
||||
ASSERT_EQ(page.items.as_array().size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(rig.hub->seen().back().target, std::string("/api/collections?owner=nvidia&limit=5&item=model%2Fopenai-community%2Fgpt2"));
|
||||
}
|
||||
|
||||
TEST(huggingface_community, webhook_spec_is_validated_before_anything_is_sent) {
|
||||
Rig rig;
|
||||
Webhooks w(rig.platform.hub);
|
||||
WebhookSpec s;
|
||||
s.url = "https://example.com/hook";
|
||||
s.watched.push_back({"model", "openai-community/gpt2"});
|
||||
s.domains = {"repo"};
|
||||
rig.hub->expect_response(Method::kPost, "/api/settings/webhooks", reply("{\"webhook\":{\"id\":\"w1\"}}"));
|
||||
w.create(s);
|
||||
ASSERT_EQ(rig.hub->seen().back().body,
|
||||
std::string("{\"domains\":[\"repo\"],\"url\":\"https://example.com/hook\",\"watched\":[{\"name\":\"openai-community/gpt2\",\"type\":\"model\"}]}"));
|
||||
WebhookSpec bad = s;
|
||||
bad.url = "ftp://x/y/z/w";
|
||||
ASSERT_THROW(w.create(bad), ApiError);
|
||||
bad = s;
|
||||
bad.watched.clear();
|
||||
ASSERT_THROW(w.create(bad), ApiError);
|
||||
bad = s;
|
||||
bad.watched[0].type = "planet";
|
||||
ASSERT_THROW(w.create(bad), ApiError);
|
||||
bad = s;
|
||||
bad.domains = {"everything"};
|
||||
ASSERT_THROW(w.create(bad), ApiError);
|
||||
bad = s;
|
||||
bad.secret = "caf\xc3\xa9";
|
||||
ASSERT_THROW(w.create(bad), ApiError);
|
||||
ASSERT_EQ(rig.hub->send_count(), static_cast<std::size_t>(1));
|
||||
}
|
||||
|
||||
TEST(huggingface_community, webhook_lifecycle_paths) {
|
||||
Rig rig;
|
||||
Webhooks w(rig.platform.hub);
|
||||
rig.hub->expect_response(Method::kPost, "/api/settings/webhooks/w1/enable", reply("{}"));
|
||||
rig.hub->expect_response(Method::kPost, "/api/settings/webhooks/w1/disable", reply("{}"));
|
||||
rig.hub->expect_response(Method::kPost, "/api/settings/webhooks/w1/replay/l9", reply("{}"));
|
||||
rig.hub->expect_response(Method::kDelete, "/api/settings/webhooks/w1", reply("{}"));
|
||||
w.enable("w1");
|
||||
w.disable("w1");
|
||||
w.replay("w1", "l9");
|
||||
w.remove_webhook("w1");
|
||||
rig.hub->verify();
|
||||
ASSERT_THROW(w.get("../x"), ApiError);
|
||||
}
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
// Hub reads and the shared service rules (token, budget, paging) through the
|
||||
// real ApiClient over a mock transport. Replies are RECORDED trims or SHAPE.
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/hub.h"
|
||||
#include "vendors/huggingface/platform.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using namespace hf_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// RECORDED 2026-10-25: GET https://huggingface.co/api/models/openai-community/gpt2/refs (no token)
|
||||
const char* kRefs =
|
||||
"{\"tags\":[],\"branches\":[{\"name\":\"main\",\"ref\":\"refs/heads/main\","
|
||||
"\"targetCommit\":\"607a30d783dfa663caf39e06633721c8d4cfcd7e\"}],\"converts\":[]}";
|
||||
// RECORDED 2026-10-25: GET /api/models/openai-community/gpt2/tree/main (first 3 entries)
|
||||
const char* kTree =
|
||||
"[{\"type\":\"directory\",\"oid\":\"d03ec5ec179df58241d27d55f92a674f5f44197f\",\"size\":0,\"path\":\"onnx\"},"
|
||||
"{\"type\":\"file\",\"oid\":\"602b71f15d40ed68c5f96330e3f3175a76a32126\",\"size\":445,\"path\":\".gitattributes\"}]";
|
||||
// RECORDED 2026-10-25: GET /api/whoami-v2 without a token -> 401
|
||||
const char* kUnauth = "{\"error\":\"Invalid username or password.\"}";
|
||||
// RECORDED 2026-10-25: GET /api/models?limit=1 (trimmed fields)
|
||||
const char* kModel =
|
||||
"{\"_id\":\"621ffdc036468d709f17434d\",\"id\":\"openai-community/gpt2\",\"likes\":3891,\"private\":false,"
|
||||
"\"downloads\":20867408,\"pipeline_tag\":\"text-generation\",\"library_name\":\"transformers\"}";
|
||||
|
||||
ClientResponse paged(const std::string& body, const std::string& next_cursor) {
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", "application/json");
|
||||
if (!next_cursor.empty())
|
||||
h.set("Link", "<https://huggingface.co/api/models?limit=1&cursor=" + next_cursor + ">; rel=\"next\"");
|
||||
return ClientResponse(200, h, body);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(huggingface_hub, anonymous_reads_send_no_authorization) {
|
||||
Rig rig("");
|
||||
rig.hub->expect_response(Method::kGet, "/api/models/openai-community/gpt2/refs", reply(kRefs));
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
ASSERT_FALSE(hub.has_token());
|
||||
Json refs = hub.refs(RepoType::kModel, RepoRef::parse("openai-community/gpt2"));
|
||||
ASSERT_EQ(refs.find("branches")->as_array().front().find("name")->as_string(), std::string("main"));
|
||||
auto seen = rig.hub->seen();
|
||||
ASSERT_EQ(seen.size(), static_cast<std::size_t>(1));
|
||||
ASSERT_EQ(Rig::header(seen[0], "Authorization"), std::string(""));
|
||||
ASSERT_EQ(Rig::header(seen[0], "X-Molo-Anonymous"), std::string("1"));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, token_travels_as_bearer_only) {
|
||||
Rig rig;
|
||||
rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs));
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
hub.refs(RepoType::kModel, RepoRef::parse("o/n"), true);
|
||||
auto seen = rig.hub->seen();
|
||||
ASSERT_EQ(Rig::header(seen[0], "Authorization"), std::string("Bearer ") + kToken);
|
||||
ASSERT_TRUE(seen[0].target.find(kToken) == std::string::npos);
|
||||
ASSERT_EQ(seen[0].target, std::string("/api/models/o/n/refs?include_prs=true"));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, whoami_without_token_fails_before_any_traffic) {
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
try {
|
||||
hub.whoami();
|
||||
FAIL("expected NOT_CONFIGURED");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 503);
|
||||
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
|
||||
ASSERT_TRUE(has(e.message(), "HF_TOKEN"));
|
||||
}
|
||||
ASSERT_EQ(rig.hub->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, hub_error_envelope_is_kept) {
|
||||
Rig rig;
|
||||
rig.hub->expect_response(Method::kGet, "/api/whoami-v2", reply(kUnauth, 401));
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
try {
|
||||
hub.whoami();
|
||||
FAIL("expected 401");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 401);
|
||||
ASSERT_EQ(e.message(), std::string("Invalid username or password."));
|
||||
ASSERT_TRUE(!has(e.message(), kToken));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, list_walks_link_cursors_and_reports_page_truncation) {
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
// Three pages exist; the walk is capped at two pages.
|
||||
rig.hub->expect_response(Method::kGet, "/api/models", paged(std::string("[") + kModel + "]", "AAA"));
|
||||
rig.hub->expect_response(Method::kGet, "/api/models", paged(std::string("[") + kModel + "]", "BBB"));
|
||||
RepoListParams p;
|
||||
p.limit = 1;
|
||||
p.sort = "downloads";
|
||||
Page page = hub.list_repos(RepoType::kModel, p, 100, 2);
|
||||
ASSERT_EQ(page.items.as_array().size(), static_cast<std::size_t>(2));
|
||||
ASSERT_TRUE(page.truncated);
|
||||
ASSERT_EQ(page.truncated_by, std::string("pages"));
|
||||
ASSERT_EQ(page.next_cursor, std::string("BBB"));
|
||||
auto seen = rig.hub->seen();
|
||||
ASSERT_EQ(seen[0].target, std::string("/api/models?sort=downloads&limit=1"));
|
||||
ASSERT_EQ(seen[1].target, std::string("/api/models?sort=downloads&limit=1&cursor=AAA"));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, list_reports_item_truncation_and_clean_end) {
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
rig.hub->expect_response(Method::kGet, "/api/datasets",
|
||||
paged(std::string("[") + kModel + "," + kModel + "," + kModel + "]", ""));
|
||||
Page cut = hub.list_repos(RepoType::kDataset, RepoListParams(), 2, 5);
|
||||
ASSERT_EQ(cut.items.as_array().size(), static_cast<std::size_t>(2));
|
||||
ASSERT_TRUE(cut.truncated);
|
||||
ASSERT_EQ(cut.truncated_by, std::string("items"));
|
||||
rig.hub->expect_response(Method::kGet, "/api/spaces", paged(std::string("[") + kModel + "]", ""));
|
||||
Page done = hub.list_repos(RepoType::kSpace, RepoListParams(), 10, 5);
|
||||
ASSERT_FALSE(done.truncated);
|
||||
ASSERT_EQ(done.pages, static_cast<std::size_t>(1));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, link_header_to_another_host_is_refused) {
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", "application/json");
|
||||
h.set("Link", "<https://evil.example/api/models?cursor=x>; rel=\"next\"");
|
||||
rig.hub->expect_response(Method::kGet, "/api/models", ClientResponse(200, h, "[]"));
|
||||
ASSERT_THROW(hub.list_repos(RepoType::kModel, RepoListParams()), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, link_header_parsing) {
|
||||
ASSERT_EQ(next_cursor_of("<https://huggingface.co/api/models?limit=2&cursor=abc%3D>; rel=\"next\"", "huggingface.co"),
|
||||
std::string("abc="));
|
||||
ASSERT_EQ(next_cursor_of("<https://huggingface.co/api/models?cursor=p>; rel=\"prev\"", "huggingface.co"), std::string(""));
|
||||
ASSERT_EQ(next_cursor_of("", "huggingface.co"), std::string(""));
|
||||
ASSERT_EQ(next_cursor_of("<https://huggingface.co/x?cursor=a>; rel=\"prev\", <https://huggingface.co/x?cursor=b>; rel=\"next\"",
|
||||
"huggingface.co"),
|
||||
std::string("b"));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, list_params_are_validated) {
|
||||
RepoListParams p;
|
||||
p.limit = 1001;
|
||||
ASSERT_THROW(p.to_query(), ApiError);
|
||||
p.limit = -1;
|
||||
ASSERT_THROW(p.to_query(), ApiError);
|
||||
RepoListParams q;
|
||||
q.pipeline_tag = "text-generation";
|
||||
q.expand = {"downloads", "likes"};
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
// pipeline_tag is a models-only filter: the datasets row does not list it.
|
||||
ASSERT_THROW(hub.list_repos(RepoType::kDataset, q), ApiError);
|
||||
ASSERT_EQ(rig.hub->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, tree_paths_info_and_download) {
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
rig.hub->expect_response(Method::kGet, "/api/models/o/n/tree/main", reply(kTree));
|
||||
Json t = hub.tree(RepoType::kModel, RepoRef::parse("o/n"), "main");
|
||||
ASSERT_EQ(t.as_array().size(), static_cast<std::size_t>(2));
|
||||
|
||||
rig.hub->expect_response(Method::kPost, "/api/models/o/n/paths-info/main",
|
||||
reply("[{\"type\":\"file\",\"oid\":\"10c6\",\"size\":665,\"path\":\"config.json\"}]"));
|
||||
hub.paths_info(RepoType::kModel, RepoRef::parse("o/n"), "main", {"config.json"});
|
||||
auto seen = rig.hub->seen();
|
||||
ASSERT_EQ(seen.back().body, std::string("{\"paths\":[\"config.json\"]}"));
|
||||
ASSERT_THROW(hub.paths_info(RepoType::kModel, RepoRef::parse("o/n"), "main", {}), ApiError);
|
||||
ASSERT_THROW(hub.paths_info(RepoType::kModel, RepoRef::parse("o/n"), "main", {"../x"}), ApiError);
|
||||
|
||||
HeaderMap h;
|
||||
h.set("Content-Type", "text/plain; charset=utf-8");
|
||||
h.set("X-Repo-Commit", "607a30d7");
|
||||
h.set("ETag", "\"abc\"");
|
||||
rig.hub->expect_response(Method::kGet, "/o/n/resolve/main/config.json", ClientResponse(200, h, "{\"a\":1}"));
|
||||
Download d = hub.download(RepoType::kModel, RepoRef::parse("o/n"), "main", "config.json");
|
||||
ASSERT_EQ(d.bytes, std::string("{\"a\":1}"));
|
||||
ASSERT_EQ(d.commit, std::string("607a30d7"));
|
||||
ASSERT_EQ(d.etag, std::string("\"abc\""));
|
||||
}
|
||||
|
||||
TEST(huggingface_hub, search_and_catalogs_validate_input) {
|
||||
Rig rig("");
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
ASSERT_THROW(hub.quicksearch(""), ApiError);
|
||||
ASSERT_THROW(hub.quicksearch("bert", 5000), ApiError);
|
||||
ASSERT_THROW(hub.tags_by_type(RepoType::kSpace), ApiError);
|
||||
ASSERT_THROW(hub.paper("../x"), ApiError);
|
||||
ASSERT_THROW(hub.partner_models("a/b"), ApiError);
|
||||
rig.hub->expect_response(Method::kGet, "/api/quicksearch", reply("{\"models\":[],\"datasets\":[],\"spaces\":[]}"));
|
||||
hub.quicksearch("bert", 2, "model");
|
||||
ASSERT_EQ(rig.hub->seen().back().target, std::string("/api/quicksearch?q=bert&limit=2&type=model"));
|
||||
}
|
||||
|
||||
TEST(huggingface_budget, spends_down_and_resets_with_the_window) {
|
||||
long now = 1000;
|
||||
auto budget = std::make_shared<RequestBudget>(2, 300, [&] { return now; });
|
||||
Rig rig("", budget);
|
||||
Hub hub = Hub::from_platform(rig.platform);
|
||||
rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs));
|
||||
rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs));
|
||||
rig.hub->expect_response(Method::kGet, "/api/models/o/n/refs", reply(kRefs));
|
||||
RepoRef r = RepoRef::parse("o/n");
|
||||
hub.refs(RepoType::kModel, r);
|
||||
hub.refs(RepoType::kModel, r);
|
||||
try {
|
||||
hub.refs(RepoType::kModel, r);
|
||||
FAIL("expected the local budget to stop the call");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 429);
|
||||
ASSERT_EQ(e.type(), std::string("LOCAL_BUDGET"));
|
||||
}
|
||||
ASSERT_EQ(rig.hub->send_count(), static_cast<std::size_t>(2));
|
||||
now += 301;
|
||||
hub.refs(RepoType::kModel, r);
|
||||
ASSERT_EQ(rig.hub->send_count(), static_cast<std::size_t>(3));
|
||||
}
|
||||
|
||||
TEST(huggingface_platform, server_urls_are_validated) {
|
||||
ASSERT_EQ(Platform::check_server_url("http://127.0.0.1:8080/"), std::string("http://127.0.0.1:8080"));
|
||||
ASSERT_EQ(Platform::check_server_url("https://x.endpoints.huggingface.cloud"), std::string("https://x.endpoints.huggingface.cloud"));
|
||||
ASSERT_THROW(Platform::check_server_url(""), ApiError);
|
||||
ASSERT_THROW(Platform::check_server_url("ftp://x"), ApiError);
|
||||
ASSERT_THROW(Platform::check_server_url("http://user:pw@host"), ApiError);
|
||||
ASSERT_THROW(Platform::check_server_url("http://host/?a=b"), ApiError);
|
||||
ASSERT_THROW(Platform::check_server_url("not a url"), ApiError);
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
// Opt-in live checks. MOLO_LIVE_NET=1 enables the network. Public Hub reads,
|
||||
// the router model list and the Inference Endpoints vendor list need NO key
|
||||
// and run under MOLO_LIVE_NET=1 alone. Token-gated reads also need HF_TOKEN
|
||||
// (via llm::Config); the one tiny chat completion additionally needs
|
||||
// MOLO_LIVE_PAID=1. Without them each test prints a loud SKIP. Nothing is
|
||||
// created, changed or deleted.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "clients/transports/http_transport.h"
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/chat_model.h"
|
||||
#include "vendors/huggingface/community.h"
|
||||
#include "vendors/huggingface/endpoints.h"
|
||||
#include "vendors/huggingface/hub.h"
|
||||
#include "vendors/huggingface/providers.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
bool net() { return std::getenv("MOLO_LIVE_NET") != nullptr; }
|
||||
bool paid() { return std::getenv("MOLO_LIVE_PAID") != nullptr; }
|
||||
Platform anonymous_platform() {
|
||||
return Platform::with(std::make_unique<molo::clients::transports::HttpTransport>(),
|
||||
std::make_unique<molo::clients::transports::HttpTransport>(),
|
||||
std::make_unique<molo::clients::transports::HttpTransport>(), "");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(huggingface_live, public_hub_reads_without_a_key) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP huggingface_live.public_hub_reads_without_a_key (set MOLO_LIVE_NET=1; no key needed)\n";
|
||||
return;
|
||||
}
|
||||
Platform p = anonymous_platform();
|
||||
Hub hub = Hub::from_platform(p);
|
||||
ASSERT_FALSE(hub.has_token());
|
||||
RepoRef gpt2 = RepoRef::parse("openai-community/gpt2");
|
||||
|
||||
RepoListParams params;
|
||||
params.limit = 2;
|
||||
params.sort = "downloads";
|
||||
Page models = hub.list_repos(RepoType::kModel, params, 4, 2);
|
||||
std::cout << " models listed: " << models.items.as_array().size() << " pages=" << models.pages
|
||||
<< " truncated=" << models.truncated << "\n";
|
||||
ASSERT_TRUE(models.items.as_array().size() >= 2);
|
||||
|
||||
Json info = hub.repo_info(RepoType::kModel, gpt2);
|
||||
ASSERT_EQ(info.find("id")->as_string(), std::string("openai-community/gpt2"));
|
||||
Json refs = hub.refs(RepoType::kModel, gpt2);
|
||||
ASSERT_TRUE(refs.find("branches") != nullptr);
|
||||
Json tree = hub.tree(RepoType::kModel, gpt2, "main");
|
||||
ASSERT_TRUE(tree.is_array() && !tree.as_array().empty());
|
||||
Download cfg = hub.download(RepoType::kModel, gpt2, "main", "config.json");
|
||||
std::cout << " config.json: " << cfg.bytes.size() << " bytes commit=" << cfg.commit << "\n";
|
||||
ASSERT_TRUE(cfg.bytes.find("GPT2LMHeadModel") != std::string::npos);
|
||||
Json ds = hub.repo_info(RepoType::kDataset, RepoRef::parse("rajpurkar/squad"));
|
||||
ASSERT_EQ(ds.find("id")->as_string(), std::string("rajpurkar/squad"));
|
||||
Json sp = hub.repo_info(RepoType::kSpace, RepoRef::parse("gradio/hello_world"));
|
||||
ASSERT_EQ(sp.find("id")->as_string(), std::string("gradio/hello_world"));
|
||||
Json search = hub.quicksearch("bert", 2);
|
||||
ASSERT_TRUE(search.is_object());
|
||||
Json hw = hub.spaces_hardware();
|
||||
ASSERT_TRUE(hw.is_array());
|
||||
Json trending = hub.trending("model", 3);
|
||||
ASSERT_TRUE(trending.is_object());
|
||||
Json mapping = hub.partner_models("groq");
|
||||
ASSERT_TRUE(mapping.is_object());
|
||||
|
||||
Discussions d(p.hub);
|
||||
Json disc = d.list(RepoType::kModel, gpt2);
|
||||
ASSERT_TRUE(disc.is_object());
|
||||
Collections c(p.hub);
|
||||
Json coll = c.get("nvidia", "flora-6ad6a8986320a55467825cf5");
|
||||
ASSERT_EQ(coll.find("title")->as_string(), std::string("Flora"));
|
||||
}
|
||||
|
||||
TEST(huggingface_live, router_models_and_endpoint_vendors_without_a_key) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP huggingface_live.router_models_and_endpoint_vendors_without_a_key (set MOLO_LIVE_NET=1; no key needed)\n";
|
||||
return;
|
||||
}
|
||||
Platform p = anonymous_platform();
|
||||
Json models = RouterChat::from_platform(p).models();
|
||||
std::cout << " router models: " << models.find("data")->as_array().size() << "\n";
|
||||
ASSERT_TRUE(models.find("data")->as_array().size() > 10);
|
||||
Json vendors = Endpoints::from_platform(p).vendors();
|
||||
ASSERT_TRUE(vendors.find("vendors")->as_array().size() >= 1);
|
||||
}
|
||||
|
||||
TEST(huggingface_live, invalid_token_answers_401_on_whoami_and_endpoints) {
|
||||
if (!net()) {
|
||||
std::cout << "SKIP huggingface_live.invalid_token_answers_401_on_whoami_and_endpoints (set MOLO_LIVE_NET=1)\n";
|
||||
return;
|
||||
}
|
||||
Platform p = Platform::with(std::make_unique<molo::clients::transports::HttpTransport>(),
|
||||
std::make_unique<molo::clients::transports::HttpTransport>(),
|
||||
std::make_unique<molo::clients::transports::HttpTransport>(), "hf_invalid_molo_probe");
|
||||
try {
|
||||
Hub::from_platform(p).whoami();
|
||||
FAIL("expected 401");
|
||||
} catch (const ApiError& e) {
|
||||
std::cout << " GET /api/whoami-v2 with an invalid token: " << e.http_status() << " " << e.message() << "\n";
|
||||
ASSERT_EQ(e.http_status(), 401);
|
||||
}
|
||||
try {
|
||||
Endpoints::from_platform(p).list("huggingface");
|
||||
FAIL("expected 401");
|
||||
} catch (const ApiError& e) {
|
||||
std::cout << " GET /v2/endpoint/huggingface with an invalid token: " << e.http_status() << " " << e.message() << "\n";
|
||||
ASSERT_EQ(e.http_status(), 401);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(huggingface_live, token_reads_and_one_tiny_completion) {
|
||||
Platform p = Platform::from_env();
|
||||
if (!net() || !p.configured()) {
|
||||
std::cout << "SKIP huggingface_live.token_reads_and_one_tiny_completion (set MOLO_LIVE_NET=1 and HF_TOKEN)\n";
|
||||
return;
|
||||
}
|
||||
Json who = Hub::from_platform(p).whoami();
|
||||
ASSERT_TRUE(who.find("name") != nullptr);
|
||||
std::cout << " whoami ok\n";
|
||||
if (!paid()) {
|
||||
std::cout << "SKIP huggingface_live.token_reads_and_one_tiny_completion chat part (also set MOLO_LIVE_PAID=1)\n";
|
||||
return;
|
||||
}
|
||||
auto model = HuggingfaceChatModel::over_router(p, "openai/gpt-oss-20b");
|
||||
molo::llm::ChatRequest req;
|
||||
req.messages.push_back(molo::llm::Message::user("Reply with the single word: ok"));
|
||||
req.max_tokens = 16;
|
||||
molo::llm::ChatResponse r = model->generate(req);
|
||||
std::cout << " completion finish=" << r.finish_reason << " tokens=" << r.usage.total_tokens << "\n";
|
||||
ASSERT_TRUE(r.usage.total_tokens > 0);
|
||||
}
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
// Inference Providers router, hf-inference tasks, the chat wire and
|
||||
// HuggingfaceChatModel. Replies are SHAPE (documented schema) or RECORDED.
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/chat_model.h"
|
||||
#include "vendors/huggingface/providers.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using namespace hf_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// SHAPE: chat completion per huggingface.js tasks/chat-completion/spec/output.json
|
||||
const char* kChat =
|
||||
"{\"id\":\"cmpl-1\",\"model\":\"openai/gpt-oss-20b\",\"created\":1,\"object\":\"chat.completion\","
|
||||
"\"choices\":[{\"index\":0,\"finish_reason\":\"stop\",\"message\":{\"role\":\"assistant\",\"content\":\"Hello!\","
|
||||
"\"reasoning\":\"greeting\"}}],\"usage\":{\"prompt_tokens\":9,\"completion_tokens\":3,\"total_tokens\":12,"
|
||||
"\"completion_tokens_details\":{\"reasoning_tokens\":2}}}";
|
||||
const char* kToolReply =
|
||||
"{\"id\":\"cmpl-2\",\"model\":\"m\",\"choices\":[{\"index\":0,\"finish_reason\":\"tool_calls\",\"message\":{\"role\":\"assistant\","
|
||||
"\"content\":null,\"tool_calls\":[{\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\","
|
||||
"\"arguments\":\"{\\\"city\\\":\\\"Paris\\\"}\"}},{\"id\":\"call_2\",\"type\":\"function\",\"function\":{\"name\":\"f\","
|
||||
"\"arguments\":\"{broken\"}}]}}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}";
|
||||
// SHAPE: chat.completion.chunk stream, as OpenAI-compatible providers send it
|
||||
const char* kStream =
|
||||
"data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"He\"}}]}\n\n"
|
||||
"data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"llo\"}}]}\n\n"
|
||||
"data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\"\"}}]}}]}\n\n"
|
||||
"data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\":1}\"}}]}}]}\n\n"
|
||||
"data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n"
|
||||
"data: {\"id\":\"c\",\"choices\":[],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":2,\"total_tokens\":6}}\n\n"
|
||||
"data: [DONE]\n\n";
|
||||
// RECORDED 2026-10-25: GET https://router.huggingface.co/v1/models (first entry, 2 of its providers)
|
||||
const char* kRouterModels =
|
||||
"{\"object\":\"list\",\"data\":[{\"id\":\"deepseek-ai/DeepSeek-V4.1-Flash-1017\",\"object\":\"model\",\"created\":1792204838,"
|
||||
"\"owned_by\":\"deepseek-ai\",\"providers\":[{\"provider\":\"novita\",\"status\":\"live\",\"context_length\":1048576,"
|
||||
"\"supports_tools\":true}]}]}";
|
||||
|
||||
struct FakeEndpoint : ChatEndpoint {
|
||||
Json last_body;
|
||||
std::string reply_json;
|
||||
std::string stream_text;
|
||||
Json create(const Json& body) override {
|
||||
last_body = body;
|
||||
return Json::parse(reply_json);
|
||||
}
|
||||
void stream(const Json& body, const molo::llm::SseDecoder::Sink& sink) override {
|
||||
last_body = body;
|
||||
molo::llm::SseDecoder dec("huggingface", sink);
|
||||
dec.feed(stream_text);
|
||||
dec.finish();
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
TEST(huggingface_router, model_selectors) {
|
||||
RouterModel m = RouterModel::parse("openai/gpt-oss-20b:fastest");
|
||||
ASSERT_EQ(m.id, std::string("openai/gpt-oss-20b"));
|
||||
ASSERT_EQ(m.selector, std::string("fastest"));
|
||||
ASSERT_EQ(m.str(), std::string("openai/gpt-oss-20b:fastest"));
|
||||
ASSERT_EQ(RouterModel::parse("a/b").selector, std::string(""));
|
||||
ASSERT_THROW(RouterModel::parse("a/b:x:y"), ApiError);
|
||||
ASSERT_THROW(RouterModel::parse("a/b:"), ApiError) ;
|
||||
ASSERT_THROW(RouterModel::parse("a b"), ApiError);
|
||||
ASSERT_THROW(RouterModel::parse(""), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_router, chat_request_and_bill_to_header) {
|
||||
Rig rig;
|
||||
RouterChat chat = RouterChat::from_platform(rig.platform);
|
||||
chat.set_bill_to("acme");
|
||||
rig.router->expect_response(Method::kPost, "/v1/chat/completions", reply(kChat));
|
||||
Json body = Json::object();
|
||||
body.set("model", Json("openai/gpt-oss-20b:cheapest"));
|
||||
Json msgs = Json::array();
|
||||
Json m = Json::object();
|
||||
m.set("role", Json("user"));
|
||||
m.set("content", Json("hi"));
|
||||
msgs.push(m);
|
||||
body.set("messages", msgs);
|
||||
Json out = chat.create(body);
|
||||
ASSERT_EQ(out.find("id")->as_string(), std::string("cmpl-1"));
|
||||
auto seen = rig.router->seen();
|
||||
ASSERT_EQ(Rig::header(seen[0], "X-HF-Bill-To"), std::string("acme"));
|
||||
ASSERT_EQ(Rig::header(seen[0], "Authorization"), std::string("Bearer ") + kToken);
|
||||
ASSERT_THROW(chat.set_bill_to("a b"), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_router, chat_refuses_bad_bodies_before_sending) {
|
||||
Rig rig;
|
||||
RouterChat chat = RouterChat::from_platform(rig.platform);
|
||||
Json no_model = Json::object();
|
||||
ASSERT_THROW(chat.create(no_model), ApiError);
|
||||
Json unknown = Json::object();
|
||||
unknown.set("model", Json("a/b"));
|
||||
unknown.set("provider_secret_sauce", Json(1));
|
||||
ASSERT_THROW(chat.create(unknown), ApiError);
|
||||
Json streaming = Json::object();
|
||||
streaming.set("model", Json("a/b"));
|
||||
streaming.set("stream", Json(true));
|
||||
ASSERT_THROW(chat.create(streaming), ApiError);
|
||||
ASSERT_EQ(rig.router->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(huggingface_router, chat_without_token_is_not_configured) {
|
||||
Rig rig("");
|
||||
RouterChat chat = RouterChat::from_platform(rig.platform);
|
||||
Json b = Json::object();
|
||||
b.set("model", Json("a/b"));
|
||||
try {
|
||||
chat.create(b);
|
||||
FAIL("expected NOT_CONFIGURED");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
|
||||
}
|
||||
ASSERT_EQ(rig.router->send_count(), static_cast<std::size_t>(0));
|
||||
}
|
||||
|
||||
TEST(huggingface_router, models_list_is_public) {
|
||||
Rig rig("");
|
||||
RouterChat chat = RouterChat::from_platform(rig.platform);
|
||||
rig.router->expect_response(Method::kGet, "/v1/models", reply(kRouterModels));
|
||||
Json m = chat.models();
|
||||
ASSERT_EQ(m.find("data")->as_array().front().find("owned_by")->as_string(), std::string("deepseek-ai"));
|
||||
ASSERT_EQ(Rig::header(rig.router->seen()[0], "Authorization"), std::string(""));
|
||||
}
|
||||
|
||||
TEST(huggingface_tasks, catalog_matches_the_hf_inference_map) {
|
||||
ASSERT_EQ(TaskCatalog::count(), static_cast<std::size_t>(25));
|
||||
ASSERT_TRUE(TaskCatalog::find("feature-extraction")->pipeline_path);
|
||||
ASSERT_TRUE(TaskCatalog::find("sentence-similarity")->pipeline_path);
|
||||
ASSERT_TRUE(TaskCatalog::find("image-classification")->binary_input);
|
||||
ASSERT_FALSE(TaskCatalog::find("translation")->binary_input);
|
||||
ASSERT_TRUE(TaskCatalog::find("conversational") == nullptr); // chat has its own surface
|
||||
}
|
||||
|
||||
TEST(huggingface_tasks, json_task_request) {
|
||||
Rig rig;
|
||||
InferenceTasks t = InferenceTasks::from_platform(rig.platform);
|
||||
rig.router->expect_response(Method::kPost, "/hf-inference/models/distilbert/distilbert-base-uncased-finetuned-sst-2-english",
|
||||
reply("[[{\"label\":\"POSITIVE\",\"score\":0.99}]]"));
|
||||
Json params = Json::object();
|
||||
params.set("top_k", Json(1));
|
||||
InferenceResult r = t.run_json("text-classification", "distilbert/distilbert-base-uncased-finetuned-sst-2-english",
|
||||
Json("I like you"), params);
|
||||
ASSERT_TRUE(r.is_json);
|
||||
ASSERT_EQ(rig.router->seen()[0].body, std::string("{\"inputs\":\"I like you\",\"parameters\":{\"top_k\":1}}"));
|
||||
}
|
||||
|
||||
TEST(huggingface_tasks, pipeline_tasks_put_parameters_at_the_top_level) {
|
||||
Rig rig;
|
||||
InferenceTasks t = InferenceTasks::from_platform(rig.platform);
|
||||
rig.router->expect_response(Method::kPost, "/hf-inference/models/BAAI/bge-small-en-v1.5/pipeline/feature-extraction",
|
||||
reply("[[0.1,0.2]]"));
|
||||
Json e = t.embed("BAAI/bge-small-en-v1.5", {"hello"}, true);
|
||||
ASSERT_TRUE(e.is_array());
|
||||
ASSERT_EQ(rig.router->seen()[0].body, std::string("{\"inputs\":\"hello\",\"normalize\":true}"));
|
||||
ASSERT_THROW(t.embed("BAAI/bge-small-en-v1.5", {}), ApiError);
|
||||
ASSERT_THROW(t.embed("BAAI/bge-small-en-v1.5", std::vector<std::string>(257, "x")), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_tasks, binary_tasks_send_raw_bytes_or_base64_with_parameters) {
|
||||
Rig rig;
|
||||
InferenceTasks t = InferenceTasks::from_platform(rig.platform);
|
||||
rig.router->expect_response(Method::kPost, "/hf-inference/models/google/vit-base-patch16-224", reply("[{\"label\":\"cat\",\"score\":0.9}]"));
|
||||
rig.router->expect_response(Method::kPost, "/hf-inference/models/google/vit-base-patch16-224", reply("[]"));
|
||||
t.run_binary("image-classification", "google/vit-base-patch16-224", "\x89PNG", "image/png");
|
||||
auto first = rig.router->seen()[0];
|
||||
ASSERT_EQ(first.body, std::string("\x89PNG"));
|
||||
ASSERT_EQ(Rig::header(first, "Content-Type"), std::string("image/png"));
|
||||
Json p = Json::object();
|
||||
p.set("top_k", Json(2));
|
||||
t.run_binary("image-classification", "google/vit-base-patch16-224", "abc", "image/png", p);
|
||||
ASSERT_EQ(rig.router->seen()[1].body, std::string("{\"inputs\":\"YWJj\",\"parameters\":{\"top_k\":2}}"));
|
||||
ASSERT_THROW(t.run_json("image-classification", "m", Json("x")), ApiError);
|
||||
ASSERT_THROW(t.run_binary("translation", "m", "x", "text/plain"), ApiError);
|
||||
ASSERT_THROW(t.run_binary("image-classification", "m", "", "image/png"), ApiError);
|
||||
ASSERT_THROW(t.run_json("no-such-task", "m", Json("x")), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_tasks, binary_replies_are_returned_as_bytes) {
|
||||
Rig rig;
|
||||
InferenceTasks t = InferenceTasks::from_platform(rig.platform);
|
||||
rig.router->expect_response(Method::kPost, "/hf-inference/models/black-forest-labs/FLUX.1-dev",
|
||||
reply("\x89PNG\r\n", 200, "image/png"));
|
||||
InferenceResult r = t.run_json("text-to-image", "black-forest-labs/FLUX.1-dev", Json("a cat"));
|
||||
ASSERT_FALSE(r.is_json);
|
||||
ASSERT_EQ(r.content_type, std::string("image/png"));
|
||||
ASSERT_EQ(r.bytes.size(), static_cast<std::size_t>(6));
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, neutral_request_maps_to_the_wire_body) {
|
||||
molo::llm::ChatRequest req;
|
||||
req.model = "openai/gpt-oss-20b:fastest";
|
||||
req.system = "be brief";
|
||||
req.messages.push_back(molo::llm::Message::user("hi"));
|
||||
req.max_tokens = 32;
|
||||
req.temperature = 0.5;
|
||||
req.stop = {"END"};
|
||||
req.json_mode = true;
|
||||
req.reasoning_effort = "low";
|
||||
molo::llm::ToolSpec tool;
|
||||
tool.name = "get_weather";
|
||||
tool.description = "weather";
|
||||
req.tools.push_back(tool);
|
||||
req.tool_choice = "get_weather";
|
||||
Json b = ChatWire::to_body(req, "ignored", false);
|
||||
ASSERT_EQ(b.find("model")->as_string(), std::string("openai/gpt-oss-20b:fastest"));
|
||||
ASSERT_EQ(b.find("messages")->as_array().size(), static_cast<std::size_t>(2));
|
||||
ASSERT_EQ(b.find("messages")->as_array()[0].find("role")->as_string(), std::string("system"));
|
||||
ASSERT_EQ(b.find("response_format")->find("type")->as_string(), std::string("json_object"));
|
||||
ASSERT_EQ(b.find("tool_choice")->find("function")->find("name")->as_string(), std::string("get_weather"));
|
||||
ASSERT_EQ(b.find("reasoning_effort")->as_string(), std::string("low"));
|
||||
ASSERT_TRUE(b.find("stream") == nullptr);
|
||||
Json s = ChatWire::to_body(req, "x", true);
|
||||
ASSERT_TRUE(s.find("stream")->as_bool());
|
||||
ASSERT_TRUE(s.find("stream_options")->find("include_usage")->as_bool());
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, requests_are_validated) {
|
||||
molo::llm::ChatRequest req;
|
||||
req.messages.push_back(molo::llm::Message::user("hi"));
|
||||
ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError); // no model anywhere
|
||||
req.model = "a/b";
|
||||
req.stop = {"1", "2", "3", "4", "5"};
|
||||
ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError);
|
||||
req.stop.clear();
|
||||
req.json_mode = true;
|
||||
req.response_schema = Json::object();
|
||||
ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError);
|
||||
req.json_mode = false;
|
||||
req.response_schema = Json();
|
||||
req.temperature = 2.5;
|
||||
ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError);
|
||||
req.temperature.reset();
|
||||
req.max_tokens = 0;
|
||||
ASSERT_THROW(ChatWire::to_body(req, "", false), ApiError);
|
||||
req.max_tokens.reset();
|
||||
req.extra = Json::object();
|
||||
req.extra.set("seed", Json(7));
|
||||
ASSERT_TRUE(ChatWire::to_body(req, "", false).find("seed") != nullptr);
|
||||
molo::llm::ChatRequest empty;
|
||||
empty.model = "a/b";
|
||||
ASSERT_THROW(ChatWire::to_body(empty, "", false), ApiError); // no messages
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, response_parsing_with_reasoning_and_usage) {
|
||||
molo::llm::ChatResponse r = ChatWire::to_response(Json::parse(kChat));
|
||||
ASSERT_EQ(r.text(), std::string("Hello!"));
|
||||
ASSERT_EQ(r.finish_reason, std::string("stop"));
|
||||
ASSERT_EQ(r.usage.input_tokens, 9);
|
||||
ASSERT_EQ(r.usage.output_tokens, 3);
|
||||
ASSERT_EQ(r.usage.reasoning_tokens, 2);
|
||||
ASSERT_EQ(r.message.parts.front().reasoning, std::string("greeting"));
|
||||
ASSERT_THROW(ChatWire::to_response(Json::parse("{\"choices\":[]}")), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, tool_calls_keep_invalid_json_visible) {
|
||||
molo::llm::ChatResponse r = ChatWire::to_response(Json::parse(kToolReply));
|
||||
std::vector<molo::llm::ToolCall> calls = r.tool_calls();
|
||||
ASSERT_EQ(calls.size(), static_cast<std::size_t>(2));
|
||||
ASSERT_EQ(calls[0].name, std::string("get_weather"));
|
||||
ASSERT_EQ(calls[0].arguments.find("city")->as_string(), std::string("Paris"));
|
||||
ASSERT_EQ(calls[1].arguments.find("_invalid_json")->as_string(), std::string("{broken"));
|
||||
ASSERT_EQ(r.finish_reason, std::string("tool_calls"));
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, model_over_a_fake_endpoint) {
|
||||
auto ep = std::make_shared<FakeEndpoint>();
|
||||
ep->reply_json = kChat;
|
||||
HuggingfaceChatModel model(ep, "openai/gpt-oss-20b");
|
||||
molo::llm::ChatRequest req;
|
||||
req.messages.push_back(molo::llm::Message::user("hi"));
|
||||
molo::llm::ChatResponse r = model.generate(req);
|
||||
ASSERT_EQ(r.text(), std::string("Hello!"));
|
||||
ASSERT_EQ(ep->last_body.find("model")->as_string(), std::string("openai/gpt-oss-20b"));
|
||||
ASSERT_EQ(model.vendor(), std::string("huggingface"));
|
||||
ASSERT_THROW(HuggingfaceChatModel(nullptr, "m"), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, stream_transcript_and_early_stop) {
|
||||
auto ep = std::make_shared<FakeEndpoint>();
|
||||
ep->stream_text = kStream;
|
||||
HuggingfaceChatModel model(ep, "a/b");
|
||||
molo::llm::ChatRequest req;
|
||||
req.messages.push_back(molo::llm::Message::user("hi"));
|
||||
std::string text, args, id, finish;
|
||||
int total = 0;
|
||||
model.stream(req, [&](const molo::llm::ChatChunk& c) {
|
||||
text += c.text_delta;
|
||||
args += c.tool_args_delta;
|
||||
if (!c.tool_call_id.empty()) id = c.tool_call_id;
|
||||
if (!c.finish_reason.empty()) finish = c.finish_reason;
|
||||
if (c.usage) total = c.usage->total_tokens;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("Hello"));
|
||||
ASSERT_EQ(args, std::string("{\"a\":1}"));
|
||||
ASSERT_EQ(id, std::string("call_1"));
|
||||
ASSERT_EQ(finish, std::string("stop"));
|
||||
ASSERT_EQ(total, 6);
|
||||
ASSERT_TRUE(ep->last_body.find("stream")->as_bool());
|
||||
|
||||
int seen = 0;
|
||||
model.stream(req, [&](const molo::llm::ChatChunk&) { return ++seen < 2; });
|
||||
ASSERT_EQ(seen, 2);
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, error_event_in_stream_throws) {
|
||||
auto ep = std::make_shared<FakeEndpoint>();
|
||||
ep->stream_text = "data: {\"error\":{\"message\":\"provider failed\"}}\n\n";
|
||||
HuggingfaceChatModel model(ep, "a/b");
|
||||
molo::llm::ChatRequest req;
|
||||
req.messages.push_back(molo::llm::Message::user("hi"));
|
||||
ASSERT_THROW(model.stream(req, [](const molo::llm::ChatChunk&) { return true; }), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_chat, router_model_streams_over_the_real_client) {
|
||||
Rig rig;
|
||||
rig.router->expect_response(Method::kPost, "/v1/chat/completions", sse(kStream));
|
||||
auto model = HuggingfaceChatModel::over_router(rig.platform, "a/b");
|
||||
molo::llm::ChatRequest req;
|
||||
req.messages.push_back(molo::llm::Message::user("hi"));
|
||||
std::string text;
|
||||
model->stream(req, [&](const molo::llm::ChatChunk& c) {
|
||||
text += c.text_delta;
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("Hello"));
|
||||
ASSERT_TRUE(has(rig.router->seen()[0].body, "\"stream\":true"));
|
||||
}
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
// Inference Endpoints management, TGI and TEI. Requests are checked against
|
||||
// the call table; replies are SHAPE or RECORDED.
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/endpoints.h"
|
||||
#include "vendors/huggingface/servers.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using namespace hf_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// RECORDED 2026-10-25: GET https://api.endpoints.huggingface.cloud/v2/provider (no token), trimmed
|
||||
const char* kVendors =
|
||||
"{\"vendors\":[{\"name\":\"aws\",\"status\":\"available\",\"regions\":[{\"name\":\"eu-west-1\",\"label\":\"Ireland\","
|
||||
"\"status\":\"available\",\"computes\":[{\"id\":\"aws-eu-west-1-intel-spr-x1\",\"accelerator\":\"cpu\","
|
||||
"\"instanceType\":\"intel-spr\",\"instanceSize\":\"x1\",\"status\":\"available\",\"pricePerHour\":0.033}]}]}]}";
|
||||
// RECORDED 2026-10-25: any call without a valid token -> 401
|
||||
const char* kIeUnauth = "{\"error\":\"401 Unauthorized\",\"code\":\"UNAUTHORIZED\"}";
|
||||
|
||||
Json spec() {
|
||||
return Json::parse(
|
||||
"{\"name\":\"my-ep\",\"type\":\"protected\",\"provider\":{\"vendor\":\"aws\",\"region\":\"us-east-1\"},"
|
||||
"\"compute\":{},\"model\":{}}");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(huggingface_endpoints, vendors_list_is_public_everything_else_needs_a_token) {
|
||||
Rig rig("");
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/provider", reply(kVendors));
|
||||
Json v = ie.vendors();
|
||||
ASSERT_EQ(v.find("vendors")->as_array().front().find("name")->as_string(), std::string("aws"));
|
||||
try {
|
||||
ie.list("acme");
|
||||
FAIL("expected NOT_CONFIGURED");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
|
||||
}
|
||||
ASSERT_EQ(rig.endpoints->send_count(), static_cast<std::size_t>(1));
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, error_code_is_kept) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/huggingface", reply(kIeUnauth, 401));
|
||||
try {
|
||||
ie.list("huggingface");
|
||||
FAIL("expected 401");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 401);
|
||||
ASSERT_TRUE(has(e.message(), "401 Unauthorized"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, reads_build_the_documented_paths) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme", reply("{\"items\":[]}"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/replica", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/logs", reply("line1\nline2\n", 200, "text/plain"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/search/acme/states", reply("{}"));
|
||||
EndpointListParams p;
|
||||
p.limit = 5;
|
||||
p.state = "running";
|
||||
ie.list("acme", p);
|
||||
ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme?state=running&limit=5"));
|
||||
ie.get("acme", "ep");
|
||||
ie.replicas("acme", "ep");
|
||||
ASSERT_EQ(ie.logs("acme", "ep", 50), std::string("line1\nline2\n"));
|
||||
ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme/ep/logs?tail=50"));
|
||||
ie.state_counts("acme");
|
||||
ASSERT_THROW(ie.get("acme", "../x"), ApiError);
|
||||
ASSERT_THROW(ie.list("acme", EndpointListParams{"", "", "", "", "", "", 101}), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, metrics_use_post_with_a_time_window) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/metrics/inference/ttft", reply("{}"));
|
||||
ie.metrics(MetricKind::kTtft, "acme", "ep", "2021-01-01T00:00:00Z", "2021-01-02T00:00:00Z", "r1");
|
||||
ASSERT_EQ(rig.endpoints->seen().back().body,
|
||||
std::string("{\"replicaId\":\"r1\",\"start\":\"2021-01-01T00:00:00Z\",\"stop\":\"2021-01-02T00:00:00Z\"}"));
|
||||
ASSERT_THROW(ie.metrics(MetricKind::kTtft, "acme", "ep", "yesterday", "today"), ApiError);
|
||||
MetricKind k;
|
||||
ASSERT_TRUE(parse_metric_kind("kv-cache", k));
|
||||
ASSERT_FALSE(parse_metric_kind("nope", k));
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, writes_are_validated_and_explicit) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
ASSERT_THROW(ie.create("acme", spec()), ApiError); // type "protected" is not a type
|
||||
Json good = spec();
|
||||
good.set("type", Json("private"));
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme", reply("{}"));
|
||||
ie.create("acme", good);
|
||||
Json bad_name = good;
|
||||
bad_name.set("name", Json("My_Endpoint"));
|
||||
ASSERT_THROW(ie.create("acme", bad_name), ApiError);
|
||||
Json missing = Json::object();
|
||||
missing.set("name", Json("x"));
|
||||
ASSERT_THROW(ie.create("acme", missing), ApiError);
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/pause", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/resume", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/scale-to-zero", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kDelete, "/v2/endpoint/acme/ep", reply("{}"));
|
||||
ie.pause("acme", "ep");
|
||||
ie.resume("acme", "ep");
|
||||
ie.scale_to_zero("acme", "ep");
|
||||
ie.remove_endpoint("acme", "ep");
|
||||
rig.endpoints->verify();
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, sse_streams_are_listed_but_refused) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
ASSERT_EQ(rig.endpoints->send_count(), static_cast<std::size_t>(0));
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i)
|
||||
if (Calls::at(i).has_flag("unsupported")) ASSERT_TRUE(std::string(Calls::at(i).path).find("sse") != std::string::npos ||
|
||||
std::string(Calls::at(i).path).find("/sse") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(huggingface_tgi, native_and_openai_routes) {
|
||||
Rig rig; // transports unused: the server has its own
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TgiServer tgi(Platform::server_with(std::move(mock), "http://127.0.0.1:8080/", ""));
|
||||
ASSERT_FALSE(tgi.has_token());
|
||||
t->expect_response(Method::kGet, "/health", reply("", 200, "text/plain"));
|
||||
MockExpectation starting; // the client retries a 503 GET, so allow it to answer more than once
|
||||
starting.method = Method::kGet;
|
||||
starting.path = "/health";
|
||||
starting.response = reply("starting", 503, "text/plain");
|
||||
starting.times = 5;
|
||||
t->expect(starting);
|
||||
ASSERT_TRUE(tgi.ready());
|
||||
ASSERT_FALSE(tgi.ready());
|
||||
t->expect_response(Method::kPost, "/generate", reply("{\"generated_text\":\"hi\"}"));
|
||||
Json params = Json::object();
|
||||
params.set("max_new_tokens", Json(8));
|
||||
tgi.generate("Hello", params);
|
||||
ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"Hello\",\"parameters\":{\"max_new_tokens\":8}}"));
|
||||
ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string(""));
|
||||
ASSERT_THROW(tgi.generate("", params), ApiError);
|
||||
Json bad = Json::object();
|
||||
bad.set("max_new_tokens", Json(8));
|
||||
ASSERT_THROW(tgi.completions(bad), ApiError); // body needs a table field (prompt is optional, max_new_tokens is not a field)
|
||||
}
|
||||
|
||||
TEST(huggingface_tgi, generate_stream_decodes_token_events) {
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TgiServer tgi(Platform::server_with(std::move(mock), "http://tgi.local:8080", "hf_tok"));
|
||||
ASSERT_TRUE(tgi.has_token());
|
||||
// SHAPE: TGI streams one StreamResponse per `data:` line
|
||||
t->expect_response(Method::kPost, "/generate_stream",
|
||||
sse("data:{\"index\":1,\"token\":{\"id\":1,\"text\":\"a\",\"logprob\":-0.1,\"special\":false},\"generated_text\":null}\n\n"
|
||||
"data:{\"index\":2,\"token\":{\"id\":2,\"text\":\"b\",\"logprob\":-0.2,\"special\":false},\"generated_text\":\"ab\"}\n\n"));
|
||||
std::string text;
|
||||
tgi.generate_stream("x", Json(), [&](const molo::llm::SseEvent& e) {
|
||||
text += e.data.find("token")->find("text")->as_string();
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("ab"));
|
||||
ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("Bearer hf_tok"));
|
||||
}
|
||||
|
||||
TEST(huggingface_tei, embed_rerank_and_validation) {
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", ""));
|
||||
t->expect_response(Method::kPost, "/embed", reply("[[0.1,0.2]]"));
|
||||
Json opts = Json::object();
|
||||
opts.set("normalize", Json(true));
|
||||
tei.embed(Json("hello"), opts);
|
||||
ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"hello\",\"normalize\":true}"));
|
||||
t->expect_response(Method::kPost, "/rerank", reply("[{\"index\":0,\"score\":0.9}]"));
|
||||
tei.rerank("q", {"a", "b"}, true);
|
||||
ASSERT_EQ(t->seen().back().body, std::string("{\"query\":\"q\",\"return_text\":true,\"texts\":[\"a\",\"b\"]}"));
|
||||
ASSERT_THROW(tei.embed(Json(""), Json()), ApiError);
|
||||
ASSERT_THROW(tei.embed(Json(7), Json()), ApiError);
|
||||
ASSERT_THROW(tei.rerank("q", {}), ApiError);
|
||||
Json many = Json::array();
|
||||
for (int i = 0; i < 257; ++i) many.push(Json("x"));
|
||||
ASSERT_THROW(tei.embed(many, Json()), ApiError);
|
||||
Json unknown = Json::object();
|
||||
unknown.set("hack", Json(1));
|
||||
ASSERT_THROW(tei.embed(Json("x"), unknown), ApiError);
|
||||
ASSERT_THROW(tei.decode({}, true), ApiError);
|
||||
ASSERT_THROW(tei.openai_embeddings(Json("x"), "", -1), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_servers, call_from_the_wrong_surface_is_refused) {
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", ""));
|
||||
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
|
||||
ASSERT_TRUE(tei.surface() == Surface::kTei);
|
||||
}
|
||||
Reference in New Issue
Block a user