feat(clients): huggingface: Hub, Inference Providers, Endpoints, TGI, TEI, chat model, demo

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 15:01:40 +01:00
co-authored by Claude Sonnet 5.5
parent 3db67fcf1f
commit 0b806eb7e3
36 changed files with 5949 additions and 3 deletions
+200
View File
@@ -0,0 +1,200 @@
// Inference Endpoints management, TGI and TEI. Requests are checked against
// the call table; replies are SHAPE or RECORDED.
#include "support.h"
#include "vendors/huggingface/endpoints.h"
#include "vendors/huggingface/servers.h"
using namespace molo::huggingface;
using namespace hf_test;
using molo::llm::ApiError;
namespace {
// RECORDED 2026-10-25: GET https://api.endpoints.huggingface.cloud/v2/provider (no token), trimmed
const char* kVendors =
"{\"vendors\":[{\"name\":\"aws\",\"status\":\"available\",\"regions\":[{\"name\":\"eu-west-1\",\"label\":\"Ireland\","
"\"status\":\"available\",\"computes\":[{\"id\":\"aws-eu-west-1-intel-spr-x1\",\"accelerator\":\"cpu\","
"\"instanceType\":\"intel-spr\",\"instanceSize\":\"x1\",\"status\":\"available\",\"pricePerHour\":0.033}]}]}]}";
// RECORDED 2026-10-25: any call without a valid token -> 401
const char* kIeUnauth = "{\"error\":\"401 Unauthorized\",\"code\":\"UNAUTHORIZED\"}";
Json spec() {
return Json::parse(
"{\"name\":\"my-ep\",\"type\":\"protected\",\"provider\":{\"vendor\":\"aws\",\"region\":\"us-east-1\"},"
"\"compute\":{},\"model\":{}}");
}
} // namespace
TEST(huggingface_endpoints, vendors_list_is_public_everything_else_needs_a_token) {
Rig rig("");
Endpoints ie = Endpoints::from_platform(rig.platform);
rig.endpoints->expect_response(Method::kGet, "/v2/provider", reply(kVendors));
Json v = ie.vendors();
ASSERT_EQ(v.find("vendors")->as_array().front().find("name")->as_string(), std::string("aws"));
try {
ie.list("acme");
FAIL("expected NOT_CONFIGURED");
} catch (const ApiError& e) {
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
}
ASSERT_EQ(rig.endpoints->send_count(), static_cast<std::size_t>(1));
}
TEST(huggingface_endpoints, error_code_is_kept) {
Rig rig;
Endpoints ie = Endpoints::from_platform(rig.platform);
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/huggingface", reply(kIeUnauth, 401));
try {
ie.list("huggingface");
FAIL("expected 401");
} catch (const ApiError& e) {
ASSERT_EQ(e.http_status(), 401);
ASSERT_TRUE(has(e.message(), "401 Unauthorized"));
}
}
TEST(huggingface_endpoints, reads_build_the_documented_paths) {
Rig rig;
Endpoints ie = Endpoints::from_platform(rig.platform);
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme", reply("{\"items\":[]}"));
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep", reply("{}"));
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/replica", reply("{}"));
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/logs", reply("line1\nline2\n", 200, "text/plain"));
rig.endpoints->expect_response(Method::kGet, "/v2/search/acme/states", reply("{}"));
EndpointListParams p;
p.limit = 5;
p.state = "running";
ie.list("acme", p);
ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme?state=running&limit=5"));
ie.get("acme", "ep");
ie.replicas("acme", "ep");
ASSERT_EQ(ie.logs("acme", "ep", 50), std::string("line1\nline2\n"));
ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme/ep/logs?tail=50"));
ie.state_counts("acme");
ASSERT_THROW(ie.get("acme", "../x"), ApiError);
ASSERT_THROW(ie.list("acme", EndpointListParams{"", "", "", "", "", "", 101}), ApiError);
}
TEST(huggingface_endpoints, metrics_use_post_with_a_time_window) {
Rig rig;
Endpoints ie = Endpoints::from_platform(rig.platform);
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/metrics/inference/ttft", reply("{}"));
ie.metrics(MetricKind::kTtft, "acme", "ep", "2021-01-01T00:00:00Z", "2021-01-02T00:00:00Z", "r1");
ASSERT_EQ(rig.endpoints->seen().back().body,
std::string("{\"replicaId\":\"r1\",\"start\":\"2021-01-01T00:00:00Z\",\"stop\":\"2021-01-02T00:00:00Z\"}"));
ASSERT_THROW(ie.metrics(MetricKind::kTtft, "acme", "ep", "yesterday", "today"), ApiError);
MetricKind k;
ASSERT_TRUE(parse_metric_kind("kv-cache", k));
ASSERT_FALSE(parse_metric_kind("nope", k));
}
TEST(huggingface_endpoints, writes_are_validated_and_explicit) {
Rig rig;
Endpoints ie = Endpoints::from_platform(rig.platform);
ASSERT_THROW(ie.create("acme", spec()), ApiError); // type "protected" is not a type
Json good = spec();
good.set("type", Json("private"));
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme", reply("{}"));
ie.create("acme", good);
Json bad_name = good;
bad_name.set("name", Json("My_Endpoint"));
ASSERT_THROW(ie.create("acme", bad_name), ApiError);
Json missing = Json::object();
missing.set("name", Json("x"));
ASSERT_THROW(ie.create("acme", missing), ApiError);
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/pause", reply("{}"));
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/resume", reply("{}"));
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/scale-to-zero", reply("{}"));
rig.endpoints->expect_response(Method::kDelete, "/v2/endpoint/acme/ep", reply("{}"));
ie.pause("acme", "ep");
ie.resume("acme", "ep");
ie.scale_to_zero("acme", "ep");
ie.remove_endpoint("acme", "ep");
rig.endpoints->verify();
}
TEST(huggingface_endpoints, sse_streams_are_listed_but_refused) {
Rig rig;
Endpoints ie = Endpoints::from_platform(rig.platform);
ASSERT_EQ(rig.endpoints->send_count(), static_cast<std::size_t>(0));
for (std::size_t i = 0; i < Calls::count(); ++i)
if (Calls::at(i).has_flag("unsupported")) ASSERT_TRUE(std::string(Calls::at(i).path).find("sse") != std::string::npos ||
std::string(Calls::at(i).path).find("/sse") != std::string::npos);
}
TEST(huggingface_tgi, native_and_openai_routes) {
Rig rig; // transports unused: the server has its own
auto mock = std::make_unique<MockTransport>();
MockTransport* t = mock.get();
TgiServer tgi(Platform::server_with(std::move(mock), "http://127.0.0.1:8080/", ""));
ASSERT_FALSE(tgi.has_token());
t->expect_response(Method::kGet, "/health", reply("", 200, "text/plain"));
MockExpectation starting; // the client retries a 503 GET, so allow it to answer more than once
starting.method = Method::kGet;
starting.path = "/health";
starting.response = reply("starting", 503, "text/plain");
starting.times = 5;
t->expect(starting);
ASSERT_TRUE(tgi.ready());
ASSERT_FALSE(tgi.ready());
t->expect_response(Method::kPost, "/generate", reply("{\"generated_text\":\"hi\"}"));
Json params = Json::object();
params.set("max_new_tokens", Json(8));
tgi.generate("Hello", params);
ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"Hello\",\"parameters\":{\"max_new_tokens\":8}}"));
ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string(""));
ASSERT_THROW(tgi.generate("", params), ApiError);
Json bad = Json::object();
bad.set("max_new_tokens", Json(8));
ASSERT_THROW(tgi.completions(bad), ApiError); // body needs a table field (prompt is optional, max_new_tokens is not a field)
}
TEST(huggingface_tgi, generate_stream_decodes_token_events) {
auto mock = std::make_unique<MockTransport>();
MockTransport* t = mock.get();
TgiServer tgi(Platform::server_with(std::move(mock), "http://tgi.local:8080", "hf_tok"));
ASSERT_TRUE(tgi.has_token());
// SHAPE: TGI streams one StreamResponse per `data:` line
t->expect_response(Method::kPost, "/generate_stream",
sse("data:{\"index\":1,\"token\":{\"id\":1,\"text\":\"a\",\"logprob\":-0.1,\"special\":false},\"generated_text\":null}\n\n"
"data:{\"index\":2,\"token\":{\"id\":2,\"text\":\"b\",\"logprob\":-0.2,\"special\":false},\"generated_text\":\"ab\"}\n\n"));
std::string text;
tgi.generate_stream("x", Json(), [&](const molo::llm::SseEvent& e) {
text += e.data.find("token")->find("text")->as_string();
return true;
});
ASSERT_EQ(text, std::string("ab"));
ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("Bearer hf_tok"));
}
TEST(huggingface_tei, embed_rerank_and_validation) {
auto mock = std::make_unique<MockTransport>();
MockTransport* t = mock.get();
TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", ""));
t->expect_response(Method::kPost, "/embed", reply("[[0.1,0.2]]"));
Json opts = Json::object();
opts.set("normalize", Json(true));
tei.embed(Json("hello"), opts);
ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"hello\",\"normalize\":true}"));
t->expect_response(Method::kPost, "/rerank", reply("[{\"index\":0,\"score\":0.9}]"));
tei.rerank("q", {"a", "b"}, true);
ASSERT_EQ(t->seen().back().body, std::string("{\"query\":\"q\",\"return_text\":true,\"texts\":[\"a\",\"b\"]}"));
ASSERT_THROW(tei.embed(Json(""), Json()), ApiError);
ASSERT_THROW(tei.embed(Json(7), Json()), ApiError);
ASSERT_THROW(tei.rerank("q", {}), ApiError);
Json many = Json::array();
for (int i = 0; i < 257; ++i) many.push(Json("x"));
ASSERT_THROW(tei.embed(many, Json()), ApiError);
Json unknown = Json::object();
unknown.set("hack", Json(1));
ASSERT_THROW(tei.embed(Json("x"), unknown), ApiError);
ASSERT_THROW(tei.decode({}, true), ApiError);
ASSERT_THROW(tei.openai_embeddings(Json("x"), "", -1), ApiError);
}
TEST(huggingface_servers, call_from_the_wrong_surface_is_refused) {
auto mock = std::make_unique<MockTransport>();
MockTransport* t = mock.get();
TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", ""));
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
ASSERT_TRUE(tei.surface() == Surface::kTei);
}