feat(clients): huggingface: Hub, Inference Providers, Endpoints, TGI, TEI, chat model, demo
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+200
@@ -0,0 +1,200 @@
|
||||
// Inference Endpoints management, TGI and TEI. Requests are checked against
|
||||
// the call table; replies are SHAPE or RECORDED.
|
||||
#include "support.h"
|
||||
#include "vendors/huggingface/endpoints.h"
|
||||
#include "vendors/huggingface/servers.h"
|
||||
|
||||
using namespace molo::huggingface;
|
||||
using namespace hf_test;
|
||||
using molo::llm::ApiError;
|
||||
|
||||
namespace {
|
||||
// RECORDED 2026-10-25: GET https://api.endpoints.huggingface.cloud/v2/provider (no token), trimmed
|
||||
const char* kVendors =
|
||||
"{\"vendors\":[{\"name\":\"aws\",\"status\":\"available\",\"regions\":[{\"name\":\"eu-west-1\",\"label\":\"Ireland\","
|
||||
"\"status\":\"available\",\"computes\":[{\"id\":\"aws-eu-west-1-intel-spr-x1\",\"accelerator\":\"cpu\","
|
||||
"\"instanceType\":\"intel-spr\",\"instanceSize\":\"x1\",\"status\":\"available\",\"pricePerHour\":0.033}]}]}]}";
|
||||
// RECORDED 2026-10-25: any call without a valid token -> 401
|
||||
const char* kIeUnauth = "{\"error\":\"401 Unauthorized\",\"code\":\"UNAUTHORIZED\"}";
|
||||
|
||||
Json spec() {
|
||||
return Json::parse(
|
||||
"{\"name\":\"my-ep\",\"type\":\"protected\",\"provider\":{\"vendor\":\"aws\",\"region\":\"us-east-1\"},"
|
||||
"\"compute\":{},\"model\":{}}");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(huggingface_endpoints, vendors_list_is_public_everything_else_needs_a_token) {
|
||||
Rig rig("");
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/provider", reply(kVendors));
|
||||
Json v = ie.vendors();
|
||||
ASSERT_EQ(v.find("vendors")->as_array().front().find("name")->as_string(), std::string("aws"));
|
||||
try {
|
||||
ie.list("acme");
|
||||
FAIL("expected NOT_CONFIGURED");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED"));
|
||||
}
|
||||
ASSERT_EQ(rig.endpoints->send_count(), static_cast<std::size_t>(1));
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, error_code_is_kept) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/huggingface", reply(kIeUnauth, 401));
|
||||
try {
|
||||
ie.list("huggingface");
|
||||
FAIL("expected 401");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 401);
|
||||
ASSERT_TRUE(has(e.message(), "401 Unauthorized"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, reads_build_the_documented_paths) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme", reply("{\"items\":[]}"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/replica", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/logs", reply("line1\nline2\n", 200, "text/plain"));
|
||||
rig.endpoints->expect_response(Method::kGet, "/v2/search/acme/states", reply("{}"));
|
||||
EndpointListParams p;
|
||||
p.limit = 5;
|
||||
p.state = "running";
|
||||
ie.list("acme", p);
|
||||
ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme?state=running&limit=5"));
|
||||
ie.get("acme", "ep");
|
||||
ie.replicas("acme", "ep");
|
||||
ASSERT_EQ(ie.logs("acme", "ep", 50), std::string("line1\nline2\n"));
|
||||
ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme/ep/logs?tail=50"));
|
||||
ie.state_counts("acme");
|
||||
ASSERT_THROW(ie.get("acme", "../x"), ApiError);
|
||||
ASSERT_THROW(ie.list("acme", EndpointListParams{"", "", "", "", "", "", 101}), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, metrics_use_post_with_a_time_window) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/metrics/inference/ttft", reply("{}"));
|
||||
ie.metrics(MetricKind::kTtft, "acme", "ep", "2021-01-01T00:00:00Z", "2021-01-02T00:00:00Z", "r1");
|
||||
ASSERT_EQ(rig.endpoints->seen().back().body,
|
||||
std::string("{\"replicaId\":\"r1\",\"start\":\"2021-01-01T00:00:00Z\",\"stop\":\"2021-01-02T00:00:00Z\"}"));
|
||||
ASSERT_THROW(ie.metrics(MetricKind::kTtft, "acme", "ep", "yesterday", "today"), ApiError);
|
||||
MetricKind k;
|
||||
ASSERT_TRUE(parse_metric_kind("kv-cache", k));
|
||||
ASSERT_FALSE(parse_metric_kind("nope", k));
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, writes_are_validated_and_explicit) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
ASSERT_THROW(ie.create("acme", spec()), ApiError); // type "protected" is not a type
|
||||
Json good = spec();
|
||||
good.set("type", Json("private"));
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme", reply("{}"));
|
||||
ie.create("acme", good);
|
||||
Json bad_name = good;
|
||||
bad_name.set("name", Json("My_Endpoint"));
|
||||
ASSERT_THROW(ie.create("acme", bad_name), ApiError);
|
||||
Json missing = Json::object();
|
||||
missing.set("name", Json("x"));
|
||||
ASSERT_THROW(ie.create("acme", missing), ApiError);
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/pause", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/resume", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/scale-to-zero", reply("{}"));
|
||||
rig.endpoints->expect_response(Method::kDelete, "/v2/endpoint/acme/ep", reply("{}"));
|
||||
ie.pause("acme", "ep");
|
||||
ie.resume("acme", "ep");
|
||||
ie.scale_to_zero("acme", "ep");
|
||||
ie.remove_endpoint("acme", "ep");
|
||||
rig.endpoints->verify();
|
||||
}
|
||||
|
||||
TEST(huggingface_endpoints, sse_streams_are_listed_but_refused) {
|
||||
Rig rig;
|
||||
Endpoints ie = Endpoints::from_platform(rig.platform);
|
||||
ASSERT_EQ(rig.endpoints->send_count(), static_cast<std::size_t>(0));
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i)
|
||||
if (Calls::at(i).has_flag("unsupported")) ASSERT_TRUE(std::string(Calls::at(i).path).find("sse") != std::string::npos ||
|
||||
std::string(Calls::at(i).path).find("/sse") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(huggingface_tgi, native_and_openai_routes) {
|
||||
Rig rig; // transports unused: the server has its own
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TgiServer tgi(Platform::server_with(std::move(mock), "http://127.0.0.1:8080/", ""));
|
||||
ASSERT_FALSE(tgi.has_token());
|
||||
t->expect_response(Method::kGet, "/health", reply("", 200, "text/plain"));
|
||||
MockExpectation starting; // the client retries a 503 GET, so allow it to answer more than once
|
||||
starting.method = Method::kGet;
|
||||
starting.path = "/health";
|
||||
starting.response = reply("starting", 503, "text/plain");
|
||||
starting.times = 5;
|
||||
t->expect(starting);
|
||||
ASSERT_TRUE(tgi.ready());
|
||||
ASSERT_FALSE(tgi.ready());
|
||||
t->expect_response(Method::kPost, "/generate", reply("{\"generated_text\":\"hi\"}"));
|
||||
Json params = Json::object();
|
||||
params.set("max_new_tokens", Json(8));
|
||||
tgi.generate("Hello", params);
|
||||
ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"Hello\",\"parameters\":{\"max_new_tokens\":8}}"));
|
||||
ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string(""));
|
||||
ASSERT_THROW(tgi.generate("", params), ApiError);
|
||||
Json bad = Json::object();
|
||||
bad.set("max_new_tokens", Json(8));
|
||||
ASSERT_THROW(tgi.completions(bad), ApiError); // body needs a table field (prompt is optional, max_new_tokens is not a field)
|
||||
}
|
||||
|
||||
TEST(huggingface_tgi, generate_stream_decodes_token_events) {
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TgiServer tgi(Platform::server_with(std::move(mock), "http://tgi.local:8080", "hf_tok"));
|
||||
ASSERT_TRUE(tgi.has_token());
|
||||
// SHAPE: TGI streams one StreamResponse per `data:` line
|
||||
t->expect_response(Method::kPost, "/generate_stream",
|
||||
sse("data:{\"index\":1,\"token\":{\"id\":1,\"text\":\"a\",\"logprob\":-0.1,\"special\":false},\"generated_text\":null}\n\n"
|
||||
"data:{\"index\":2,\"token\":{\"id\":2,\"text\":\"b\",\"logprob\":-0.2,\"special\":false},\"generated_text\":\"ab\"}\n\n"));
|
||||
std::string text;
|
||||
tgi.generate_stream("x", Json(), [&](const molo::llm::SseEvent& e) {
|
||||
text += e.data.find("token")->find("text")->as_string();
|
||||
return true;
|
||||
});
|
||||
ASSERT_EQ(text, std::string("ab"));
|
||||
ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("Bearer hf_tok"));
|
||||
}
|
||||
|
||||
TEST(huggingface_tei, embed_rerank_and_validation) {
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", ""));
|
||||
t->expect_response(Method::kPost, "/embed", reply("[[0.1,0.2]]"));
|
||||
Json opts = Json::object();
|
||||
opts.set("normalize", Json(true));
|
||||
tei.embed(Json("hello"), opts);
|
||||
ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"hello\",\"normalize\":true}"));
|
||||
t->expect_response(Method::kPost, "/rerank", reply("[{\"index\":0,\"score\":0.9}]"));
|
||||
tei.rerank("q", {"a", "b"}, true);
|
||||
ASSERT_EQ(t->seen().back().body, std::string("{\"query\":\"q\",\"return_text\":true,\"texts\":[\"a\",\"b\"]}"));
|
||||
ASSERT_THROW(tei.embed(Json(""), Json()), ApiError);
|
||||
ASSERT_THROW(tei.embed(Json(7), Json()), ApiError);
|
||||
ASSERT_THROW(tei.rerank("q", {}), ApiError);
|
||||
Json many = Json::array();
|
||||
for (int i = 0; i < 257; ++i) many.push(Json("x"));
|
||||
ASSERT_THROW(tei.embed(many, Json()), ApiError);
|
||||
Json unknown = Json::object();
|
||||
unknown.set("hack", Json(1));
|
||||
ASSERT_THROW(tei.embed(Json("x"), unknown), ApiError);
|
||||
ASSERT_THROW(tei.decode({}, true), ApiError);
|
||||
ASSERT_THROW(tei.openai_embeddings(Json("x"), "", -1), ApiError);
|
||||
}
|
||||
|
||||
TEST(huggingface_servers, call_from_the_wrong_surface_is_refused) {
|
||||
auto mock = std::make_unique<MockTransport>();
|
||||
MockTransport* t = mock.get();
|
||||
TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", ""));
|
||||
ASSERT_EQ(t->send_count(), static_cast<std::size_t>(0));
|
||||
ASSERT_TRUE(tei.surface() == Surface::kTei);
|
||||
}
|
||||
Reference in New Issue
Block a user