// Inference Endpoints management, TGI and TEI. Requests are checked against // the call table; replies are SHAPE or RECORDED. #include "support.h" #include "vendors/huggingface/endpoints.h" #include "vendors/huggingface/servers.h" using namespace molo::huggingface; using namespace hf_test; using molo::llm::ApiError; namespace { // RECORDED 2026-10-25: GET https://api.endpoints.huggingface.cloud/v2/provider (no token), trimmed const char* kVendors = "{\"vendors\":[{\"name\":\"aws\",\"status\":\"available\",\"regions\":[{\"name\":\"eu-west-1\",\"label\":\"Ireland\"," "\"status\":\"available\",\"computes\":[{\"id\":\"aws-eu-west-1-intel-spr-x1\",\"accelerator\":\"cpu\"," "\"instanceType\":\"intel-spr\",\"instanceSize\":\"x1\",\"status\":\"available\",\"pricePerHour\":0.033}]}]}]}"; // RECORDED 2026-10-25: any call without a valid token -> 401 const char* kIeUnauth = "{\"error\":\"401 Unauthorized\",\"code\":\"UNAUTHORIZED\"}"; Json spec() { return Json::parse( "{\"name\":\"my-ep\",\"type\":\"protected\",\"provider\":{\"vendor\":\"aws\",\"region\":\"us-east-1\"}," "\"compute\":{},\"model\":{}}"); } } // namespace TEST(huggingface_endpoints, vendors_list_is_public_everything_else_needs_a_token) { Rig rig(""); Endpoints ie = Endpoints::from_platform(rig.platform); rig.endpoints->expect_response(Method::kGet, "/v2/provider", reply(kVendors)); Json v = ie.vendors(); ASSERT_EQ(v.find("vendors")->as_array().front().find("name")->as_string(), std::string("aws")); try { ie.list("acme"); FAIL("expected NOT_CONFIGURED"); } catch (const ApiError& e) { ASSERT_EQ(e.type(), std::string("NOT_CONFIGURED")); } ASSERT_EQ(rig.endpoints->send_count(), static_cast(1)); } TEST(huggingface_endpoints, error_code_is_kept) { Rig rig; Endpoints ie = Endpoints::from_platform(rig.platform); rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/huggingface", reply(kIeUnauth, 401)); try { ie.list("huggingface"); FAIL("expected 401"); } catch (const ApiError& e) { ASSERT_EQ(e.http_status(), 401); ASSERT_TRUE(has(e.message(), "401 Unauthorized")); } } TEST(huggingface_endpoints, reads_build_the_documented_paths) { Rig rig; Endpoints ie = Endpoints::from_platform(rig.platform); rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme", reply("{\"items\":[]}")); rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep", reply("{}")); rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/replica", reply("{}")); rig.endpoints->expect_response(Method::kGet, "/v2/endpoint/acme/ep/logs", reply("line1\nline2\n", 200, "text/plain")); rig.endpoints->expect_response(Method::kGet, "/v2/search/acme/states", reply("{}")); EndpointListParams p; p.limit = 5; p.state = "running"; ie.list("acme", p); ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme?state=running&limit=5")); ie.get("acme", "ep"); ie.replicas("acme", "ep"); ASSERT_EQ(ie.logs("acme", "ep", 50), std::string("line1\nline2\n")); ASSERT_EQ(rig.endpoints->seen().back().target, std::string("/v2/endpoint/acme/ep/logs?tail=50")); ie.state_counts("acme"); ASSERT_THROW(ie.get("acme", "../x"), ApiError); ASSERT_THROW(ie.list("acme", EndpointListParams{"", "", "", "", "", "", 101}), ApiError); } TEST(huggingface_endpoints, metrics_use_post_with_a_time_window) { Rig rig; Endpoints ie = Endpoints::from_platform(rig.platform); rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/metrics/inference/ttft", reply("{}")); ie.metrics(MetricKind::kTtft, "acme", "ep", "2021-01-01T00:00:00Z", "2021-01-02T00:00:00Z", "r1"); ASSERT_EQ(rig.endpoints->seen().back().body, std::string("{\"replicaId\":\"r1\",\"start\":\"2021-01-01T00:00:00Z\",\"stop\":\"2021-01-02T00:00:00Z\"}")); ASSERT_THROW(ie.metrics(MetricKind::kTtft, "acme", "ep", "yesterday", "today"), ApiError); MetricKind k; ASSERT_TRUE(parse_metric_kind("kv-cache", k)); ASSERT_FALSE(parse_metric_kind("nope", k)); } TEST(huggingface_endpoints, writes_are_validated_and_explicit) { Rig rig; Endpoints ie = Endpoints::from_platform(rig.platform); ASSERT_THROW(ie.create("acme", spec()), ApiError); // type "protected" is not a type Json good = spec(); good.set("type", Json("private")); rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme", reply("{}")); ie.create("acme", good); Json bad_name = good; bad_name.set("name", Json("My_Endpoint")); ASSERT_THROW(ie.create("acme", bad_name), ApiError); Json missing = Json::object(); missing.set("name", Json("x")); ASSERT_THROW(ie.create("acme", missing), ApiError); rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/pause", reply("{}")); rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/resume", reply("{}")); rig.endpoints->expect_response(Method::kPost, "/v2/endpoint/acme/ep/scale-to-zero", reply("{}")); rig.endpoints->expect_response(Method::kDelete, "/v2/endpoint/acme/ep", reply("{}")); ie.pause("acme", "ep"); ie.resume("acme", "ep"); ie.scale_to_zero("acme", "ep"); ie.remove_endpoint("acme", "ep"); rig.endpoints->verify(); } TEST(huggingface_endpoints, sse_streams_are_listed_but_refused) { Rig rig; Endpoints ie = Endpoints::from_platform(rig.platform); ASSERT_EQ(rig.endpoints->send_count(), static_cast(0)); for (std::size_t i = 0; i < Calls::count(); ++i) if (Calls::at(i).has_flag("unsupported")) ASSERT_TRUE(std::string(Calls::at(i).path).find("sse") != std::string::npos || std::string(Calls::at(i).path).find("/sse") != std::string::npos); } TEST(huggingface_tgi, native_and_openai_routes) { Rig rig; // transports unused: the server has its own auto mock = std::make_unique(); MockTransport* t = mock.get(); TgiServer tgi(Platform::server_with(std::move(mock), "http://127.0.0.1:8080/", "")); ASSERT_FALSE(tgi.has_token()); t->expect_response(Method::kGet, "/health", reply("", 200, "text/plain")); MockExpectation starting; // the client retries a 503 GET, so allow it to answer more than once starting.method = Method::kGet; starting.path = "/health"; starting.response = reply("starting", 503, "text/plain"); starting.times = 5; t->expect(starting); ASSERT_TRUE(tgi.ready()); ASSERT_FALSE(tgi.ready()); t->expect_response(Method::kPost, "/generate", reply("{\"generated_text\":\"hi\"}")); Json params = Json::object(); params.set("max_new_tokens", Json(8)); tgi.generate("Hello", params); ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"Hello\",\"parameters\":{\"max_new_tokens\":8}}")); ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("")); ASSERT_THROW(tgi.generate("", params), ApiError); Json bad = Json::object(); bad.set("max_new_tokens", Json(8)); ASSERT_THROW(tgi.completions(bad), ApiError); // body needs a table field (prompt is optional, max_new_tokens is not a field) } TEST(huggingface_tgi, generate_stream_decodes_token_events) { auto mock = std::make_unique(); MockTransport* t = mock.get(); TgiServer tgi(Platform::server_with(std::move(mock), "http://tgi.local:8080", "hf_tok")); ASSERT_TRUE(tgi.has_token()); // SHAPE: TGI streams one StreamResponse per `data:` line t->expect_response(Method::kPost, "/generate_stream", sse("data:{\"index\":1,\"token\":{\"id\":1,\"text\":\"a\",\"logprob\":-0.1,\"special\":false},\"generated_text\":null}\n\n" "data:{\"index\":2,\"token\":{\"id\":2,\"text\":\"b\",\"logprob\":-0.2,\"special\":false},\"generated_text\":\"ab\"}\n\n")); std::string text; tgi.generate_stream("x", Json(), [&](const molo::llm::SseEvent& e) { text += e.data.find("token")->find("text")->as_string(); return true; }); ASSERT_EQ(text, std::string("ab")); ASSERT_EQ(Rig::header(t->seen().back(), "Authorization"), std::string("Bearer hf_tok")); } TEST(huggingface_tei, embed_rerank_and_validation) { auto mock = std::make_unique(); MockTransport* t = mock.get(); TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", "")); t->expect_response(Method::kPost, "/embed", reply("[[0.1,0.2]]")); Json opts = Json::object(); opts.set("normalize", Json(true)); tei.embed(Json("hello"), opts); ASSERT_EQ(t->seen().back().body, std::string("{\"inputs\":\"hello\",\"normalize\":true}")); t->expect_response(Method::kPost, "/rerank", reply("[{\"index\":0,\"score\":0.9}]")); tei.rerank("q", {"a", "b"}, true); ASSERT_EQ(t->seen().back().body, std::string("{\"query\":\"q\",\"return_text\":true,\"texts\":[\"a\",\"b\"]}")); ASSERT_THROW(tei.embed(Json(""), Json()), ApiError); ASSERT_THROW(tei.embed(Json(7), Json()), ApiError); ASSERT_THROW(tei.rerank("q", {}), ApiError); Json many = Json::array(); for (int i = 0; i < 257; ++i) many.push(Json("x")); ASSERT_THROW(tei.embed(many, Json()), ApiError); Json unknown = Json::object(); unknown.set("hack", Json(1)); ASSERT_THROW(tei.embed(Json("x"), unknown), ApiError); ASSERT_THROW(tei.decode({}, true), ApiError); ASSERT_THROW(tei.openai_embeddings(Json("x"), "", -1), ApiError); } TEST(huggingface_servers, call_from_the_wrong_surface_is_refused) { auto mock = std::make_unique(); MockTransport* t = mock.get(); TeiServer tei(Platform::server_with(std::move(mock), "http://127.0.0.1:8081", "")); ASSERT_EQ(t->send_count(), static_cast(0)); ASSERT_TRUE(tei.surface() == Surface::kTei); }