#pragma once // Text Generation Inference (TGI) and Text Embeddings Inference (TEI) server // APIs. Both run anywhere: an Inference Endpoint URL, a container on // localhost, a Space. The base URL is given by the caller (Platform::server // validates it); the token is optional because self-hosted servers usually // have none. Request shapes follow the servers' own OpenAPI documents. #include #include #include "vendors/huggingface/chat_wire.h" #include "vendors/huggingface/hf_service.h" namespace molo::huggingface { class TgiServer : public HfService, public ChatEndpoint { public: explicit TgiServer(Access access) : HfService(std::move(access), Surface::kTgi) {} bool ready(); // GET /health: true on 2xx, false on 503, throws otherwise Json info(); // GET /info std::string metrics(); // GET /metrics (Prometheus text) Json models(); // GET /v1/models // Native API. Json generate(const std::string& inputs, const Json& parameters = Json()); // Streams token events; sink returns false to stop. void generate_stream(const std::string& inputs, const Json& parameters, const llm::SseDecoder::Sink& sink); Json tokenize(const std::string& inputs, const Json& parameters = Json()); Json chat_tokenize(const Json& chat_body); // OpenAI-compatible API. Json create(const Json& body) override; // POST /v1/chat/completions void stream(const Json& body, const llm::SseDecoder::Sink& sink) override; Json completions(const Json& body); // POST /v1/completions Json invocations(const Json& body); // POST /invocations (SageMaker compatibility) }; class TeiServer : public HfService { public: explicit TeiServer(Access access) : HfService(std::move(access), Surface::kTei) {} bool ready(); Json info(); std::string metrics(); // inputs: one string or an array of strings (1..HfCaps::kMaxBatchInputs). Json embed(const Json& inputs, const Json& options = Json()); Json embed_all(const Json& inputs, const Json& options = Json()); Json embed_sparse(const Json& inputs, const Json& options = Json()); Json predict(const Json& inputs, const Json& options = Json()); Json rerank(const std::string& query, const std::vector& texts, bool return_text = false, bool raw_scores = false); Json similarity(const Json& inputs, const Json& parameters = Json()); Json tokenize(const Json& inputs, const Json& options = Json()); Json decode(const std::vector& ids, bool skip_special_tokens = true); Json openai_embeddings(const Json& input, const std::string& model = "", int dimensions = 0); // Counts and size of an `inputs` value: refuses anything over the caps. static void check_inputs(const Json& inputs); }; } // namespace molo::huggingface