Files
moloapp/src/vendors/huggingface/servers.h
T

65 lines
2.7 KiB
C++

#pragma once
// Text Generation Inference (TGI) and Text Embeddings Inference (TEI) server
// APIs. Both run anywhere: an Inference Endpoint URL, a container on
// localhost, a Space. The base URL is given by the caller (Platform::server
// validates it); the token is optional because self-hosted servers usually
// have none. Request shapes follow the servers' own OpenAPI documents.
#include <string>
#include <vector>
#include "vendors/huggingface/chat_wire.h"
#include "vendors/huggingface/hf_service.h"
namespace molo::huggingface {
class TgiServer : public HfService, public ChatEndpoint {
public:
explicit TgiServer(Access access) : HfService(std::move(access), Surface::kTgi) {}
bool ready(); // GET /health: true on 2xx, false on 503, throws otherwise
Json info(); // GET /info
std::string metrics(); // GET /metrics (Prometheus text)
Json models(); // GET /v1/models
// Native API.
Json generate(const std::string& inputs, const Json& parameters = Json());
// Streams token events; sink returns false to stop.
void generate_stream(const std::string& inputs, const Json& parameters, const llm::SseDecoder::Sink& sink);
Json tokenize(const std::string& inputs, const Json& parameters = Json());
Json chat_tokenize(const Json& chat_body);
// OpenAI-compatible API.
Json create(const Json& body) override; // POST /v1/chat/completions
void stream(const Json& body, const llm::SseDecoder::Sink& sink) override;
Json completions(const Json& body); // POST /v1/completions
Json invocations(const Json& body); // POST /invocations (SageMaker compatibility)
};
class TeiServer : public HfService {
public:
explicit TeiServer(Access access) : HfService(std::move(access), Surface::kTei) {}
bool ready();
Json info();
std::string metrics();
// inputs: one string or an array of strings (1..HfCaps::kMaxBatchInputs).
Json embed(const Json& inputs, const Json& options = Json());
Json embed_all(const Json& inputs, const Json& options = Json());
Json embed_sparse(const Json& inputs, const Json& options = Json());
Json predict(const Json& inputs, const Json& options = Json());
Json rerank(const std::string& query, const std::vector<std::string>& texts, bool return_text = false,
bool raw_scores = false);
Json similarity(const Json& inputs, const Json& parameters = Json());
Json tokenize(const Json& inputs, const Json& options = Json());
Json decode(const std::vector<long>& ids, bool skip_special_tokens = true);
Json openai_embeddings(const Json& input, const std::string& model = "", int dimensions = 0);
// Counts and size of an `inputs` value: refuses anything over the caps.
static void check_inputs(const Json& inputs);
};
} // namespace molo::huggingface