65 lines
2.7 KiB
C++
65 lines
2.7 KiB
C++
#pragma once
|
|
|
|
// Text Generation Inference (TGI) and Text Embeddings Inference (TEI) server
|
|
// APIs. Both run anywhere: an Inference Endpoint URL, a container on
|
|
// localhost, a Space. The base URL is given by the caller (Platform::server
|
|
// validates it); the token is optional because self-hosted servers usually
|
|
// have none. Request shapes follow the servers' own OpenAPI documents.
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "vendors/huggingface/chat_wire.h"
|
|
#include "vendors/huggingface/hf_service.h"
|
|
|
|
namespace molo::huggingface {
|
|
|
|
class TgiServer : public HfService, public ChatEndpoint {
|
|
public:
|
|
explicit TgiServer(Access access) : HfService(std::move(access), Surface::kTgi) {}
|
|
|
|
bool ready(); // GET /health: true on 2xx, false on 503, throws otherwise
|
|
Json info(); // GET /info
|
|
std::string metrics(); // GET /metrics (Prometheus text)
|
|
Json models(); // GET /v1/models
|
|
|
|
// Native API.
|
|
Json generate(const std::string& inputs, const Json& parameters = Json());
|
|
// Streams token events; sink returns false to stop.
|
|
void generate_stream(const std::string& inputs, const Json& parameters, const llm::SseDecoder::Sink& sink);
|
|
Json tokenize(const std::string& inputs, const Json& parameters = Json());
|
|
Json chat_tokenize(const Json& chat_body);
|
|
|
|
// OpenAI-compatible API.
|
|
Json create(const Json& body) override; // POST /v1/chat/completions
|
|
void stream(const Json& body, const llm::SseDecoder::Sink& sink) override;
|
|
Json completions(const Json& body); // POST /v1/completions
|
|
Json invocations(const Json& body); // POST /invocations (SageMaker compatibility)
|
|
};
|
|
|
|
class TeiServer : public HfService {
|
|
public:
|
|
explicit TeiServer(Access access) : HfService(std::move(access), Surface::kTei) {}
|
|
|
|
bool ready();
|
|
Json info();
|
|
std::string metrics();
|
|
|
|
// inputs: one string or an array of strings (1..HfCaps::kMaxBatchInputs).
|
|
Json embed(const Json& inputs, const Json& options = Json());
|
|
Json embed_all(const Json& inputs, const Json& options = Json());
|
|
Json embed_sparse(const Json& inputs, const Json& options = Json());
|
|
Json predict(const Json& inputs, const Json& options = Json());
|
|
Json rerank(const std::string& query, const std::vector<std::string>& texts, bool return_text = false,
|
|
bool raw_scores = false);
|
|
Json similarity(const Json& inputs, const Json& parameters = Json());
|
|
Json tokenize(const Json& inputs, const Json& options = Json());
|
|
Json decode(const std::vector<long>& ids, bool skip_special_tokens = true);
|
|
Json openai_embeddings(const Json& input, const std::string& model = "", int dimensions = 0);
|
|
|
|
// Counts and size of an `inputs` value: refuses anything over the caps.
|
|
static void check_inputs(const Json& inputs);
|
|
};
|
|
|
|
} // namespace molo::huggingface
|