Files
moloapp/src/vendors/huggingface/chat_model.h
T

41 lines
1.6 KiB
C++

#pragma once
// HuggingfaceChatModel: the llm::ChatModel seam over any Hugging Face chat
// endpoint that speaks the OpenAI-compatible chat wire: the Inference
// Providers router (default), hf-inference, or a TGI server. The mapping rules
// are in chat_wire.h. The default model must be given by the caller: the
// router has no default (model ids change; GET /v1/models lists them).
#include <memory>
#include <string>
#include "vendors/huggingface/chat_wire.h"
#include "vendors/huggingface/platform.h"
#include "llm/chat_types.h"
namespace molo::huggingface {
class HuggingfaceChatModel : public llm::ChatModel {
public:
HuggingfaceChatModel(std::shared_ptr<ChatEndpoint> endpoint, std::string default_model);
// Router with HF_TOKEN from the environment.
static std::shared_ptr<HuggingfaceChatModel> from_env(const std::string& default_model);
// Router over an explicit platform (tests).
static std::shared_ptr<HuggingfaceChatModel> over_router(const Platform& platform,
const std::string& default_model);
std::string vendor() const override { return "huggingface"; }
llm::ChatResponse generate(const llm::ChatRequest& request) override;
void stream(const llm::ChatRequest& request,
const std::function<bool(const llm::ChatChunk&)>& on_chunk) override;
// The wire body a neutral request maps to (tests, dry runs).
clients::body::Json to_body(const llm::ChatRequest& request, bool stream) const;
private:
std::shared_ptr<ChatEndpoint> endpoint_;
std::string default_model_;
};
} // namespace molo::huggingface