Files
moloapp/src/vendors/huggingface/platform.h
T

84 lines
3.2 KiB
C++

#pragma once
// Wiring for the Hugging Face hosts and one shared request budget.
//
// One credential, HF_TOKEN (the legacy name HUGGING_FACE_HUB_TOKEN is also
// read), through llm::Config. The Hub serves public reads without any token,
// but llm::ApiClient cannot send an unauthenticated request. Workaround kept
// inside this package: without a token the clients are built with a harmless
// marker header instead of Authorization (reported in SPEC.md ## Gaps).
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include "clients/transport.h"
#include "llm/api_client.h"
namespace molo::huggingface {
// Fixed-window call counter. Local ceiling so one process never trips the
// server limit; the clock is injectable for tests.
class RequestBudget {
public:
using Clock = std::function<long()>; // monotonic seconds
RequestBudget(int max_calls, int window_seconds, Clock clock = nullptr);
// Throws ApiError 429 LOCAL_BUDGET when the window is spent.
void spend();
int used() const;
int max_calls() const { return max_calls_; }
private:
mutable std::mutex mutex_;
int max_calls_;
int window_;
Clock clock_;
long window_start_ = 0;
int used_ = 0;
};
// What a service needs: a client, whether a token backs it, the shared budget.
struct Access {
std::shared_ptr<llm::ApiClient> client;
bool has_token = false;
std::shared_ptr<RequestBudget> budget;
};
struct Platform {
static constexpr const char* kHubHost = "https://huggingface.co";
static constexpr const char* kRouterHost = "https://router.huggingface.co";
static constexpr const char* kEndpointsHost = "https://api.endpoints.huggingface.cloud";
static constexpr const char* kKeyEnv = "HF_TOKEN";
static constexpr const char* kLegacyKeyEnv = "HUGGING_FACE_HUB_TOKEN";
Access hub;
Access router;
Access endpoints;
// HF_TOKEN, else HUGGING_FACE_HUB_TOKEN, else empty.
static std::string token_from_env();
// Real HTTPS transports. A missing token still builds working anonymous
// clients (public Hub reads); calls that need a token fail NOT_CONFIGURED.
static Platform from_env();
static Platform with(std::unique_ptr<clients::Transport> hub_transport,
std::unique_ptr<clients::Transport> router_transport,
std::unique_ptr<clients::Transport> endpoints_transport,
const std::string& token,
std::shared_ptr<RequestBudget> budget = nullptr);
bool configured() const { return hub.has_token; }
// A TGI / TEI server (Inference Endpoint URL, localhost, any host). The token
// is optional: self-hosted servers usually have none.
static Access server(const std::string& base_url, const std::string& token,
std::shared_ptr<RequestBudget> budget = nullptr);
static Access server_with(std::unique_ptr<clients::Transport> transport, const std::string& base_url,
const std::string& token, std::shared_ptr<RequestBudget> budget = nullptr);
// http(s) URL with a host, no userinfo, query or fragment; trailing / removed.
static std::string check_server_url(const std::string& url);
static llm::ApiClientConfig config(const std::string& base_url, bool has_token);
};
} // namespace molo::huggingface