84 lines
3.2 KiB
C++
84 lines
3.2 KiB
C++
#pragma once
|
|
|
|
// Wiring for the Hugging Face hosts and one shared request budget.
|
|
//
|
|
// One credential, HF_TOKEN (the legacy name HUGGING_FACE_HUB_TOKEN is also
|
|
// read), through llm::Config. The Hub serves public reads without any token,
|
|
// but llm::ApiClient cannot send an unauthenticated request. Workaround kept
|
|
// inside this package: without a token the clients are built with a harmless
|
|
// marker header instead of Authorization (reported in SPEC.md ## Gaps).
|
|
|
|
#include <functional>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <string>
|
|
|
|
#include "clients/transport.h"
|
|
#include "llm/api_client.h"
|
|
|
|
namespace molo::huggingface {
|
|
|
|
// Fixed-window call counter. Local ceiling so one process never trips the
|
|
// server limit; the clock is injectable for tests.
|
|
class RequestBudget {
|
|
public:
|
|
using Clock = std::function<long()>; // monotonic seconds
|
|
RequestBudget(int max_calls, int window_seconds, Clock clock = nullptr);
|
|
// Throws ApiError 429 LOCAL_BUDGET when the window is spent.
|
|
void spend();
|
|
int used() const;
|
|
int max_calls() const { return max_calls_; }
|
|
|
|
private:
|
|
mutable std::mutex mutex_;
|
|
int max_calls_;
|
|
int window_;
|
|
Clock clock_;
|
|
long window_start_ = 0;
|
|
int used_ = 0;
|
|
};
|
|
|
|
// What a service needs: a client, whether a token backs it, the shared budget.
|
|
struct Access {
|
|
std::shared_ptr<llm::ApiClient> client;
|
|
bool has_token = false;
|
|
std::shared_ptr<RequestBudget> budget;
|
|
};
|
|
|
|
struct Platform {
|
|
static constexpr const char* kHubHost = "https://huggingface.co";
|
|
static constexpr const char* kRouterHost = "https://router.huggingface.co";
|
|
static constexpr const char* kEndpointsHost = "https://api.endpoints.huggingface.cloud";
|
|
static constexpr const char* kKeyEnv = "HF_TOKEN";
|
|
static constexpr const char* kLegacyKeyEnv = "HUGGING_FACE_HUB_TOKEN";
|
|
|
|
Access hub;
|
|
Access router;
|
|
Access endpoints;
|
|
|
|
// HF_TOKEN, else HUGGING_FACE_HUB_TOKEN, else empty.
|
|
static std::string token_from_env();
|
|
// Real HTTPS transports. A missing token still builds working anonymous
|
|
// clients (public Hub reads); calls that need a token fail NOT_CONFIGURED.
|
|
static Platform from_env();
|
|
static Platform with(std::unique_ptr<clients::Transport> hub_transport,
|
|
std::unique_ptr<clients::Transport> router_transport,
|
|
std::unique_ptr<clients::Transport> endpoints_transport,
|
|
const std::string& token,
|
|
std::shared_ptr<RequestBudget> budget = nullptr);
|
|
bool configured() const { return hub.has_token; }
|
|
|
|
// A TGI / TEI server (Inference Endpoint URL, localhost, any host). The token
|
|
// is optional: self-hosted servers usually have none.
|
|
static Access server(const std::string& base_url, const std::string& token,
|
|
std::shared_ptr<RequestBudget> budget = nullptr);
|
|
static Access server_with(std::unique_ptr<clients::Transport> transport, const std::string& base_url,
|
|
const std::string& token, std::shared_ptr<RequestBudget> budget = nullptr);
|
|
// http(s) URL with a host, no userinfo, query or fragment; trailing / removed.
|
|
static std::string check_server_url(const std::string& url);
|
|
|
|
static llm::ApiClientConfig config(const std::string& base_url, bool has_token);
|
|
};
|
|
|
|
} // namespace molo::huggingface
|