58 lines
2.0 KiB
C++
58 lines
2.0 KiB
C++
#pragma once
|
|
|
|
// AzureChatModel: llm::ChatModel over a deployment of an Azure resource.
|
|
// The request's `model` is the deployment name. Three dialects:
|
|
// kChatCompletions /chat/completions (default; the widest model support)
|
|
// kResponses /responses
|
|
// kFoundry Azure AI model inference /models/chat/completions
|
|
// The neutral <-> wire mapping is the OpenAI one (openai::core::ChatWire /
|
|
// ResponsesWire): Azure OpenAI speaks the same shapes.
|
|
|
|
#include <functional>
|
|
#include <memory>
|
|
#include <string>
|
|
|
|
#include "llm/chat_types.h"
|
|
#include "openai/core/wire/chat_wire.h"
|
|
#include "vendors/azure/chat.h"
|
|
#include "vendors/azure/foundry.h"
|
|
#include "vendors/azure/responses.h"
|
|
|
|
namespace molo::azure {
|
|
|
|
enum class Dialect { kChatCompletions, kResponses, kFoundry };
|
|
|
|
struct ChatModelConfig {
|
|
Dialect dialect = Dialect::kChatCompletions;
|
|
openai::core::WireOptions wire;
|
|
std::string extra_parameters; // Foundry dialect: error, drop, pass-through
|
|
};
|
|
|
|
class AzureChatModel : public llm::ChatModel {
|
|
public:
|
|
explicit AzureChatModel(std::shared_ptr<AzureClient> azure, ChatModelConfig config = {});
|
|
// Settings from the environment; not configured without endpoint and credential.
|
|
static std::shared_ptr<AzureChatModel> from_env(ChatModelConfig config = {});
|
|
|
|
std::string vendor() const override { return "azure"; }
|
|
llm::ChatResponse generate(const llm::ChatRequest& request) override;
|
|
void stream(const llm::ChatRequest& request,
|
|
const std::function<bool(const llm::ChatChunk&)>& on_chunk) override;
|
|
|
|
Dialect dialect() const { return config_.dialect; }
|
|
bool configured() const { return azure_->configured(); }
|
|
// The wire body generate() would send for a request (no traffic).
|
|
clients::body::Json wire_body(const llm::ChatRequest& request, bool stream) const;
|
|
|
|
private:
|
|
void check(const llm::ChatRequest& request) const;
|
|
|
|
std::shared_ptr<AzureClient> azure_;
|
|
ChatModelConfig config_;
|
|
ChatService chat_;
|
|
ResponsesService responses_;
|
|
FoundryService foundry_;
|
|
};
|
|
|
|
} // namespace molo::azure
|