64 lines
2.4 KiB
C++
64 lines
2.4 KiB
C++
#pragma once
|
|||
|
|
|
||
|
|
// OpenrouterChatModel: llm::ChatModel over POST /chat/completions. Maps the
|
||
|
|
// neutral request (messages, tools, structured output, reasoning effort) to
|
||
|
|
// the OpenRouter wire format and the response back, and carries the routing
|
||
|
|
// defaults (provider preferences, fallback models, plugins, reasoning) that
|
||
|
|
// every request of this model object should use.
|
||
|
|
//
|
||
|
|
// Mapping notes:
|
||
|
|
// - `ChatRequest::extra` is merged last and may only hold fields the table
|
||
|
|
// declares for chat completions; it overrides typed values.
|
||
|
|
// - `ChatRequest::reasoning_effort` becomes the `reasoning` object
|
||
|
|
// {"effort": ...} (overriding the default reasoning); the model's
|
||
|
|
// reasoning text lands in ContentPart::reasoning.
|
||
|
|
// - `reasoning_details` (needed to continue a tool loop with reasoning
|
||
|
|
// models) are kept in ChatResponse::raw and cannot be replayed through the
|
||
|
|
// neutral types; use ChatService with raw messages for that.
|
||
|
|
// - A provider error mid-stream is thrown after the chunks before it.
|
||
|
|
|
||
|
|
#include <functional>
|
||
|
|
#include <memory>
|
||
|
|
#include <optional>
|
||
|
|
#include <string>
|
||
|
|
#include <vector>
|
||
|
|
|
||
|
|
#include "llm/chat_types.h"
|
||
|
|
#include "openrouter/chat_service.h"
|
||
|
|
|
||
|
|
namespace molo::openrouter {
|
||
|
|
|
||
|
|
struct ChatModelDefaults {
|
||
|
|
std::optional<ProviderPreferences> provider;
|
||
|
|
std::vector<std::string> fallback_models; // sent as `models`
|
||
|
|
std::vector<PluginPtr> plugins;
|
||
|
|
std::optional<ReasoningConfig> reasoning;
|
||
|
|
std::string route; // "", "fallback", "sort"
|
||
|
|
};
|
||
|
|
|
||
|
|
class OpenrouterChatModel : public llm::ChatModel {
|
||
|
|
public:
|
||
|
|
OpenrouterChatModel(std::shared_ptr<Gateway> gateway, std::string default_model,
|
||
|
|
ChatModelDefaults defaults = {});
|
||
|
|
|
||
|
|
std::string vendor() const override { return "openrouter"; }
|
||
|
|
llm::ChatResponse generate(const llm::ChatRequest& request) override;
|
||
|
|
void stream(const llm::ChatRequest& request,
|
||
|
|
const std::function<bool(const llm::ChatChunk&)>& on_chunk) override;
|
||
|
|
|
||
|
|
// The request body for a neutral request (validated; stream flag not set).
|
||
|
|
clients::body::Json to_body(const llm::ChatRequest& request) const;
|
||
|
|
// Neutral view of a parsed result.
|
||
|
|
static llm::ChatResponse to_response(const ChatResult& result);
|
||
|
|
|
||
|
|
const std::string& default_model() const { return default_model_; }
|
||
|
|
const ChatService& service() const { return service_; }
|
||
|
|
|
||
|
|
private:
|
||
|
|
ChatService service_;
|
||
|
|
std::string default_model_;
|
||
|
|
ChatModelDefaults defaults_;
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace molo::openrouter
|