feat(clients): llm vendor foundation: api client, SSE, errors, neutral chat types, ai.mk, example stubs
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -36,6 +36,7 @@ clean::
|
|||||||
-include src/supervisor/supervisor.mk
|
-include src/supervisor/supervisor.mk
|
||||||
-include src/clients/clients.mk
|
-include src/clients/clients.mk
|
||||||
-include src/google/google.mk
|
-include src/google/google.mk
|
||||||
|
-include src/ai.mk
|
||||||
# NOTE: supplychain.mk and operability.mk precede cli.mk and supervisor.mk
|
# NOTE: supplychain.mk and operability.mk precede cli.mk and supervisor.mk
|
||||||
# so SUPPLY_LIB / OPERABILITY_LIB are defined when those add link edges
|
# so SUPPLY_LIB / OPERABILITY_LIB are defined when those add link edges
|
||||||
# (supply PATCHES section 2, operability PATCHES sections 5-6 and 10).
|
# (supply PATCHES section 2, operability PATCHES sections 5-6 and 10).
|
||||||
@@ -55,6 +56,7 @@ all:: app-build
|
|||||||
all:: $(REALTIME_LIB)
|
all:: $(REALTIME_LIB)
|
||||||
all:: clients
|
all:: clients
|
||||||
all:: google
|
all:: google
|
||||||
|
all:: ai
|
||||||
all:: $(MOLO_BIN)
|
all:: $(MOLO_BIN)
|
||||||
all:: tenancy-tests
|
all:: tenancy-tests
|
||||||
all:: audit-tests
|
all:: audit-tests
|
||||||
@@ -72,6 +74,7 @@ test:: app-test
|
|||||||
test:: realtime-test
|
test:: realtime-test
|
||||||
test:: clients-test
|
test:: clients-test
|
||||||
test:: google-test
|
test:: google-test
|
||||||
|
test:: ai-test
|
||||||
test:: cli-test
|
test:: cli-test
|
||||||
test:: tenancy-check
|
test:: tenancy-check
|
||||||
test:: audit-check
|
test:: audit-check
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// LLM vendor platform demos (xAI, Anthropic, Meta, DeepSeek). One register
|
||||||
|
// function per work package, each in its own file under example/app/ai/.
|
||||||
|
// Every route lives under /ai/<vendor> and is excluded from locust (paid
|
||||||
|
// APIs). Rules for a demo:
|
||||||
|
// - GET pages and status routes answer 2xx without credentials (the matrix
|
||||||
|
// test crawls them); a route that would call the vendor is POST or DELETE
|
||||||
|
// and answers 503 not_configured without a credential, 422 on bad input,
|
||||||
|
// and costly or destructive ones are dry runs unless the body has
|
||||||
|
// "confirm":true.
|
||||||
|
// - State lives in objects created inside the register function and
|
||||||
|
// captured by the handlers; do not edit Services.
|
||||||
|
// - Clients come from llm::ApiClient::from_env(...) (null key is fine: the
|
||||||
|
// vendor service then reports not configured).
|
||||||
|
|
||||||
|
#include "App.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
void register_ai_xai_demo(fw::Router& router, Services& services);
|
||||||
|
void register_ai_xaiplatform_demo(fw::Router& router, Services& services);
|
||||||
|
void register_ai_anthropic_demo(fw::Router& router, Services& services);
|
||||||
|
void register_ai_anthropicadmin_demo(fw::Router& router, Services& services);
|
||||||
|
void register_ai_meta_demo(fw::Router& router, Services& services);
|
||||||
|
void register_ai_deepseek_demo(fw::Router& router, Services& services);
|
||||||
|
|
||||||
|
// Mounts every vendor router at "" (called once from build_application).
|
||||||
|
void register_ai_demos(std::shared_ptr<fw::Application> app, Services& services);
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
#include "App.hpp"
|
#include "App.hpp"
|
||||||
#include "AgentDemos.hpp"
|
#include "AgentDemos.hpp"
|
||||||
|
#include "AiDemos.hpp"
|
||||||
|
|
||||||
namespace molo::example::app {
|
namespace molo::example::app {
|
||||||
|
|
||||||
@@ -73,6 +74,7 @@ std::shared_ptr<fw::Application> build_application(Services& services) {
|
|||||||
app->include_router("", voice);
|
app->include_router("", voice);
|
||||||
|
|
||||||
register_agent_demos(app, services);
|
register_agent_demos(app, services);
|
||||||
|
register_ai_demos(app, services);
|
||||||
|
|
||||||
register_chain_demo(app, services);
|
register_chain_demo(app, services);
|
||||||
return app;
|
return app;
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
void register_ai_demos(std::shared_ptr<fw::Application> app, Services& services) {
|
||||||
|
{
|
||||||
|
fw::Router r;
|
||||||
|
register_ai_xai_demo(r, services);
|
||||||
|
app->include_router("", r);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
fw::Router r;
|
||||||
|
register_ai_xaiplatform_demo(r, services);
|
||||||
|
app->include_router("", r);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
fw::Router r;
|
||||||
|
register_ai_anthropic_demo(r, services);
|
||||||
|
app->include_router("", r);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
fw::Router r;
|
||||||
|
register_ai_anthropicadmin_demo(r, services);
|
||||||
|
app->include_router("", r);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
fw::Router r;
|
||||||
|
register_ai_meta_demo(r, services);
|
||||||
|
app->include_router("", r);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
fw::Router r;
|
||||||
|
register_ai_deepseek_demo(r, services);
|
||||||
|
app->include_router("", r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
// Anthropic Claude: admin API (orgs, workspaces, keys, usage, cost) and managed agents. Stub: the module agent fills this in.
|
||||||
|
void register_ai_anthropicadmin_demo(fw::Router& router, Services& services) {
|
||||||
|
(void)router;
|
||||||
|
(void)services;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
// Anthropic Claude: messages, streaming, tools, batches, tokens, models, files, skills. Stub: the module agent fills this in.
|
||||||
|
void register_ai_anthropic_demo(fw::Router& router, Services& services) {
|
||||||
|
(void)router;
|
||||||
|
(void)services;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
// DeepSeek API. Stub: the module agent fills this in.
|
||||||
|
void register_ai_deepseek_demo(fw::Router& router, Services& services) {
|
||||||
|
(void)router;
|
||||||
|
(void)services;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
// Meta Llama API. Stub: the module agent fills this in.
|
||||||
|
void register_ai_meta_demo(fw::Router& router, Services& services) {
|
||||||
|
(void)router;
|
||||||
|
(void)services;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
// xAI Grok: chat, responses, tools, media, batch, models. Stub: the module agent fills this in.
|
||||||
|
void register_ai_xai_demo(fw::Router& router, Services& services) {
|
||||||
|
(void)router;
|
||||||
|
(void)services;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
#include "../AiDemos.hpp"
|
||||||
|
|
||||||
|
#include "../../fw/Router.hpp"
|
||||||
|
|
||||||
|
namespace molo::example::app {
|
||||||
|
|
||||||
|
// xAI Grok: voice/realtime, collections, files, management API. Stub: the module agent fills this in.
|
||||||
|
void register_ai_xaiplatform_demo(fw::Router& router, Services& services) {
|
||||||
|
(void)router;
|
||||||
|
(void)services;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::example::app
|
||||||
@@ -17,8 +17,8 @@ LOCUST = HERE / "locustfile.py"
|
|||||||
EXCLUDED = {"/voice/say", "/voice/hear", "/voice/turn", "/voice/reset"}
|
EXCLUDED = {"/voice/say", "/voice/hear", "/voice/turn", "/voice/reset"}
|
||||||
# Whole areas that talk to (or front) paid Google APIs. Routes under these
|
# Whole areas that talk to (or front) paid Google APIs. Routes under these
|
||||||
# prefixes are never load tested and never required to be covered.
|
# prefixes are never load tested and never required to be covered.
|
||||||
EXCLUDED_PREFIXES = ("/voice", "/google", "/agent", "/embedded/voice.js",
|
EXCLUDED_PREFIXES = ("/voice", "/google", "/agent", "/ai", "/embedded/voice.js",
|
||||||
"/embedded/agent_")
|
"/embedded/agent_", "/embedded/ai_")
|
||||||
|
|
||||||
|
|
||||||
def is_excluded(route: str) -> bool:
|
def is_excluded(route: str) -> bool:
|
||||||
|
|||||||
+4
-2
@@ -54,7 +54,8 @@ EXAMPLE_SRCS = \
|
|||||||
$(EXAMPLE_DIR)/app/EnterpriseDemo.cpp \
|
$(EXAMPLE_DIR)/app/EnterpriseDemo.cpp \
|
||||||
$(EXAMPLE_DIR)/app/ResilienceDemo.cpp \
|
$(EXAMPLE_DIR)/app/ResilienceDemo.cpp \
|
||||||
$(EXAMPLE_DIR)/app/VoiceDemo.cpp \
|
$(EXAMPLE_DIR)/app/VoiceDemo.cpp \
|
||||||
$(sort $(wildcard $(EXAMPLE_DIR)/app/agent/*.cpp))
|
$(sort $(wildcard $(EXAMPLE_DIR)/app/agent/*.cpp)) \
|
||||||
|
$(sort $(wildcard $(EXAMPLE_DIR)/app/ai/*.cpp))
|
||||||
|
|
||||||
# Tests are discovered: every tests/example/**/*_test.cpp is one route test.
|
# Tests are discovered: every tests/example/**/*_test.cpp is one route test.
|
||||||
EXAMPLE_TEST_JOBS ?= 8
|
EXAMPLE_TEST_JOBS ?= 8
|
||||||
@@ -71,7 +72,7 @@ EXAMPLE_ASSETS = $(sort $(wildcard $(EXAMPLE_DIR)/assets/*))
|
|||||||
EXAMPLE_INC = -I$(EXAMPLE_DIR) -I$(EXAMPLE_BUILD) -Isrc
|
EXAMPLE_INC = -I$(EXAMPLE_DIR) -I$(EXAMPLE_BUILD) -Isrc
|
||||||
# Real framework parts the example links: Google Cloud (src/google) over the
|
# Real framework parts the example links: Google Cloud (src/google) over the
|
||||||
# HTTP client (src/clients, wolfSSL TLS) and the operability breaker.
|
# HTTP client (src/clients, wolfSSL TLS) and the operability breaker.
|
||||||
EXAMPLE_REAL_LIBS = build/google/libgoogle.a build/clients/libclients.a \
|
EXAMPLE_REAL_LIBS = build/ai/libai.a build/google/libgoogle.a build/clients/libclients.a \
|
||||||
build/liboperability.a
|
build/liboperability.a
|
||||||
|
|
||||||
.PHONY: example-deploy example-verify deploy verify example example-real-libs example-assets example-test example-clean example-locust
|
.PHONY: example-deploy example-verify deploy verify example example-real-libs example-assets example-test example-clean example-locust
|
||||||
@@ -93,6 +94,7 @@ example-real-libs:
|
|||||||
$(MAKE) -f src/operability.mk build/liboperability.a
|
$(MAKE) -f src/operability.mk build/liboperability.a
|
||||||
$(MAKE) -f src/clients/clients.mk clients
|
$(MAKE) -f src/clients/clients.mk clients
|
||||||
$(MAKE) -f src/google/google.mk google
|
$(MAKE) -f src/google/google.mk google
|
||||||
|
$(MAKE) -f src/ai.mk ai
|
||||||
|
|
||||||
example: example-assets example-real-libs
|
example: example-assets example-real-libs
|
||||||
mkdir -p $(EXAMPLE_BUILD)/obj
|
mkdir -p $(EXAMPLE_BUILD)/obj
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ GETS = [
|
|||||||
"/webdav/files/test.txt",
|
"/webdav/files/test.txt",
|
||||||
]
|
]
|
||||||
|
|
||||||
# NOT load tested on purpose: everything under /voice, /google and /agent
|
# NOT load tested on purpose: everything under /voice, /google, /agent and /ai
|
||||||
# calls or fronts paid Google APIs (check_locust_coverage.py enforces this).
|
# calls or fronts paid Google APIs (check_locust_coverage.py enforces this).
|
||||||
POSTS = [
|
POSTS = [
|
||||||
("/audit/log", {"event": "locust"}),
|
("/audit/log", {"event": "locust"}),
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# Per-part fragment for the LLM vendor platforms (owned dirs: src/llm shared
|
||||||
|
# layer, src/xai, src/anthropic, src/meta, src/deepseek). Included by the root
|
||||||
|
# Makefile, or run standalone:
|
||||||
|
# make -f src/ai.mk [ai|ai-test|ai-clean]
|
||||||
|
# Root wiring: `all` depends on `ai`, `test` on `ai-test`, `clean` on `ai-clean`.
|
||||||
|
# xai realtime reuses the WebSocket client in src/google/live, so the test
|
||||||
|
# binary links libgoogle too.
|
||||||
|
|
||||||
|
ROOT ?= .
|
||||||
|
CXX ?= g++
|
||||||
|
AI_CXXFLAGS ?= -std=c++23 -Wall -Wextra -Werror -DMOLO_HAS_WOLFSSL
|
||||||
|
AI_INCS := -I$(ROOT)/src -I$(ROOT)/tests/src/clients
|
||||||
|
|
||||||
|
CLIENTS_LIB ?= $(ROOT)/build/clients/libclients.a
|
||||||
|
OPERABILITY_LIB ?= build/liboperability.a
|
||||||
|
GOOGLE_LIB ?= $(ROOT)/build/google/libgoogle.a
|
||||||
|
|
||||||
|
# Recursive: every vendor module builds without editing this file.
|
||||||
|
AI_SRCS := $(sort $(shell find $(ROOT)/src/llm $(ROOT)/src/xai $(ROOT)/src/anthropic \
|
||||||
|
$(ROOT)/src/meta $(ROOT)/src/deepseek -name "*.cpp")) \
|
||||||
|
$(ROOT)/src/security/crypto/Base64.cpp
|
||||||
|
AI_TEST_SRCS := $(sort $(shell find $(ROOT)/tests/src/llm $(ROOT)/tests/src/xai \
|
||||||
|
$(ROOT)/tests/src/anthropic $(ROOT)/tests/src/meta \
|
||||||
|
$(ROOT)/tests/src/deepseek -name "*.cpp"))
|
||||||
|
|
||||||
|
AI_BUILD := $(ROOT)/build/ai
|
||||||
|
AI_OBJDIR := $(AI_BUILD)/objs
|
||||||
|
AI_LIB := $(AI_BUILD)/libai.a
|
||||||
|
AI_TEST_BIN := $(ROOT)/build/ai_test
|
||||||
|
|
||||||
|
AI_OBJS := $(patsubst $(ROOT)/%.cpp,$(AI_OBJDIR)/%.o,$(AI_SRCS))
|
||||||
|
|
||||||
|
.PHONY: ai ai-test ai-clean
|
||||||
|
|
||||||
|
ai: $(AI_LIB)
|
||||||
|
|
||||||
|
$(AI_OBJDIR)/%.o: $(ROOT)/%.cpp
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
$(CXX) $(AI_CXXFLAGS) -MMD -MP -I$(ROOT)/src -c $< -o $@
|
||||||
|
|
||||||
|
$(AI_LIB): $(AI_OBJS)
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
$(AR) rcs $@ $^
|
||||||
|
|
||||||
|
-include $(AI_OBJS:.o=.d)
|
||||||
|
|
||||||
|
# The clients test harness (framework.h plus main.cpp) is reused, not copied.
|
||||||
|
ai-test: $(AI_TEST_BIN)
|
||||||
|
MOLO_ENV_FILES=off $(AI_TEST_BIN)
|
||||||
|
|
||||||
|
$(AI_TEST_BIN): $(AI_OBJS) $(AI_TEST_SRCS) $(CLIENTS_LIB) $(GOOGLE_LIB) $(OPERABILITY_LIB) $(ROOT)/tests/src/clients/main.cpp
|
||||||
|
@mkdir -p $(dir $@)
|
||||||
|
$(CXX) $(AI_CXXFLAGS) $(AI_INCS) $(AI_OBJS) $(AI_TEST_SRCS) $(ROOT)/tests/src/clients/main.cpp $(GOOGLE_LIB) $(CLIENTS_LIB) $(OPERABILITY_LIB) -lwolfssl -o $@
|
||||||
|
|
||||||
|
ai-clean:
|
||||||
|
rm -rf $(AI_BUILD) $(AI_TEST_BIN)
|
||||||
@@ -0,0 +1,176 @@
|
|||||||
|
#include "llm/api_client.h"
|
||||||
|
|
||||||
|
#include "clients/transports/http_transport.h"
|
||||||
|
#include "llm/caps.h"
|
||||||
|
#include "llm/config.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
using clients::body::Json;
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
clients::ClientConfig make_config() {
|
||||||
|
clients::ClientConfig c;
|
||||||
|
c.persist_cookies = false;
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
ApiClient::ApiClient(std::unique_ptr<clients::Transport> transport,
|
||||||
|
ApiClientConfig config, std::string key)
|
||||||
|
: config_(std::move(config)),
|
||||||
|
key_(std::move(key)),
|
||||||
|
client_(std::move(transport), make_config()) {}
|
||||||
|
|
||||||
|
std::shared_ptr<ApiClient> ApiClient::from_env(ApiClientConfig config) {
|
||||||
|
std::string key = Config::get(config.key_env.c_str());
|
||||||
|
return std::make_shared<ApiClient>(
|
||||||
|
std::make_unique<clients::transports::HttpTransport>(), std::move(config),
|
||||||
|
std::move(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
void ApiClient::guard() const {
|
||||||
|
if (!configured()) throw NotConfiguredError(config_.vendor, config_.key_env);
|
||||||
|
}
|
||||||
|
|
||||||
|
clients::ClientRequest ApiClient::build(clients::Method method, const std::string& path,
|
||||||
|
const Headers& extra) const {
|
||||||
|
if (path.empty() || path.front() != '/' || path.find("..") != std::string::npos)
|
||||||
|
throw ApiError::invalid(config_.vendor, "bad request path");
|
||||||
|
clients::ClientRequest req(method, clients::Url::parse(config_.base_url + path));
|
||||||
|
if (config_.auth == AuthStyle::kBearer) req.bearer(key_);
|
||||||
|
else req.header(config_.auth_header, key_);
|
||||||
|
for (const auto& [k, v] : config_.default_headers) req.header(k, v);
|
||||||
|
for (const auto& [k, v] : extra) req.header(k, v);
|
||||||
|
return req;
|
||||||
|
}
|
||||||
|
|
||||||
|
RawResponse ApiClient::finish(const clients::ClientResponse& response) const {
|
||||||
|
if (!response.is_success()) throw error_from_response(config_.vendor, response);
|
||||||
|
if (response.body().size() > Caps::kMaxResponseBytes)
|
||||||
|
throw ApiError(config_.vendor, 502, "BAD_RESPONSE", "response over the byte cap");
|
||||||
|
RawResponse out;
|
||||||
|
out.status = response.status();
|
||||||
|
out.body = response.body();
|
||||||
|
out.content_type = response.content_type();
|
||||||
|
out.headers = response.headers();
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
RawResponse ApiClient::request_raw(clients::Method method, const std::string& path,
|
||||||
|
const std::string* body,
|
||||||
|
const std::string& content_type,
|
||||||
|
const Headers& extra) {
|
||||||
|
guard();
|
||||||
|
try {
|
||||||
|
clients::ClientRequest req = build(method, path, extra);
|
||||||
|
if (body) {
|
||||||
|
if (body->size() > Caps::kMaxRequestBytes)
|
||||||
|
throw ApiError::invalid(config_.vendor, "request body over the byte cap");
|
||||||
|
req.bytes(*body, content_type);
|
||||||
|
}
|
||||||
|
return finish(client_.send_blocking(std::move(req)));
|
||||||
|
} catch (const ApiError&) {
|
||||||
|
throw;
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Json ApiClient::request(clients::Method method, const std::string& path,
|
||||||
|
const Json* body, const Headers& extra) {
|
||||||
|
guard();
|
||||||
|
try {
|
||||||
|
clients::ClientRequest req = build(method, path, extra);
|
||||||
|
if (body) {
|
||||||
|
req.json(*body);
|
||||||
|
if (std::holds_alternative<std::string>(req.body()) &&
|
||||||
|
req.body_bytes().size() > Caps::kMaxRequestBytes)
|
||||||
|
throw ApiError::invalid(config_.vendor, "request body over the byte cap");
|
||||||
|
}
|
||||||
|
RawResponse raw = finish(client_.send_blocking(std::move(req)));
|
||||||
|
if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object();
|
||||||
|
try {
|
||||||
|
return Json::parse(raw.body);
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(config_.vendor, 502, "BAD_RESPONSE", e.what());
|
||||||
|
}
|
||||||
|
} catch (const ApiError&) {
|
||||||
|
throw;
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Json ApiClient::get(const std::string& path, const Headers& extra) {
|
||||||
|
return request(clients::Method::kGet, path, nullptr, extra);
|
||||||
|
}
|
||||||
|
Json ApiClient::post(const std::string& path, const Json& body, const Headers& extra) {
|
||||||
|
return request(clients::Method::kPost, path, &body, extra);
|
||||||
|
}
|
||||||
|
Json ApiClient::patch(const std::string& path, const Json& body, const Headers& extra) {
|
||||||
|
return request(clients::Method::kPatch, path, &body, extra);
|
||||||
|
}
|
||||||
|
Json ApiClient::del(const std::string& path, const Headers& extra) {
|
||||||
|
return request(clients::Method::kDelete, path, nullptr, extra);
|
||||||
|
}
|
||||||
|
|
||||||
|
Json ApiClient::post_multipart(const std::string& path,
|
||||||
|
const clients::body::MultipartBody& form,
|
||||||
|
const Headers& extra) {
|
||||||
|
guard();
|
||||||
|
try {
|
||||||
|
if (form.encoded_size() > Caps::kMaxUploadBytes)
|
||||||
|
throw ApiError::invalid(config_.vendor, "upload over the byte cap");
|
||||||
|
clients::ClientRequest req = build(clients::Method::kPost, path, extra);
|
||||||
|
req.multipart(form);
|
||||||
|
RawResponse raw = finish(client_.send_blocking(std::move(req)));
|
||||||
|
if (raw.body.find_first_not_of(" \t\r\n") == std::string::npos) return Json::object();
|
||||||
|
return Json::parse(raw.body);
|
||||||
|
} catch (const ApiError&) {
|
||||||
|
throw;
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void ApiClient::stream(const std::string& path, const Json& body,
|
||||||
|
const SseDecoder::Sink& sink, const Headers& extra) {
|
||||||
|
guard();
|
||||||
|
SseDecoder decoder(config_.vendor, sink);
|
||||||
|
try {
|
||||||
|
clients::ClientRequest req = build(clients::Method::kPost, path, extra);
|
||||||
|
req.json(body);
|
||||||
|
req.header("Accept", "text/event-stream");
|
||||||
|
client_.send_stream_blocking(std::move(req), [&](const std::string& chunk) {
|
||||||
|
return decoder.feed(chunk);
|
||||||
|
});
|
||||||
|
decoder.finish();
|
||||||
|
} catch (const ApiError&) {
|
||||||
|
throw;
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(config_.vendor, 502, "UNAVAILABLE", e.what());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Service::valid_id(const std::string& id) {
|
||||||
|
if (id.empty() || id.size() > 256 || id == "." || id == "..") return false;
|
||||||
|
for (char c : id)
|
||||||
|
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
|
||||||
|
c == '-' || c == '_' || c == '.' || c == ':'))
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void Service::check_id(const std::string& what, const std::string& id) const {
|
||||||
|
if (!valid_id(id))
|
||||||
|
throw ApiError::invalid(client_->config().vendor, "bad " + what + " id");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string Service::string_at(const Json& doc, const std::string& key,
|
||||||
|
const std::string& fallback) {
|
||||||
|
const Json* v = doc.is_object() ? doc.find(key) : nullptr;
|
||||||
|
return v && v->is_string() ? v->as_string() : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// ApiClient: the one place a vendor module talks HTTP. Applies the vendor's
|
||||||
|
// auth style, sends JSON / multipart / raw bodies, streams SSE, maps every
|
||||||
|
// failure to ApiError. Runs over any clients::Transport (HttpTransport in
|
||||||
|
// production).
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
|
#include <map>
|
||||||
|
#include <memory>
|
||||||
|
#include <optional>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "clients/body/form.h"
|
||||||
|
#include "clients/body/json.h"
|
||||||
|
#include "clients/client.h"
|
||||||
|
#include "llm/error.h"
|
||||||
|
#include "llm/sse.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
enum class AuthStyle { kBearer, kHeader };
|
||||||
|
|
||||||
|
struct ApiClientConfig {
|
||||||
|
std::string vendor; // "anthropic"
|
||||||
|
std::string base_url; // "https://api.anthropic.com", no trailing slash
|
||||||
|
std::string key_env; // "ANTHROPIC_API_KEY" (looked up via llm::Config)
|
||||||
|
AuthStyle auth = AuthStyle::kBearer;
|
||||||
|
std::string auth_header = "x-api-key"; // used for kHeader
|
||||||
|
std::map<std::string, std::string> default_headers; // e.g. anthropic-version
|
||||||
|
};
|
||||||
|
|
||||||
|
using Headers = std::map<std::string, std::string>;
|
||||||
|
|
||||||
|
struct RawResponse {
|
||||||
|
int status = 0;
|
||||||
|
std::string body;
|
||||||
|
std::string content_type;
|
||||||
|
clients::HeaderMap headers;
|
||||||
|
};
|
||||||
|
|
||||||
|
class ApiClient {
|
||||||
|
public:
|
||||||
|
// Empty key: not configured, every call throws NotConfiguredError before
|
||||||
|
// any network traffic.
|
||||||
|
ApiClient(std::unique_ptr<clients::Transport> transport, ApiClientConfig config,
|
||||||
|
std::string key);
|
||||||
|
|
||||||
|
// Real HTTPS transport, key from Config::get(config.key_env).
|
||||||
|
static std::shared_ptr<ApiClient> from_env(ApiClientConfig config);
|
||||||
|
|
||||||
|
bool configured() const { return !key_.empty(); }
|
||||||
|
const ApiClientConfig& config() const { return config_; }
|
||||||
|
clients::ClientStatsSnapshot stats() const { return client_.stats(); }
|
||||||
|
|
||||||
|
// JSON in, JSON out. body may be null (GET, DELETE). Empty 2xx body -> {}.
|
||||||
|
clients::body::Json request(clients::Method method, const std::string& path,
|
||||||
|
const clients::body::Json* body = nullptr,
|
||||||
|
const Headers& extra = {});
|
||||||
|
clients::body::Json get(const std::string& path, const Headers& extra = {});
|
||||||
|
clients::body::Json post(const std::string& path, const clients::body::Json& body,
|
||||||
|
const Headers& extra = {});
|
||||||
|
clients::body::Json patch(const std::string& path, const clients::body::Json& body,
|
||||||
|
const Headers& extra = {});
|
||||||
|
clients::body::Json del(const std::string& path, const Headers& extra = {});
|
||||||
|
|
||||||
|
// Bytes in/out for non-JSON payloads (file content, images, audio).
|
||||||
|
RawResponse request_raw(clients::Method method, const std::string& path,
|
||||||
|
const std::string* body = nullptr,
|
||||||
|
const std::string& content_type = "",
|
||||||
|
const Headers& extra = {});
|
||||||
|
clients::body::Json post_multipart(const std::string& path,
|
||||||
|
const clients::body::MultipartBody& form,
|
||||||
|
const Headers& extra = {});
|
||||||
|
|
||||||
|
// Streaming POST decoded as SSE; sink returns false to stop.
|
||||||
|
void stream(const std::string& path, const clients::body::Json& body,
|
||||||
|
const SseDecoder::Sink& sink, const Headers& extra = {});
|
||||||
|
|
||||||
|
private:
|
||||||
|
clients::ClientRequest build(clients::Method method, const std::string& path,
|
||||||
|
const Headers& extra) const;
|
||||||
|
void guard() const;
|
||||||
|
RawResponse finish(const clients::ClientResponse& response) const;
|
||||||
|
|
||||||
|
ApiClientConfig config_;
|
||||||
|
std::string key_;
|
||||||
|
clients::Client client_;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Base for vendor service classes.
|
||||||
|
class Service {
|
||||||
|
public:
|
||||||
|
explicit Service(std::shared_ptr<ApiClient> client) : client_(std::move(client)) {}
|
||||||
|
virtual ~Service() = default;
|
||||||
|
bool configured() const { return client_->configured(); }
|
||||||
|
ApiClient& client() const { return *client_; }
|
||||||
|
|
||||||
|
// True for a single URL path segment: 1..256 of [A-Za-z0-9._:-] and not
|
||||||
|
// "." or "..". Use before putting any id in a path.
|
||||||
|
static bool valid_id(const std::string& id);
|
||||||
|
// Throws ApiError::invalid when the id is not a valid segment.
|
||||||
|
void check_id(const std::string& what, const std::string& id) const;
|
||||||
|
static std::string string_at(const clients::body::Json& doc, const std::string& key,
|
||||||
|
const std::string& fallback = "");
|
||||||
|
|
||||||
|
protected:
|
||||||
|
std::shared_ptr<ApiClient> client_;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// Static ceilings shared by every LLM vendor module. Never autotuned.
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
struct Caps {
|
||||||
|
static constexpr std::size_t kMaxRequestBytes = 32u * 1024u * 1024u;
|
||||||
|
static constexpr std::size_t kMaxResponseBytes = 64u * 1024u * 1024u;
|
||||||
|
static constexpr std::size_t kMaxStreamBytes = 64u * 1024u * 1024u;
|
||||||
|
static constexpr std::size_t kMaxSseEventBytes = 4u * 1024u * 1024u;
|
||||||
|
static constexpr std::size_t kMaxUploadBytes = 512u * 1024u * 1024u;
|
||||||
|
static constexpr std::size_t kMaxMessages = 4096;
|
||||||
|
static constexpr std::size_t kMaxTools = 256;
|
||||||
|
static constexpr std::size_t kMaxPages = 100;
|
||||||
|
static constexpr std::size_t kMaxConfigBytes = 64u * 1024u;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
#include "llm/chat_types.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
const char* role_name(Role role) {
|
||||||
|
switch (role) {
|
||||||
|
case Role::kSystem: return "system";
|
||||||
|
case Role::kUser: return "user";
|
||||||
|
case Role::kAssistant: return "assistant";
|
||||||
|
case Role::kTool: return "tool";
|
||||||
|
}
|
||||||
|
return "user";
|
||||||
|
}
|
||||||
|
|
||||||
|
ContentPart ContentPart::of_text(std::string text) {
|
||||||
|
ContentPart p;
|
||||||
|
p.text = std::move(text);
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
ContentPart ContentPart::of_image(ImageData image) {
|
||||||
|
ContentPart p;
|
||||||
|
p.image = std::move(image);
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
ContentPart ContentPart::of_call(ToolCall call) {
|
||||||
|
ContentPart p;
|
||||||
|
p.tool_call = std::move(call);
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
ContentPart ContentPart::of_result(ToolResult result) {
|
||||||
|
ContentPart p;
|
||||||
|
p.tool_result = std::move(result);
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
Message make(Role role, std::string text) {
|
||||||
|
Message m;
|
||||||
|
m.role = role;
|
||||||
|
m.parts.push_back(ContentPart::of_text(std::move(text)));
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
Message Message::system(std::string text) { return make(Role::kSystem, std::move(text)); }
|
||||||
|
Message Message::user(std::string text) { return make(Role::kUser, std::move(text)); }
|
||||||
|
Message Message::assistant(std::string text) { return make(Role::kAssistant, std::move(text)); }
|
||||||
|
|
||||||
|
std::string Message::text() const {
|
||||||
|
std::string out;
|
||||||
|
for (const ContentPart& p : parts) out += p.text;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<ToolCall> ChatResponse::tool_calls() const {
|
||||||
|
std::vector<ToolCall> out;
|
||||||
|
for (const ContentPart& p : message.parts)
|
||||||
|
if (p.tool_call) out.push_back(*p.tool_call);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// Vendor-neutral chat model: messages, tools, requests, responses, stream
|
||||||
|
// chunks, and the ChatModel seam. Each vendor module implements ChatModel
|
||||||
|
// over its native wire format (and keeps its native types for everything
|
||||||
|
// beyond chat). Agents, evaluation and apps code against ChatModel.
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
|
#include <optional>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "clients/body/json.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
enum class Role { kSystem, kUser, kAssistant, kTool };
|
||||||
|
|
||||||
|
const char* role_name(Role role);
|
||||||
|
|
||||||
|
struct ImageData {
|
||||||
|
std::string mime;
|
||||||
|
std::string bytes; // Raw bytes; empty when url is set.
|
||||||
|
std::string url;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ToolCall {
|
||||||
|
std::string id;
|
||||||
|
std::string name;
|
||||||
|
clients::body::Json arguments = clients::body::Json::object();
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ToolResult {
|
||||||
|
std::string call_id;
|
||||||
|
std::string content;
|
||||||
|
bool is_error = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ContentPart {
|
||||||
|
std::string text;
|
||||||
|
std::string reasoning; // Model thinking text (never sent back unless the vendor requires it)
|
||||||
|
std::string reasoning_signature; // Opaque; replay it exactly (Anthropic thinking)
|
||||||
|
std::optional<ImageData> image;
|
||||||
|
std::optional<ToolCall> tool_call;
|
||||||
|
std::optional<ToolResult> tool_result;
|
||||||
|
|
||||||
|
static ContentPart of_text(std::string text);
|
||||||
|
static ContentPart of_image(ImageData image);
|
||||||
|
static ContentPart of_call(ToolCall call);
|
||||||
|
static ContentPart of_result(ToolResult result);
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Message {
|
||||||
|
Role role = Role::kUser;
|
||||||
|
std::vector<ContentPart> parts;
|
||||||
|
std::string name; // Optional participant name.
|
||||||
|
|
||||||
|
static Message system(std::string text);
|
||||||
|
static Message user(std::string text);
|
||||||
|
static Message assistant(std::string text);
|
||||||
|
std::string text() const; // Concatenated text parts.
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ToolSpec {
|
||||||
|
std::string name;
|
||||||
|
std::string description;
|
||||||
|
clients::body::Json parameters = clients::body::Json::object(); // JSON Schema
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ChatRequest {
|
||||||
|
std::string model;
|
||||||
|
std::string system; // Convenience: prepended as a system message / field.
|
||||||
|
std::vector<Message> messages;
|
||||||
|
std::vector<ToolSpec> tools;
|
||||||
|
std::string tool_choice; // "", "auto", "none", "required", or a tool name
|
||||||
|
std::optional<int> max_tokens;
|
||||||
|
std::optional<double> temperature;
|
||||||
|
std::optional<double> top_p;
|
||||||
|
std::vector<std::string> stop;
|
||||||
|
bool json_mode = false; // Ask for a JSON object
|
||||||
|
clients::body::Json response_schema; // null: none; else strict JSON Schema
|
||||||
|
std::string reasoning_effort; // "", "low", "medium", "high", vendor values
|
||||||
|
clients::body::Json extra; // Vendor passthrough, merged into the body
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Usage {
|
||||||
|
int input_tokens = 0;
|
||||||
|
int output_tokens = 0;
|
||||||
|
int total_tokens = 0;
|
||||||
|
int cached_tokens = 0;
|
||||||
|
int reasoning_tokens = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ChatResponse {
|
||||||
|
std::string id;
|
||||||
|
std::string model;
|
||||||
|
Message message; // role = kAssistant
|
||||||
|
std::string finish_reason; // vendor-neutral where possible: stop, length, tool_calls, content_filter
|
||||||
|
Usage usage;
|
||||||
|
clients::body::Json raw; // The vendor document, for fields the neutral type omits
|
||||||
|
|
||||||
|
std::string text() const { return message.text(); }
|
||||||
|
std::vector<ToolCall> tool_calls() const;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ChatChunk {
|
||||||
|
std::string text_delta;
|
||||||
|
std::string reasoning_delta;
|
||||||
|
std::optional<ToolCall> tool_call; // Complete call when the vendor streams it whole
|
||||||
|
std::string tool_call_id; // Partial call fragments: id + name + args_delta
|
||||||
|
std::string tool_call_name;
|
||||||
|
std::string tool_args_delta;
|
||||||
|
std::string finish_reason;
|
||||||
|
std::optional<Usage> usage;
|
||||||
|
};
|
||||||
|
|
||||||
|
class ChatModel {
|
||||||
|
public:
|
||||||
|
virtual ~ChatModel() = default;
|
||||||
|
virtual std::string vendor() const = 0;
|
||||||
|
virtual ChatResponse generate(const ChatRequest& request) = 0;
|
||||||
|
// on_chunk returns false to stop early.
|
||||||
|
virtual void stream(const ChatRequest& request,
|
||||||
|
const std::function<bool(const ChatChunk&)>& on_chunk) = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#include "llm/config.h"
|
||||||
|
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <fstream>
|
||||||
|
#include <sstream>
|
||||||
|
|
||||||
|
#include "clients/body/json.h"
|
||||||
|
#include "llm/caps.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
std::string from_json_file(const char* path, const char* name) {
|
||||||
|
std::ifstream in(path, std::ios::binary);
|
||||||
|
if (!in) return "";
|
||||||
|
std::ostringstream text;
|
||||||
|
text << in.rdbuf();
|
||||||
|
if (text.str().size() > Caps::kMaxConfigBytes) return "";
|
||||||
|
try {
|
||||||
|
clients::body::Json doc = clients::body::Json::parse(text.str());
|
||||||
|
const clients::body::Json* v = doc.is_object() ? doc.find(name) : nullptr;
|
||||||
|
return v && v->is_string() ? v->as_string() : "";
|
||||||
|
} catch (const std::exception&) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
std::string Config::get(const char* name) {
|
||||||
|
if (const char* v = std::getenv(name); v && *v) return v;
|
||||||
|
const char* files = std::getenv("MOLO_ENV_FILES");
|
||||||
|
if (files && std::string(files) == "off" && !std::getenv("MOLO_LIVE_NET")) return "";
|
||||||
|
std::string local = from_json_file(".env.json.local", name);
|
||||||
|
if (!local.empty()) return local;
|
||||||
|
return from_json_file(".env.json", name);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// Configuration lookup shared by all vendors: process environment, then
|
||||||
|
// .env.json.local (git-ignored, for secrets), then .env.json, both in the
|
||||||
|
// working directory. Test runs set MOLO_ENV_FILES=off so a developer's local
|
||||||
|
// file never changes test results; MOLO_LIVE_NET=1 re-enables it.
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
class Config {
|
||||||
|
public:
|
||||||
|
// Value of `name`, or empty when unset everywhere.
|
||||||
|
static std::string get(const char* name);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
#include "llm/error.h"
|
||||||
|
|
||||||
|
#include "clients/body/json.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
using clients::body::Json;
|
||||||
|
|
||||||
|
ApiError::ApiError(std::string vendor, int http_status, std::string type,
|
||||||
|
const std::string& message, std::string request_id)
|
||||||
|
: std::runtime_error("molo::llm::" + vendor + ": " + type + ": " + message),
|
||||||
|
vendor_(std::move(vendor)),
|
||||||
|
http_status_(http_status),
|
||||||
|
type_(std::move(type)),
|
||||||
|
message_(message),
|
||||||
|
request_id_(std::move(request_id)) {}
|
||||||
|
|
||||||
|
ApiError ApiError::invalid(const std::string& vendor, const std::string& message) {
|
||||||
|
return ApiError(vendor, 400, "INVALID_ARGUMENT", message);
|
||||||
|
}
|
||||||
|
|
||||||
|
NotConfiguredError::NotConfiguredError(const std::string& vendor,
|
||||||
|
const std::string& env_var)
|
||||||
|
: ApiError(vendor, 503, "NOT_CONFIGURED",
|
||||||
|
"no credential configured (set " + env_var + ")") {}
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
std::string text_of(const Json* v) {
|
||||||
|
if (!v) return "";
|
||||||
|
if (v->is_string()) return v->as_string();
|
||||||
|
if (v->is_number()) return std::to_string(v->as_int());
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
ApiError error_from_response(const std::string& vendor,
|
||||||
|
const clients::ClientResponse& response) {
|
||||||
|
std::string type = "UNKNOWN";
|
||||||
|
std::string message = "HTTP " + std::to_string(response.status());
|
||||||
|
std::string request_id;
|
||||||
|
for (const char* h : {"request-id", "x-request-id", "x-should-retry-request-id"})
|
||||||
|
if (auto v = response.headers().get(h); v && !v->empty()) {
|
||||||
|
request_id = *v;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Json doc = response.json();
|
||||||
|
const Json* err = doc.is_object() ? doc.find("error") : nullptr;
|
||||||
|
if (err && err->is_object()) {
|
||||||
|
std::string t = text_of(err->find("type"));
|
||||||
|
if (t.empty()) t = text_of(err->find("code"));
|
||||||
|
if (!t.empty()) type = t;
|
||||||
|
std::string m = text_of(err->find("message"));
|
||||||
|
if (!m.empty()) message = m;
|
||||||
|
} else if (err && err->is_string()) {
|
||||||
|
message = err->as_string();
|
||||||
|
} else if (doc.is_object()) {
|
||||||
|
std::string m = text_of(doc.find("message"));
|
||||||
|
if (!m.empty()) message = m;
|
||||||
|
std::string t = text_of(doc.find("type"));
|
||||||
|
if (t.empty()) t = text_of(doc.find("code"));
|
||||||
|
if (!t.empty()) type = t;
|
||||||
|
}
|
||||||
|
} catch (const std::exception&) {
|
||||||
|
// Non-JSON body: keep the generic type and message.
|
||||||
|
}
|
||||||
|
return ApiError(vendor, response.status(), type, message, request_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// One error type for every vendor call. Carries the HTTP status, the
|
||||||
|
// vendor's own error type/code and message, and the request id when the
|
||||||
|
// server sent one. Never the credential.
|
||||||
|
|
||||||
|
#include <stdexcept>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "clients/response.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
class ApiError : public std::runtime_error {
|
||||||
|
public:
|
||||||
|
ApiError(std::string vendor, int http_status, std::string type,
|
||||||
|
const std::string& message, std::string request_id = "");
|
||||||
|
|
||||||
|
const std::string& vendor() const noexcept { return vendor_; }
|
||||||
|
int http_status() const noexcept { return http_status_; }
|
||||||
|
// Vendor error type or code ("authentication_error", "invalid_request_error",
|
||||||
|
// "rate_limit_exceeded", ...), or a framework code (NOT_CONFIGURED, ...).
|
||||||
|
const std::string& type() const noexcept { return type_; }
|
||||||
|
const std::string& message() const noexcept { return message_; }
|
||||||
|
const std::string& request_id() const noexcept { return request_id_; }
|
||||||
|
|
||||||
|
static ApiError invalid(const std::string& vendor, const std::string& message);
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::string vendor_;
|
||||||
|
int http_status_;
|
||||||
|
std::string type_;
|
||||||
|
std::string message_;
|
||||||
|
std::string request_id_;
|
||||||
|
};
|
||||||
|
|
||||||
|
class NotConfiguredError : public ApiError {
|
||||||
|
public:
|
||||||
|
NotConfiguredError(const std::string& vendor, const std::string& env_var);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Parses the common error envelopes: {"error":{"type"|"code","message"}},
|
||||||
|
// {"type":"error","error":{...}}, {"error":"text"}, {"message":"text"}.
|
||||||
|
ApiError error_from_response(const std::string& vendor,
|
||||||
|
const clients::ClientResponse& response);
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
+108
@@ -0,0 +1,108 @@
|
|||||||
|
#include "llm/sse.h"
|
||||||
|
|
||||||
|
#include "llm/caps.h"
|
||||||
|
#include "llm/error.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
using clients::body::Json;
|
||||||
|
|
||||||
|
SseDecoder::SseDecoder(std::string vendor, Sink sink)
|
||||||
|
: vendor_(std::move(vendor)), sink_(std::move(sink)) {}
|
||||||
|
|
||||||
|
bool SseDecoder::dispatch(const std::string& block) {
|
||||||
|
std::string event;
|
||||||
|
std::string data;
|
||||||
|
std::size_t pos = 0;
|
||||||
|
while (pos <= block.size()) {
|
||||||
|
std::size_t eol = block.find('\n', pos);
|
||||||
|
std::string line = block.substr(pos, eol == std::string::npos ? eol : eol - pos);
|
||||||
|
if (!line.empty() && line.back() == '\r') line.pop_back();
|
||||||
|
if (line.rfind("event:", 0) == 0) {
|
||||||
|
event = line.substr(6);
|
||||||
|
if (!event.empty() && event.front() == ' ') event.erase(0, 1);
|
||||||
|
} else if (line.rfind("data:", 0) == 0) {
|
||||||
|
std::string v = line.substr(5);
|
||||||
|
if (!v.empty() && v.front() == ' ') v.erase(0, 1);
|
||||||
|
if (!data.empty()) data += "\n";
|
||||||
|
data += v;
|
||||||
|
}
|
||||||
|
if (eol == std::string::npos) break;
|
||||||
|
pos = eol + 1;
|
||||||
|
}
|
||||||
|
if (data.empty()) return true;
|
||||||
|
if (data == "[DONE]") {
|
||||||
|
done_ = true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (data.size() > Caps::kMaxSseEventBytes)
|
||||||
|
throw ApiError(vendor_, 502, "BAD_RESPONSE", "SSE event over the byte cap");
|
||||||
|
SseEvent ev;
|
||||||
|
ev.event = event;
|
||||||
|
try {
|
||||||
|
ev.data = Json::parse(data);
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(vendor_, 502, "BAD_RESPONSE", std::string("bad SSE data: ") + e.what());
|
||||||
|
}
|
||||||
|
if (!sink_(ev)) {
|
||||||
|
done_ = true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool SseDecoder::feed(const std::string& chunk) {
|
||||||
|
if (done_) return false;
|
||||||
|
total_ += chunk.size();
|
||||||
|
if (total_ > Caps::kMaxStreamBytes)
|
||||||
|
throw ApiError(vendor_, 502, "BAD_RESPONSE", "stream over the byte cap");
|
||||||
|
buffer_ += chunk;
|
||||||
|
if (!started_) {
|
||||||
|
std::size_t first = buffer_.find_first_not_of(" \t\r\n");
|
||||||
|
if (first == std::string::npos) return true;
|
||||||
|
started_ = true;
|
||||||
|
sse_ = buffer_[first] != '{' && buffer_[first] != '[';
|
||||||
|
}
|
||||||
|
if (!sse_) return true; // Plain JSON (an error body): handled in finish().
|
||||||
|
for (;;) {
|
||||||
|
std::size_t end = buffer_.find("\n\n");
|
||||||
|
std::size_t sep = 2;
|
||||||
|
std::size_t crlf = buffer_.find("\r\n\r\n");
|
||||||
|
if (crlf != std::string::npos && (end == std::string::npos || crlf < end)) {
|
||||||
|
end = crlf;
|
||||||
|
sep = 4;
|
||||||
|
}
|
||||||
|
if (end == std::string::npos) break;
|
||||||
|
std::string block = buffer_.substr(0, end);
|
||||||
|
buffer_.erase(0, end + sep);
|
||||||
|
if (!dispatch(block)) return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void SseDecoder::finish() {
|
||||||
|
if (done_ || !started_) return;
|
||||||
|
if (sse_) {
|
||||||
|
if (buffer_.find_first_not_of(" \t\r\n") != std::string::npos) dispatch(buffer_);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A JSON body where a stream was expected is the vendor's error document.
|
||||||
|
Json doc;
|
||||||
|
try {
|
||||||
|
doc = Json::parse(buffer_);
|
||||||
|
} catch (const std::exception& e) {
|
||||||
|
throw ApiError(vendor_, 502, "BAD_RESPONSE", e.what());
|
||||||
|
}
|
||||||
|
const Json* err = doc.is_object() ? doc.find("error") : nullptr;
|
||||||
|
std::string message = err && err->is_object() && err->find("message") &&
|
||||||
|
err->find("message")->is_string()
|
||||||
|
? err->find("message")->as_string()
|
||||||
|
: buffer_.substr(0, 300);
|
||||||
|
std::string type = err && err->is_object() && err->find("type") &&
|
||||||
|
err->find("type")->is_string()
|
||||||
|
? err->find("type")->as_string()
|
||||||
|
: "STREAM_ERROR";
|
||||||
|
throw ApiError(vendor_, 502, type, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
// Server-sent events decoder keeping the event name: `event: x` + `data: {...}`
|
||||||
|
// blocks (Anthropic), bare `data:` blocks (OpenAI-compatible vendors), the
|
||||||
|
// `data: [DONE]` sentinel (ends the stream, not delivered). Comment lines
|
||||||
|
// (`: ping`) are ignored. A plain JSON error body (no SSE framing) throws.
|
||||||
|
|
||||||
|
#include <functional>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "clients/body/json.h"
|
||||||
|
|
||||||
|
namespace molo::llm {
|
||||||
|
|
||||||
|
struct SseEvent {
|
||||||
|
std::string event; // Empty when the block had no `event:` line.
|
||||||
|
clients::body::Json data;
|
||||||
|
};
|
||||||
|
|
||||||
|
class SseDecoder {
|
||||||
|
public:
|
||||||
|
using Sink = std::function<bool(const SseEvent&)>; // false: stop
|
||||||
|
|
||||||
|
SseDecoder(std::string vendor, Sink sink);
|
||||||
|
// Returns false when the sink stopped the stream or [DONE] was seen.
|
||||||
|
bool feed(const std::string& chunk);
|
||||||
|
void finish();
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool dispatch(const std::string& block);
|
||||||
|
|
||||||
|
std::string vendor_;
|
||||||
|
Sink sink_;
|
||||||
|
std::string buffer_;
|
||||||
|
std::size_t total_ = 0;
|
||||||
|
bool done_ = false;
|
||||||
|
bool started_ = false;
|
||||||
|
bool sse_ = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace molo::llm
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
// Shared LLM layer: SSE decoding, auth styles, error envelopes, verbs,
|
||||||
|
// multipart, streaming through the client, config lookup, neutral types.
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
#include "framework.h"
|
||||||
|
#include "clients/transports/mock_transport.h"
|
||||||
|
#include "llm/api_client.h"
|
||||||
|
#include "llm/chat_types.h"
|
||||||
|
#include "llm/config.h"
|
||||||
|
#include "llm/sse.h"
|
||||||
|
|
||||||
|
using namespace molo::llm;
|
||||||
|
using molo::clients::ClientResponse;
|
||||||
|
using molo::clients::HeaderMap;
|
||||||
|
using molo::clients::Method;
|
||||||
|
using molo::clients::body::Json;
|
||||||
|
using molo::clients::transports::MockTransport;
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
ClientResponse ok(const std::string& body, int status = 200,
|
||||||
|
const std::string& type = "application/json") {
|
||||||
|
HeaderMap h;
|
||||||
|
h.set("Content-Type", type);
|
||||||
|
return ClientResponse(status, h, body);
|
||||||
|
}
|
||||||
|
struct Rig {
|
||||||
|
MockTransport* mock;
|
||||||
|
std::shared_ptr<ApiClient> client;
|
||||||
|
explicit Rig(AuthStyle style = AuthStyle::kBearer, const std::string& key = "SECRETKEY") {
|
||||||
|
auto t = std::make_unique<MockTransport>();
|
||||||
|
mock = t.get();
|
||||||
|
ApiClientConfig c;
|
||||||
|
c.vendor = "acme";
|
||||||
|
c.base_url = "https://api.acme.example";
|
||||||
|
c.key_env = "ACME_API_KEY";
|
||||||
|
c.auth = style;
|
||||||
|
c.auth_header = "x-api-key";
|
||||||
|
c.default_headers["acme-version"] = "2024-01-01";
|
||||||
|
client = std::make_shared<ApiClient>(std::move(t), c, key);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
TEST(llm, bearer_and_header_auth_plus_default_headers) {
|
||||||
|
Rig b;
|
||||||
|
b.mock->expect_response(Method::kGet, "/v1/x", ok("{\"a\":1}"));
|
||||||
|
ASSERT_EQ(b.client->get("/v1/x").find("a")->as_int(), 1);
|
||||||
|
auto s = b.mock->seen()[0];
|
||||||
|
ASSERT_EQ(s.headers.get_or("Authorization", ""), "Bearer SECRETKEY");
|
||||||
|
ASSERT_EQ(s.headers.get_or("acme-version", ""), "2024-01-01");
|
||||||
|
ASSERT_EQ(s.target.find("SECRETKEY"), std::string::npos);
|
||||||
|
|
||||||
|
Rig h(AuthStyle::kHeader);
|
||||||
|
h.mock->expect_response(Method::kPost, "/v1/y", ok("{}"));
|
||||||
|
Json body = Json::object();
|
||||||
|
body.set("k", Json("v"));
|
||||||
|
h.client->post("/v1/y", body, {{"extra-beta", "1"}});
|
||||||
|
auto s2 = h.mock->seen()[0];
|
||||||
|
ASSERT_EQ(s2.headers.get_or("x-api-key", ""), "SECRETKEY");
|
||||||
|
ASSERT_EQ(s2.headers.get_or("Authorization", ""), "");
|
||||||
|
ASSERT_EQ(s2.headers.get_or("extra-beta", ""), "1");
|
||||||
|
ASSERT_EQ(s2.body, "{\"k\":\"v\"}");
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, not_configured_before_network) {
|
||||||
|
Rig r(AuthStyle::kBearer, "");
|
||||||
|
ASSERT_FALSE(r.client->configured());
|
||||||
|
try {
|
||||||
|
r.client->get("/v1/x");
|
||||||
|
FAIL("expected NotConfiguredError");
|
||||||
|
} catch (const NotConfiguredError& e) {
|
||||||
|
ASSERT_EQ(e.http_status(), 503);
|
||||||
|
ASSERT_EQ(e.type(), "NOT_CONFIGURED");
|
||||||
|
ASSERT_TRUE(e.message().find("ACME_API_KEY") != std::string::npos);
|
||||||
|
}
|
||||||
|
ASSERT_EQ(r.mock->send_count(), 0u);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, error_envelopes) {
|
||||||
|
struct Case { const char* body; int status; const char* type; const char* msg; };
|
||||||
|
const Case cases[] = {
|
||||||
|
{"{\"error\":{\"type\":\"authentication_error\",\"message\":\"bad key\"}}", 401, "authentication_error", "bad key"},
|
||||||
|
{"{\"type\":\"error\",\"error\":{\"type\":\"not_found_error\",\"message\":\"nope\"}}", 404, "not_found_error", "nope"},
|
||||||
|
{"{\"error\":{\"code\":\"rate_limit_exceeded\",\"message\":\"slow\"}}", 429, "rate_limit_exceeded", "slow"},
|
||||||
|
{"{\"error\":\"plain text error\"}", 400, "UNKNOWN", "plain text error"},
|
||||||
|
{"{\"message\":\"top level\",\"code\":\"E42\"}", 400, "E42", "top level"},
|
||||||
|
{"<html>bad gateway</html>", 400, "UNKNOWN", "HTTP 400"},
|
||||||
|
};
|
||||||
|
for (const Case& c : cases) {
|
||||||
|
Rig r;
|
||||||
|
molo::clients::transports::MockExpectation ex; // 429/5xx are retried: allow every attempt
|
||||||
|
ex.method = Method::kGet;
|
||||||
|
ex.path = "/v1/e";
|
||||||
|
ex.response = ok(c.body, c.status);
|
||||||
|
ex.times = 5;
|
||||||
|
r.mock->expect(ex);
|
||||||
|
try {
|
||||||
|
r.client->get("/v1/e");
|
||||||
|
FAIL("expected ApiError");
|
||||||
|
} catch (const ApiError& e) {
|
||||||
|
ASSERT_EQ(e.http_status(), c.status);
|
||||||
|
ASSERT_EQ(e.type(), c.type);
|
||||||
|
ASSERT_EQ(e.message(), c.msg);
|
||||||
|
ASSERT_EQ(e.vendor(), "acme");
|
||||||
|
ASSERT_EQ(std::string(e.what()).find("SECRETKEY"), std::string::npos);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, request_id_header_is_captured) {
|
||||||
|
Rig r;
|
||||||
|
HeaderMap h;
|
||||||
|
h.set("Content-Type", "application/json");
|
||||||
|
h.set("request-id", "req_123");
|
||||||
|
r.mock->expect_response(Method::kGet, "/v1/e", ClientResponse(500, h, "{\"error\":{\"message\":\"x\"}}"));
|
||||||
|
try {
|
||||||
|
r.client->get("/v1/e");
|
||||||
|
FAIL("expected");
|
||||||
|
} catch (const ApiError& e) {
|
||||||
|
ASSERT_EQ(e.request_id(), "req_123");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, path_guard_and_empty_body) {
|
||||||
|
Rig r;
|
||||||
|
ASSERT_THROW(r.client->get("v1/x"), ApiError);
|
||||||
|
ASSERT_THROW(r.client->get("/v1/../x"), ApiError);
|
||||||
|
ASSERT_EQ(r.mock->send_count(), 0u);
|
||||||
|
r.mock->expect_response(Method::kDelete, "/v1/x", ok("", 204));
|
||||||
|
ASSERT_TRUE(r.client->del("/v1/x").is_object());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, raw_and_multipart) {
|
||||||
|
Rig r;
|
||||||
|
r.mock->expect_response(Method::kGet, "/v1/f", ok("BINARY\x01\x02", 200, "application/octet-stream"));
|
||||||
|
RawResponse raw = r.client->request_raw(Method::kGet, "/v1/f");
|
||||||
|
ASSERT_EQ(raw.body, std::string("BINARY\x01\x02"));
|
||||||
|
ASSERT_EQ(raw.content_type, "application/octet-stream");
|
||||||
|
|
||||||
|
r.mock->expect_response(Method::kPost, "/v1/files", ok("{\"id\":\"file_1\"}"));
|
||||||
|
molo::clients::body::MultipartBody form;
|
||||||
|
form.add_text("purpose", "assistants");
|
||||||
|
form.add_file("file", "a.txt", "text/plain", "hello");
|
||||||
|
ASSERT_EQ(r.client->post_multipart("/v1/files", form).find("id")->as_string(), "file_1");
|
||||||
|
auto seen = r.mock->seen()[1];
|
||||||
|
ASSERT_TRUE(seen.headers.get_or("Content-Type", "").find("multipart/form-data") == 0);
|
||||||
|
ASSERT_TRUE(seen.body.find("hello") != std::string::npos);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, sse_named_events_done_and_chunking) {
|
||||||
|
std::vector<std::pair<std::string, std::string>> got;
|
||||||
|
SseDecoder d("acme", [&](const SseEvent& e) {
|
||||||
|
got.push_back({e.event, e.data.dump()});
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
d.feed("event: message_start\ndata: {\"a\":1}\n\n: ping\n\nevent: delta\nda");
|
||||||
|
d.feed("ta: {\"b\":2}\r\n\r\ndata: {\"c\":3}\n\ndata: [DONE]\n\ndata: {\"never\":1}\n\n");
|
||||||
|
d.finish();
|
||||||
|
ASSERT_EQ(got.size(), 3u);
|
||||||
|
ASSERT_EQ(got[0].first, "message_start");
|
||||||
|
ASSERT_EQ(got[1].second, "{\"b\":2}");
|
||||||
|
ASSERT_EQ(got[2].first, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, sse_errors_and_stop) {
|
||||||
|
SseDecoder bad("acme", [](const SseEvent&) { return true; });
|
||||||
|
ASSERT_THROW(bad.feed("data: {not json}\n\n"), ApiError);
|
||||||
|
SseDecoder err("acme", [](const SseEvent&) { return true; });
|
||||||
|
err.feed("{\"error\":{\"type\":\"overloaded_error\",\"message\":\"busy\"}}");
|
||||||
|
try {
|
||||||
|
err.finish();
|
||||||
|
FAIL("expected");
|
||||||
|
} catch (const ApiError& e) {
|
||||||
|
ASSERT_EQ(e.type(), "overloaded_error");
|
||||||
|
ASSERT_EQ(e.message(), "busy");
|
||||||
|
}
|
||||||
|
int n = 0;
|
||||||
|
SseDecoder stop("acme", [&](const SseEvent&) { return ++n < 2; });
|
||||||
|
ASSERT_FALSE(stop.feed("data: {\"a\":1}\n\ndata: {\"a\":2}\n\ndata: {\"a\":3}\n\n"));
|
||||||
|
ASSERT_EQ(n, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, stream_through_client) {
|
||||||
|
Rig r;
|
||||||
|
r.mock->expect_response(
|
||||||
|
Method::kPost, "/v1/chat",
|
||||||
|
ok("data: {\"d\":\"a\"}\n\ndata: {\"d\":\"b\"}\n\ndata: [DONE]\n\n", 200, "text/event-stream"));
|
||||||
|
std::string text;
|
||||||
|
r.client->stream("/v1/chat", Json::object(), [&](const SseEvent& e) {
|
||||||
|
text += e.data.find("d")->as_string();
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
ASSERT_EQ(text, "ab");
|
||||||
|
ASSERT_EQ(r.mock->seen()[0].headers.get_or("Accept", ""), "text/event-stream");
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, id_validation) {
|
||||||
|
ASSERT_TRUE(Service::valid_id("msgbatch_01ABC"));
|
||||||
|
ASSERT_TRUE(Service::valid_id("grok-4.1:fast"));
|
||||||
|
ASSERT_FALSE(Service::valid_id(""));
|
||||||
|
ASSERT_FALSE(Service::valid_id(".."));
|
||||||
|
ASSERT_FALSE(Service::valid_id("a/b"));
|
||||||
|
ASSERT_FALSE(Service::valid_id("a b"));
|
||||||
|
ASSERT_FALSE(Service::valid_id(std::string(257, 'a')));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, config_lookup_order_and_off_switch) {
|
||||||
|
setenv("MOLO_ENV_FILES", "off", 1);
|
||||||
|
unsetenv("MOLO_LIVE_NET");
|
||||||
|
unsetenv("ACME_TEST_KNOB");
|
||||||
|
ASSERT_EQ(Config::get("ACME_TEST_KNOB"), "");
|
||||||
|
setenv("ACME_TEST_KNOB", "from-env", 1);
|
||||||
|
ASSERT_EQ(Config::get("ACME_TEST_KNOB"), "from-env");
|
||||||
|
unsetenv("ACME_TEST_KNOB");
|
||||||
|
unsetenv("MOLO_ENV_FILES");
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(llm, neutral_types) {
|
||||||
|
Message m = Message::user("hi");
|
||||||
|
ASSERT_EQ(m.text(), "hi");
|
||||||
|
ASSERT_EQ(std::string(role_name(m.role)), "user");
|
||||||
|
ChatResponse r;
|
||||||
|
r.message.role = Role::kAssistant;
|
||||||
|
r.message.parts.push_back(ContentPart::of_text("a"));
|
||||||
|
ToolCall c;
|
||||||
|
c.id = "t1";
|
||||||
|
c.name = "f";
|
||||||
|
r.message.parts.push_back(ContentPart::of_call(c));
|
||||||
|
ASSERT_EQ(r.text(), "a");
|
||||||
|
ASSERT_EQ(r.tool_calls().size(), 1u);
|
||||||
|
ASSERT_EQ(r.tool_calls()[0].name, "f");
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user