269 lines
11 KiB
C++
269 lines
11 KiB
C++
// Deterministic local tests for the OpenRouter module: the call table, request
|
|
// planning and validation, value classes (model variants, provider
|
|
// preferences, reasoning, plugins, server tools), stream parsing with a
|
|
// documented transcript, error semantics, key routing, PKCE. No OpenRouter
|
|
// reply is scripted anywhere: transports only record the requests they get.
|
|
#include <memory>
|
|
#include <set>
|
|
|
|
#include "clients/transports/mock_transport.h"
|
|
#include "framework.h"
|
|
#include "llm/sse.h"
|
|
#include "openrouter/account.h"
|
|
#include "openrouter/admin.h"
|
|
#include "openrouter/call_plan.h"
|
|
#include "openrouter/caps.h"
|
|
#include "openrouter/chat_model.h"
|
|
#include "openrouter/chat_params.h"
|
|
#include "openrouter/chat_service.h"
|
|
#include "openrouter/chat_stream.h"
|
|
#include "openrouter/errors.h"
|
|
#include "openrouter/gateway.h"
|
|
#include "openrouter/inference_services.h"
|
|
#include "openrouter/model_ref.h"
|
|
#include "openrouter/models.h"
|
|
#include "openrouter/platform.h"
|
|
#include "openrouter/plugins.h"
|
|
#include "openrouter/server_tools.h"
|
|
|
|
using namespace molo::openrouter;
|
|
using molo::clients::Method;
|
|
using molo::clients::body::Json;
|
|
using molo::clients::transports::MockTransport;
|
|
using molo::llm::ApiError;
|
|
|
|
namespace {
|
|
|
|
PathArgs dummy_args(const CallSpec& s) {
|
|
PathArgs a;
|
|
for (const std::string& p : CallPlanner::placeholders(s.path)) a[p] = "x1";
|
|
return a;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
TEST(openrouter, table_is_complete_and_plannable) {
|
|
ASSERT_EQ(kCallCount, 140u);
|
|
std::set<std::string> names;
|
|
int mgmt = 0, none = 0, api = 0;
|
|
for (const CallSpec& s : kCalls) {
|
|
ASSERT_TRUE(names.insert(s.name).second);
|
|
PlannedCall p = CallPlanner::plan(s.name, dummy_args(s));
|
|
ASSERT_EQ(p.name(), std::string(s.name));
|
|
ASSERT_TRUE(p.path.find('{') == std::string::npos);
|
|
ASSERT_TRUE(p.path.front() == '/');
|
|
if (p.key == KeyKind::kManagement) ++mgmt;
|
|
else if (p.key == KeyKind::kNone) ++none;
|
|
else ++api;
|
|
}
|
|
ASSERT_EQ(mgmt + none + api, 140);
|
|
ASSERT_TRUE(mgmt > 50 && none >= 10 && api > 40);
|
|
ASSERT_TRUE(CallPlanner::spec("getModels").key == std::string("none"));
|
|
ASSERT_TRUE(CallPlanner::spec("getCredits").key == std::string("mgmt"));
|
|
ASSERT_TRUE(CallPlanner::spec("sendChatCompletionRequest").key == std::string("api"));
|
|
}
|
|
|
|
TEST(openrouter, planner_rejects_what_the_table_does_not_declare) {
|
|
Json body = Json::object();
|
|
body.set("nonsense", Json(1));
|
|
ASSERT_THROW(CallPlanner::plan("noSuchCall"), ApiError);
|
|
ASSERT_THROW(CallPlanner::plan("getKey"), ApiError); // missing {hash}
|
|
ASSERT_THROW(CallPlanner::plan("getKey", {{"hash", "a/b"}}), ApiError); // bad segment
|
|
ASSERT_THROW(CallPlanner::plan("getKey", {{"hash", "x"}, {"extra", "y"}}), ApiError);
|
|
ASSERT_THROW(CallPlanner::plan("getModels", {}, Query().add("bogus", "1")), ApiError);
|
|
ASSERT_THROW(CallPlanner::plan("sendChatCompletionRequest", {}, {}, &body), ApiError);
|
|
ASSERT_THROW(CallPlanner::plan("getCredits", {}, {}, &body), ApiError); // no body allowed
|
|
Json ok = Json::object();
|
|
ok.set("messages", Json::array());
|
|
PlannedCall p = CallPlanner::plan("sendChatCompletionRequest", {}, {}, &ok);
|
|
ASSERT_TRUE(p.has_body);
|
|
ASSERT_EQ(p.path, "/chat/completions");
|
|
PlannedCall q = CallPlanner::plan("getModels", {}, Query().add("q", "a b&c").add("limit", 5));
|
|
ASSERT_EQ(q.query, "?q=a%20b%26c&limit=5");
|
|
// Partner text is encoded, "." cannot be addressed.
|
|
ASSERT_EQ(CallPlanner::plan("getExternalUser", {{"user", "a b/c"}}).path, "/external-users/a%20b%2Fc");
|
|
ASSERT_THROW(CallPlanner::plan("getExternalUser", {{"user", ".."}}), ApiError);
|
|
// Alias authors keep their marker.
|
|
ASSERT_EQ(CallPlanner::plan("getModel", {{"author", "~openai"}, {"slug", "gpt-latest"}}).path,
|
|
"/model/~openai/gpt-latest");
|
|
auto [a, s] = split_model_id("openai/gpt-4o:free");
|
|
ASSERT_EQ(a, "openai");
|
|
ASSERT_EQ(s, "gpt-4o:free");
|
|
ASSERT_THROW(split_model_id("gpt-4o"), ApiError);
|
|
ASSERT_THROW(split_model_id("a/b/c"), ApiError);
|
|
}
|
|
|
|
TEST(openrouter, model_variants) {
|
|
ModelRef m = ModelRef::parse("poolside/laguna-s-2.1:free:nitro");
|
|
ASSERT_EQ(m.base(), "poolside/laguna-s-2.1");
|
|
ASSERT_EQ(m.catalog_variant(), "free");
|
|
ASSERT_EQ(m.catalog_id(), "poolside/laguna-s-2.1:free"); // :free is kept, :nitro dropped
|
|
ASSERT_EQ(m.str(), "poolside/laguna-s-2.1:free:nitro");
|
|
ASSERT_FALSE(m.uses_deprecated_variant());
|
|
ASSERT_TRUE(ModelRef::parse("openai/gpt-5.2:online").uses_deprecated_variant());
|
|
ASSERT_EQ(ModelRef::parse("openai/gpt-5.2:nitro:exacto").catalog_id(), "openai/gpt-5.2");
|
|
ASSERT_EQ(ModelRef::parse("~openai/gpt-latest").author(), "~openai");
|
|
ASSERT_THROW(ModelRef::parse("openai/gpt:free:thinking"), ApiError); // two catalog variants
|
|
ASSERT_THROW(ModelRef::parse("openai/gpt:bogus"), ApiError);
|
|
ASSERT_THROW(ModelRef::parse("gpt"), ApiError);
|
|
ASSERT_EQ(ModelRef::parse("a/b").with_variant("floor").with_variant("floor").str(), "a/b:floor");
|
|
ASSERT_EQ(ModelRef::parse("a/b:free:nitro").without_routing_variants().str(), "a/b:free");
|
|
}
|
|
|
|
TEST(openrouter, provider_preferences) {
|
|
ProviderPreferences p;
|
|
p.order = {"anthropic", "openai"};
|
|
p.allow_fallbacks = false;
|
|
p.require_parameters = true;
|
|
p.data_collection = "deny";
|
|
p.zdr = true;
|
|
p.quantizations = {"fp8", "bf16"};
|
|
p.sort = "throughput";
|
|
p.sort_partition = "none";
|
|
p.max_price = {{"prompt", "1"}, {"completion", "2.5"}};
|
|
p.preferred_max_latency_percentiles = {{"p90", 3.0}};
|
|
p.preferred_min_throughput = 50;
|
|
Json j = p.to_json();
|
|
ASSERT_EQ(j.find("order")->as_array().size(), 2u);
|
|
ASSERT_FALSE(j.find("allow_fallbacks")->as_bool());
|
|
ASSERT_EQ(j.find("sort")->find("by")->as_string(), "throughput");
|
|
ASSERT_EQ(j.find("max_price")->find("completion")->as_string(), "2.5");
|
|
ASSERT_EQ(j.find("preferred_max_latency")->find("p90")->as_number(), 3.0);
|
|
ASSERT_EQ(j.find("preferred_min_throughput")->as_number(), 50.0);
|
|
ProviderPreferences back = ProviderPreferences::from_json(j);
|
|
ASSERT_TRUE(back.to_json().equals(j));
|
|
ASSERT_TRUE(ProviderPreferences().empty());
|
|
ASSERT_EQ(ProviderPreferences::cheapest().to_json().find("sort")->as_string(), "price");
|
|
|
|
ProviderPreferences bad = p;
|
|
bad.quantizations = {"fp99"};
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = p;
|
|
bad.data_collection = "maybe";
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = p;
|
|
bad.sort = "cheap";
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = p;
|
|
bad.max_price["prompt"] = "-1";
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = p;
|
|
bad.max_price["tokens"] = "1";
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = p;
|
|
bad.preferred_min_throughput_percentiles = {{"p50", 1}};
|
|
ASSERT_THROW(bad.validate(), ApiError); // number and percentiles together
|
|
bad = ProviderPreferences();
|
|
bad.only.assign(Caps::kMaxProviderList + 1, "x");
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = ProviderPreferences();
|
|
bad.order = {"bad slug!"};
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
}
|
|
|
|
TEST(openrouter, reasoning_plugins_server_tools) {
|
|
ASSERT_EQ(ReasoningConfig::with_effort("high").to_json().find("effort")->as_string(), "high");
|
|
ASSERT_TRUE(ReasoningConfig::hidden("low").to_json().find("exclude")->as_bool());
|
|
ASSERT_EQ(ReasoningConfig::with_budget(2000).to_json().find("max_tokens")->as_int(), 2000);
|
|
ASSERT_THROW(ReasoningConfig::with_effort("extreme"), ApiError);
|
|
ReasoningConfig both;
|
|
both.effort = "low";
|
|
both.max_tokens = 10;
|
|
ASSERT_THROW(both.validate(), ApiError);
|
|
|
|
WebSearchPlugin web;
|
|
web.engine = "exa";
|
|
web.max_results = 3;
|
|
web.include_domains = {"*.example.com"};
|
|
Json w = web.to_json();
|
|
ASSERT_EQ(w.find("id")->as_string(), "web");
|
|
ASSERT_EQ(w.find("engine")->as_string(), "exa");
|
|
web.engine = "bing";
|
|
ASSERT_THROW(web.to_json(), ApiError);
|
|
FileParserPlugin fp;
|
|
fp.pdf_engine = "mistral-ocr";
|
|
ASSERT_EQ(fp.to_json().find("pdf")->find("engine")->as_string(), "mistral-ocr");
|
|
ResponseHealingPlugin heal;
|
|
heal.enabled = false;
|
|
ASSERT_FALSE(heal.to_json().find("enabled")->as_bool());
|
|
ASSERT_EQ(ContextCompressionPlugin().to_json().find("engine")->as_string(), "middle-out");
|
|
AutoRouterPlugin ar;
|
|
ar.allowed_models = {"anthropic/*"};
|
|
ar.cost_tier = "low";
|
|
ASSERT_EQ(ar.to_json().find("cost_tier")->as_string(), "low");
|
|
ASSERT_TRUE(Plugin::is_known("pareto-router"));
|
|
ASSERT_FALSE(Plugin::is_known("nope"));
|
|
ASSERT_THROW(RawPlugin("nope", Json::object()).to_json(), ApiError);
|
|
ASSERT_EQ(RawPlugin("pareto-router", Json::object()).to_json().find("id")->as_string(), "pareto-router");
|
|
std::vector<PluginPtr> twice = {std::make_shared<ResponseHealingPlugin>(), std::make_shared<ResponseHealingPlugin>()};
|
|
ASSERT_THROW(plugins_to_json(twice), ApiError);
|
|
std::vector<PluginPtr> many(Caps::kMaxPlugins + 1, std::make_shared<ModerationPlugin>());
|
|
ASSERT_THROW(plugins_to_json(many), ApiError);
|
|
|
|
ASSERT_EQ(ServerTools::web_search().find("type")->as_string(), "openrouter:web_search");
|
|
ASSERT_THROW(ServerTools::server("web_search"), ApiError);
|
|
ASSERT_THROW(ServerTools::server("openrouter:Bad Name"), ApiError);
|
|
Json fn = ServerTools::function("get_weather", "weather", Json::object(), true);
|
|
ASSERT_TRUE(fn.find("function")->find("strict")->as_bool());
|
|
ASSERT_THROW(ServerTools::function("bad name", "", Json::object()), ApiError);
|
|
}
|
|
|
|
TEST(openrouter, chat_params_emit_only_declared_fields) {
|
|
ChatParams p = ChatParams::simple("openai/gpt-4o:nitro", "hi");
|
|
p.models = {"anthropic/claude-3.5-sonnet"};
|
|
p.temperature = 0.5;
|
|
p.top_p = 0.9;
|
|
p.top_k = 40;
|
|
p.min_p = 0.1;
|
|
p.top_a = 0.1;
|
|
p.repetition_penalty = 1.1;
|
|
p.frequency_penalty = 0.1;
|
|
p.presence_penalty = 0.1;
|
|
p.max_tokens = 100;
|
|
p.max_completion_tokens = 100;
|
|
p.seed = 7;
|
|
p.logprobs = true;
|
|
p.top_logprobs = 5;
|
|
p.stop = {"END"};
|
|
p.logit_bias = Json::object();
|
|
p.response_format = ResponseFormats::json_schema("answer", Json::object());
|
|
p.tools.push(ServerTools::web_search());
|
|
p.tool_choice = Json("auto");
|
|
p.parallel_tool_calls = true;
|
|
p.provider = ProviderPreferences::cheapest();
|
|
p.reasoning = ReasoningConfig::with_effort("medium");
|
|
p.plugins = {std::make_shared<ResponseHealingPlugin>()};
|
|
p.route = "fallback";
|
|
p.modalities = {"text"};
|
|
p.service_tier = "flex";
|
|
p.user = "u1";
|
|
p.session_id = "s1";
|
|
p.prompt_cache_key = "k";
|
|
p.metadata = Json::object();
|
|
p.metadata.set("a", Json("b"));
|
|
p.trace = Json::object();
|
|
p.prediction = Json::object();
|
|
p.debug = Json::object();
|
|
Json body = p.to_json(true);
|
|
for (const auto& [k, v] : body.as_object()) {
|
|
(void)v;
|
|
if (!CallPlanner::accepts_field("sendChatCompletionRequest", k)) FAIL("undeclared key emitted: " + k);
|
|
}
|
|
ASSERT_TRUE(body.find("stream")->as_bool());
|
|
CallPlanner::plan("sendChatCompletionRequest", {}, {}, &body); // does not throw
|
|
|
|
ChatParams bad = ChatParams::simple("a/b", "x");
|
|
bad.temperature = 3;
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = ChatParams::simple("a/b", "x");
|
|
bad.reasoning = ReasoningConfig::with_effort("low");
|
|
bad.reasoning_effort = "low";
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = ChatParams::simple("a/b", "x");
|
|
bad.stop = {"1", "2", "3", "4", "5"};
|
|
ASSERT_THROW(bad.validate(), ApiError);
|
|
bad = ChatParams::simple("a/b", "x");
|
|
bad.extra.set("not_a_field", Json(1));
|
|
ASSERT_THROW(bad.to_json(), ApiError); // extra fields must be in the table
|
|
}
|