feat(clients): openai platform: 188-call table, 217 events, WebSocket sessions, checker, demo
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
#pragma once
|
||||
|
||||
// Test rig: a transport that RECORDS the requests the module builds and
|
||||
// answers a neutral "{}" (or a queued body for list walking). It never plays
|
||||
// a vendor: assertions are about what the module sends.
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "clients/task.h"
|
||||
#include "clients/transport.h"
|
||||
#include "llm/api_client.h"
|
||||
#include "openai/platform/platform.h"
|
||||
|
||||
namespace oai_test {
|
||||
|
||||
struct Seen {
|
||||
std::string method;
|
||||
std::string target; // path + query
|
||||
molo::clients::HeaderMap headers;
|
||||
std::string body;
|
||||
};
|
||||
|
||||
class CaptureTransport : public molo::clients::Transport {
|
||||
public:
|
||||
const char* name() const override { return "capture"; }
|
||||
molo::clients::Task<molo::clients::ClientResponse> send(const molo::clients::ClientRequest& request) override {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
seen.push_back({molo::clients::method_name(request.method()), request.url().target(),
|
||||
request.headers(), request.body_bytes()});
|
||||
}
|
||||
std::string body = "{}";
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
if (!queue.empty()) {
|
||||
body = queue.front();
|
||||
queue.pop_front();
|
||||
}
|
||||
}
|
||||
molo::clients::HeaderMap h;
|
||||
h.set("Content-Type", "application/json");
|
||||
co_return molo::clients::ClientResponse(200, h, body);
|
||||
}
|
||||
std::mutex mu;
|
||||
std::vector<Seen> seen;
|
||||
std::deque<std::string> queue; // optional bodies, first in first out
|
||||
};
|
||||
|
||||
struct Rig {
|
||||
CaptureTransport* transport;
|
||||
std::shared_ptr<molo::llm::ApiClient> client;
|
||||
std::shared_ptr<molo::openai::platform::Platform> platform;
|
||||
explicit Rig(const std::string& key = "sk-test-KEY") {
|
||||
auto t = std::make_unique<CaptureTransport>();
|
||||
transport = t.get();
|
||||
molo::llm::ApiClientConfig c;
|
||||
c.vendor = "openai";
|
||||
c.base_url = molo::openai::platform::PlatformConfig::kBaseUrl;
|
||||
c.key_env = "OPENAI_API_KEY";
|
||||
client = std::make_shared<molo::llm::ApiClient>(std::move(t), c, key);
|
||||
platform = std::make_shared<molo::openai::platform::Platform>(client);
|
||||
}
|
||||
const Seen& last() const { return transport->seen.back(); }
|
||||
std::size_t sent() const { return transport->seen.size(); }
|
||||
};
|
||||
|
||||
} // namespace oai_test
|
||||
@@ -0,0 +1,355 @@
|
||||
// OpenAI platform: the call table, local validation, request construction,
|
||||
// forms, list walking. Nothing here plays the vendor.
|
||||
#include <cstdlib>
|
||||
#include <set>
|
||||
|
||||
#include "framework.h"
|
||||
#include "llm/error.h"
|
||||
#include "openai/platform/batches_api.h"
|
||||
#include "openai/platform/caps.h"
|
||||
#include "openai/platform/uploads_api.h"
|
||||
#include "rig.h"
|
||||
|
||||
using namespace molo::openai::platform;
|
||||
using molo::clients::body::Json;
|
||||
using molo::llm::ApiError;
|
||||
using molo::llm::NotConfiguredError;
|
||||
using oai_test::Rig;
|
||||
|
||||
namespace {
|
||||
PathArgs dummy_ids(const CallSpec& s) {
|
||||
PathArgs a;
|
||||
for (const std::string& n : Calls::placeholders(s)) a.emplace_back(n, "id_1");
|
||||
return a;
|
||||
}
|
||||
Json minimal_body(const CallSpec& s) {
|
||||
Json b = Json::object();
|
||||
for (const std::string& r : Calls::list(s.required_fields)) b.set(r, Json("x"));
|
||||
return b;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(oai_calls, table_shape) {
|
||||
ASSERT_EQ(Calls::count(), 188u);
|
||||
std::set<std::string> ids, enums;
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
const CallSpec& s = Calls::at(i);
|
||||
ASSERT_EQ(static_cast<std::size_t>(s.op), i);
|
||||
ASSERT_TRUE(ids.insert(s.operation_id).second);
|
||||
ASSERT_TRUE(s.path[0] == '/');
|
||||
ASSERT_TRUE(std::string(s.method) == "GET" || std::string(s.method) == "POST" ||
|
||||
std::string(s.method) == "DELETE");
|
||||
ASSERT_TRUE(Calls::find(s.operation_id) == &s);
|
||||
}
|
||||
ASSERT_TRUE(Calls::find("nope") == nullptr);
|
||||
}
|
||||
|
||||
TEST(oai_calls, every_call_prepares_with_minimal_input) {
|
||||
for (std::size_t i = 0; i < Calls::count(); ++i) {
|
||||
const CallSpec& s = Calls::at(i);
|
||||
bool json = (s.req & kReqJson) != 0;
|
||||
Json body = minimal_body(s);
|
||||
auto p = PlatformService::prepare(s.op, dummy_ids(s), json ? &body : nullptr);
|
||||
ASSERT_EQ(p.method, std::string(s.method));
|
||||
ASSERT_EQ(p.target.find("{"), std::string::npos);
|
||||
ASSERT_EQ(p.target.substr(0, 1), "/");
|
||||
ASSERT_EQ(p.has_body, json);
|
||||
// Flags drive the beta headers.
|
||||
ASSERT_EQ(p.headers.count("OpenAI-Beta") != 0, Calls::has(s, kBetaAssistants) || Calls::has(s, kBetaChatkit));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(oai_calls, unknown_and_missing_body_fields_are_rejected_before_the_network) {
|
||||
Rig r;
|
||||
Json bad = Json::object();
|
||||
bad.set("model", Json("gpt-x"));
|
||||
bad.set("not_a_field", Json(1));
|
||||
ASSERT_THROW(r.platform->assistants.create(bad), ApiError);
|
||||
ASSERT_THROW(r.platform->assistants.create(Json::object()), ApiError); // model is required
|
||||
ASSERT_THROW(r.platform->fine_tuning_jobs.create(Json::object()), ApiError);
|
||||
ASSERT_THROW(r.platform->files.retrieve("a/b"), ApiError);
|
||||
ASSERT_THROW(r.platform->files.retrieve(".."), ApiError);
|
||||
ASSERT_THROW(r.platform->files.retrieve(""), ApiError);
|
||||
ASSERT_THROW(r.platform->files.retrieve(std::string(300, 'a')), ApiError);
|
||||
ASSERT_THROW(r.platform->generic.invoke(Op::kRetrieveFile, {}), ApiError); // id missing
|
||||
ASSERT_THROW(r.platform->generic.invoke(Op::kRetrieveFile, {{"file_id", "a"}, {"x", "b"}}), ApiError); // extra id
|
||||
Json body = Json::object();
|
||||
ASSERT_THROW(r.platform->generic.invoke(Op::kRetrieveFile, {{"file_id", "a"}}, &body), ApiError); // GET body
|
||||
ASSERT_EQ(r.sent(), 0u);
|
||||
}
|
||||
|
||||
TEST(oai_calls, validation_runs_before_the_credential_check) {
|
||||
Rig r("");
|
||||
ASSERT_FALSE(r.platform->configured());
|
||||
try {
|
||||
r.platform->files.retrieve("a/b");
|
||||
FAIL("expected invalid");
|
||||
} catch (const NotConfiguredError&) {
|
||||
FAIL("validation must come first");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 400);
|
||||
}
|
||||
try {
|
||||
r.platform->files.retrieve("file-1");
|
||||
FAIL("expected not configured");
|
||||
} catch (const NotConfiguredError& e) {
|
||||
ASSERT_EQ(e.http_status(), 503);
|
||||
ASSERT_TRUE(e.message().find("OPENAI_API_KEY") != std::string::npos);
|
||||
}
|
||||
ASSERT_EQ(r.sent(), 0u);
|
||||
}
|
||||
|
||||
TEST(oai_calls, requests_carry_the_expected_method_path_auth_and_beta_headers) {
|
||||
Rig r;
|
||||
r.platform->files.retrieve("file-abc");
|
||||
ASSERT_EQ(r.last().method, "GET");
|
||||
ASSERT_EQ(r.last().target, "/v1/files/file-abc");
|
||||
ASSERT_EQ(r.last().headers.get_or("Authorization", ""), "Bearer sk-test-KEY");
|
||||
ASSERT_EQ(r.last().headers.get_or("OpenAI-Beta", ""), "");
|
||||
ASSERT_EQ(r.last().target.find("sk-test"), std::string::npos);
|
||||
|
||||
r.platform->threads.retrieve("thread_1");
|
||||
ASSERT_EQ(r.last().target, "/v1/threads/thread_1");
|
||||
ASSERT_EQ(r.last().headers.get_or("OpenAI-Beta", ""), "assistants=v2");
|
||||
|
||||
r.platform->chatkit.retrieve_thread("cthr_1");
|
||||
ASSERT_EQ(r.last().target, "/v1/chatkit/threads/cthr_1");
|
||||
ASSERT_EQ(r.last().headers.get_or("OpenAI-Beta", ""), "chatkit_beta=v1");
|
||||
|
||||
r.platform->files.remove("file-abc");
|
||||
ASSERT_EQ(r.last().method, "DELETE");
|
||||
|
||||
r.platform->fine_tuning_jobs.cancel("ftjob-1");
|
||||
ASSERT_EQ(r.last().method, "POST");
|
||||
ASSERT_EQ(r.last().target, "/v1/fine_tuning/jobs/ftjob-1/cancel");
|
||||
|
||||
Json body = Json::object();
|
||||
body.set("project_ids", Json::array());
|
||||
r.platform->fine_tuning_permissions.remove("ft:gpt-4o:org::abc:ckpt-step-1", "cp_1");
|
||||
ASSERT_EQ(r.last().target, "/v1/fine_tuning/checkpoints/ft:gpt-4o:org::abc:ckpt-step-1/permissions/cp_1");
|
||||
}
|
||||
|
||||
TEST(oai_calls, org_and_project_headers_come_from_config) {
|
||||
setenv("OPENAI_ORG_ID", "org-test_1", 1);
|
||||
setenv("OPENAI_PROJECT_ID", "proj_test-2", 1);
|
||||
auto cfg = PlatformConfig::api_config();
|
||||
ASSERT_EQ(cfg.default_headers["OpenAI-Organization"], "org-test_1");
|
||||
ASSERT_EQ(cfg.default_headers["OpenAI-Project"], "proj_test-2");
|
||||
ASSERT_EQ(cfg.key_env, "OPENAI_API_KEY");
|
||||
setenv("OPENAI_ORG_ID", "bad\r\nX: y", 1);
|
||||
ASSERT_THROW(PlatformConfig::api_config(), ApiError);
|
||||
unsetenv("OPENAI_ORG_ID");
|
||||
unsetenv("OPENAI_PROJECT_ID");
|
||||
ASSERT_TRUE(PlatformConfig::api_config().default_headers.empty());
|
||||
}
|
||||
|
||||
TEST(oai_calls, query_strings) {
|
||||
Rig r;
|
||||
Json q = Json::object();
|
||||
q.set("limit", Json(5));
|
||||
q.set("order", Json("desc"));
|
||||
q.set("after", Json("file a&b"));
|
||||
r.platform->files.list(q);
|
||||
ASSERT_EQ(r.last().target, "/v1/files?after=file%20a%26b&limit=5&order=desc");
|
||||
Json bad = Json::object();
|
||||
bad.set("nope", Json(1));
|
||||
ASSERT_THROW(r.platform->files.list(bad), ApiError);
|
||||
Json big = Json::object();
|
||||
big.set("limit", Json(10001));
|
||||
ASSERT_THROW(r.platform->files.list(big), ApiError);
|
||||
Json zero = Json::object();
|
||||
zero.set("limit", Json(0));
|
||||
ASSERT_THROW(r.platform->files.list(zero), ApiError);
|
||||
// array: repeated key; deepObject: key[sub]
|
||||
Json inc = Json::object();
|
||||
Json arr = Json::array();
|
||||
arr.push(Json("a.b"));
|
||||
arr.push(Json("c"));
|
||||
inc.set("include", arr);
|
||||
r.platform->conversation_items.list("conv_1", inc);
|
||||
ASSERT_EQ(r.last().target, "/v1/conversations/conv_1/items?include=a.b&include=c");
|
||||
Json meta = Json::object();
|
||||
Json m = Json::object();
|
||||
m.set("team", Json("x y"));
|
||||
meta.set("metadata", m);
|
||||
r.platform->fine_tuning_jobs.list(meta);
|
||||
ASSERT_EQ(r.last().target, "/v1/fine_tuning/jobs?metadata%5Bteam%5D=x%20y");
|
||||
Json steps = Json::object();
|
||||
Json sa = Json::array();
|
||||
sa.push(Json("step_details.tool_calls[*].file_search.results[*].content"));
|
||||
steps.set("include[]", sa);
|
||||
r.platform->runs.steps("t_1", "run_1", steps);
|
||||
ASSERT_TRUE(r.last().target.find("include%5B%5D=step_details.tool_calls%5B%2A%5D") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(oai_calls, idempotency_key_only_where_the_spec_allows) {
|
||||
Rig r;
|
||||
Json body = Json::object();
|
||||
Json env = Json::object();
|
||||
env.set("type", Json("x"));
|
||||
body.set("environment", env);
|
||||
r.platform->agent_environments.create(body, "key-123");
|
||||
ASSERT_EQ(r.last().headers.get_or("Idempotency-Key", ""), "key-123");
|
||||
ASSERT_THROW(r.platform->generic.invoke(Op::kRetrieveFile, {{"file_id", "a"}}, nullptr, Json::object(),
|
||||
{{"Idempotency-Key", "k"}}),
|
||||
ApiError);
|
||||
ASSERT_THROW(r.platform->agent_environments.create(body, "has space"), ApiError);
|
||||
ASSERT_THROW(r.platform->generic.invoke(Op::kRetrieveFile, {{"file_id", "a"}}, nullptr, Json::object(),
|
||||
{{"OpenAI-Beta", "x"}}),
|
||||
ApiError);
|
||||
}
|
||||
|
||||
TEST(oai_forms, files_create_is_multipart_with_bracket_notation) {
|
||||
Rig r;
|
||||
Json exp = Json::object();
|
||||
exp.set("anchor", Json("created_at"));
|
||||
exp.set("seconds", Json(3600));
|
||||
r.platform->files.create("data.jsonl", "{\"a\":1}\n", "fine-tune", "application/jsonl", &exp);
|
||||
ASSERT_EQ(r.last().method, "POST");
|
||||
ASSERT_EQ(r.last().target, "/v1/files");
|
||||
ASSERT_TRUE(r.last().headers.get_or("Content-Type", "").rfind("multipart/form-data; boundary=", 0) == 0);
|
||||
const std::string& b = r.last().body;
|
||||
ASSERT_TRUE(b.find("name=\"file\"; filename=\"data.jsonl\"") != std::string::npos);
|
||||
ASSERT_TRUE(b.find("name=\"purpose\"") != std::string::npos);
|
||||
ASSERT_TRUE(b.find("fine-tune") != std::string::npos);
|
||||
ASSERT_TRUE(b.find("name=\"expires_after[anchor]\"") != std::string::npos);
|
||||
ASSERT_TRUE(b.find("name=\"expires_after[seconds]\"") != std::string::npos);
|
||||
ASSERT_TRUE(b.find("3600") != std::string::npos);
|
||||
ASSERT_THROW(r.platform->files.create("x", "d", "bogus-purpose"), ApiError);
|
||||
ASSERT_THROW(r.platform->files.create("a/b", "d", "batch"), ApiError);
|
||||
ASSERT_THROW(r.platform->files.create("a\"b", "d", "batch"), ApiError);
|
||||
ASSERT_THROW(r.platform->files.create("", "d", "batch"), ApiError);
|
||||
}
|
||||
|
||||
TEST(oai_forms, form_fields_are_checked_against_the_table) {
|
||||
Form f;
|
||||
f.text("purpose", "batch");
|
||||
ASSERT_THROW(Calls::check_form(Op::kCreateFile, f.names()), ApiError); // file missing
|
||||
Form g;
|
||||
g.file("file", "a", "text/plain", "x");
|
||||
g.text("purpose", "batch");
|
||||
g.text("surprise", "1");
|
||||
ASSERT_THROW(Calls::check_form(Op::kCreateFile, g.names()), ApiError);
|
||||
ASSERT_THROW(Calls::check_form(Op::kListFiles, {}), ApiError); // not multipart
|
||||
Form skills;
|
||||
skills.file("files[]", "SKILL.md", "text/markdown", "# x");
|
||||
Calls::check_form(Op::kCreateSkill, skills.names());
|
||||
}
|
||||
|
||||
TEST(oai_uploads, plan_and_sequence) {
|
||||
auto plan = UploadPlan::make(10, 4);
|
||||
ASSERT_EQ(plan.parts.size(), 3u);
|
||||
ASSERT_EQ(plan.parts[2].offset, 8u);
|
||||
ASSERT_EQ(plan.parts[2].size, 2u);
|
||||
ASSERT_THROW(UploadPlan::make(0, 4), ApiError);
|
||||
ASSERT_THROW(UploadPlan::make(10, 0), ApiError);
|
||||
ASSERT_THROW(UploadPlan::make(10, Caps::kMaxUploadPartBytes + 1), ApiError);
|
||||
ASSERT_THROW(UploadPlan::make(Caps::kMaxFileBytes + 1, Caps::kMaxUploadPartBytes), ApiError);
|
||||
ASSERT_THROW(UploadPlan::make(Caps::kMaxUploadParts + 1, 1), ApiError);
|
||||
|
||||
// The module's own request sequence for a three-part upload. The queued
|
||||
// bodies only carry the ids the next call needs (documented field `id`).
|
||||
Rig r;
|
||||
r.transport->queue = {"{\"id\":\"upload_1\"}", "{\"id\":\"part_1\"}", "{\"id\":\"part_2\"}",
|
||||
"{\"id\":\"part_3\"}", "{\"id\":\"upload_1\",\"status\":\"completed\"}"};
|
||||
r.platform->uploads.upload_all("big.jsonl", "application/jsonl", "batch", "0123456789", 4);
|
||||
ASSERT_EQ(r.sent(), 5u);
|
||||
ASSERT_EQ(r.transport->seen[0].target, "/v1/uploads");
|
||||
ASSERT_TRUE(r.transport->seen[0].body.find("\"bytes\":10") != std::string::npos);
|
||||
ASSERT_EQ(r.transport->seen[1].target, "/v1/uploads/upload_1/parts");
|
||||
ASSERT_EQ(r.transport->seen[3].target, "/v1/uploads/upload_1/parts");
|
||||
ASSERT_EQ(r.transport->seen[4].target, "/v1/uploads/upload_1/complete");
|
||||
ASSERT_EQ(r.transport->seen[4].body, "{\"part_ids\":[\"part_1\",\"part_2\",\"part_3\"]}");
|
||||
}
|
||||
|
||||
TEST(oai_batches, jsonl_builder_caps) {
|
||||
BatchInput in("/v1/chat/completions");
|
||||
Json body = Json::object();
|
||||
body.set("model", Json("gpt-x"));
|
||||
in.add("req-1", body).add("req-2", body);
|
||||
ASSERT_EQ(in.size(), 2u);
|
||||
ASSERT_EQ(in.jsonl().substr(0, in.jsonl().find('\n')),
|
||||
"{\"body\":{\"model\":\"gpt-x\"},\"custom_id\":\"req-1\",\"method\":\"POST\",\"url\":\"/v1/chat/completions\"}");
|
||||
ASSERT_THROW(in.add("req-1", body), ApiError);
|
||||
ASSERT_THROW(in.add("", body), ApiError);
|
||||
ASSERT_THROW(in.add(std::string(513, 'x'), body), ApiError);
|
||||
ASSERT_THROW(in.add("r3", Json("not an object")), ApiError);
|
||||
ASSERT_THROW(BatchInput("/v1/nope"), ApiError);
|
||||
Rig r;
|
||||
ASSERT_THROW(r.platform->batches.create("file-1", "/v1/nope"), ApiError);
|
||||
r.platform->batches.create("file-1", "/v1/embeddings");
|
||||
ASSERT_EQ(r.last().body, "{\"completion_window\":\"24h\",\"endpoint\":\"/v1/embeddings\",\"input_file_id\":\"file-1\"}");
|
||||
}
|
||||
|
||||
TEST(oai_conversations, twenty_items_at_most) {
|
||||
Rig r;
|
||||
Json body = Json::object();
|
||||
Json items = Json::array();
|
||||
for (int i = 0; i < 21; ++i) items.push(Json::object());
|
||||
body.set("items", items);
|
||||
ASSERT_THROW(r.platform->conversation_items.create("conv_1", body), ApiError);
|
||||
ASSERT_THROW(r.platform->conversations.create(body), ApiError);
|
||||
ASSERT_EQ(r.sent(), 0u);
|
||||
}
|
||||
|
||||
TEST(oai_list, walks_cursor_pages_and_reports_truncation) {
|
||||
// Page shape per the spec's list schemas: data[], has_more, last_id.
|
||||
Rig r;
|
||||
r.transport->queue = {"{\"data\":[{\"id\":\"a\"},{\"id\":\"b\"}],\"has_more\":true,\"last_id\":\"b\"}",
|
||||
"{\"data\":[{\"id\":\"c\"}],\"has_more\":false,\"last_id\":\"c\"}"};
|
||||
auto l = r.platform->generic.list_all(Op::kListFiles, {}, Json::object(), 5);
|
||||
ASSERT_EQ(l.pages, 2u);
|
||||
ASSERT_EQ(l.data.as_array().size(), 3u);
|
||||
ASSERT_FALSE(l.truncated);
|
||||
ASSERT_EQ(r.transport->seen[1].target, "/v1/files?after=b");
|
||||
|
||||
Rig t;
|
||||
t.transport->queue = {"{\"data\":[{\"id\":\"a\"}],\"has_more\":true,\"last_id\":\"a\"}",
|
||||
"{\"data\":[{\"id\":\"b\"}],\"has_more\":true,\"last_id\":\"b\"}"};
|
||||
auto cut = t.platform->generic.list_all(Op::kListFiles, {}, Json::object(), 2);
|
||||
ASSERT_TRUE(cut.truncated);
|
||||
ASSERT_TRUE(cut.has_more);
|
||||
ASSERT_EQ(cut.stop_reason, "page_cap");
|
||||
ASSERT_EQ(cut.pages, 2u);
|
||||
|
||||
ASSERT_THROW(t.platform->generic.list_all(Op::kRetrieveFile, {{"file_id", "a"}}), ApiError); // not a list
|
||||
ASSERT_THROW(t.platform->generic.list_all(Op::kListFiles, {}, Json::object(), 0), ApiError);
|
||||
}
|
||||
|
||||
TEST(oai_streams, only_streamable_calls_stream) {
|
||||
Rig r;
|
||||
Json body = Json::object();
|
||||
body.set("assistant_id", Json("asst_1"));
|
||||
ASSERT_THROW(r.platform->generic.invoke_stream(Op::kRetrieveFile, {{"file_id", "a"}}, Json::object(),
|
||||
[](const molo::llm::SseEvent&) { return true; }),
|
||||
ApiError);
|
||||
// A streamable call sets "stream": true before sending.
|
||||
r.transport->queue = {"data: [DONE]\n\n"};
|
||||
r.platform->runs.create_stream("thread_1", body, [](const molo::llm::SseEvent&) { return true; });
|
||||
ASSERT_EQ(r.last().target, "/v1/threads/thread_1/runs");
|
||||
ASSERT_TRUE(r.last().body.find("\"stream\":true") != std::string::npos);
|
||||
ASSERT_EQ(r.last().headers.get_or("OpenAI-Beta", ""), "assistants=v2");
|
||||
}
|
||||
|
||||
TEST(oai_realtime_rest, call_and_secret_requests) {
|
||||
Rig r;
|
||||
r.platform->realtime.create_client_secret();
|
||||
ASSERT_EQ(r.last().target, "/v1/realtime/client_secrets");
|
||||
ASSERT_THROW(r.platform->realtime.create_call("not sdp"), ApiError);
|
||||
r.platform->realtime.create_call("v=0\r\no=- 1 1 IN IP4 0.0.0.0\r\n");
|
||||
ASSERT_EQ(r.last().target, "/v1/realtime/calls");
|
||||
ASSERT_EQ(r.last().headers.get_or("Content-Type", ""), "application/sdp");
|
||||
r.platform->realtime.create_call("v=0\r\n", Json::object());
|
||||
ASSERT_TRUE(r.last().headers.get_or("Content-Type", "").rfind("multipart/form-data", 0) == 0);
|
||||
ASSERT_TRUE(r.last().body.find("name=\"sdp\"") != std::string::npos);
|
||||
ASSERT_TRUE(r.last().body.find("name=\"session\"") != std::string::npos);
|
||||
ASSERT_THROW(r.platform->realtime.refer_call("call_1", "http://x"), ApiError);
|
||||
r.platform->realtime.refer_call("call_1", "tel:+14155550123");
|
||||
ASSERT_EQ(r.last().body, "{\"target_uri\":\"tel:+14155550123\"}");
|
||||
ASSERT_THROW(r.platform->realtime.reject_call("call_1", 200), ApiError);
|
||||
r.platform->realtime.reject_call("call_1", 486);
|
||||
ASSERT_EQ(r.last().body, "{\"status_code\":486}");
|
||||
r.platform->realtime.hangup_call("call_1");
|
||||
ASSERT_EQ(r.last().target, "/v1/realtime/calls/call_1/hangup");
|
||||
ASSERT_THROW(r.platform->live.reject("s_1", 0), ApiError);
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// OpenAI platform: event catalogs, client event construction, server event
|
||||
// parsing, and the WebSocket session over a test-owned loopback server.
|
||||
#include <cstdlib>
|
||||
|
||||
#include "../../google/live/loopback.h"
|
||||
#include "framework.h"
|
||||
#include "llm/error.h"
|
||||
#include "openai/platform/calls.h"
|
||||
#include "openai/platform/caps.h"
|
||||
#include "openai/platform/event_socket.h"
|
||||
#include "openai/platform/events.h"
|
||||
|
||||
using namespace molo::openai::platform;
|
||||
using molo::clients::body::Json;
|
||||
using molo::llm::ApiError;
|
||||
using molo::llm::NotConfiguredError;
|
||||
using live_test::LoopbackServer;
|
||||
using live_test::ServerConn;
|
||||
|
||||
namespace {
|
||||
EventSocketOptions local(const LoopbackServer& s, const std::string& key = "sk-test-KEY") {
|
||||
EventSocketOptions o;
|
||||
o.base_url = "ws://127.0.0.1:" + std::to_string(s.port());
|
||||
o.api_key = key;
|
||||
return o;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(oai_events, catalog_counts_per_profile) {
|
||||
ASSERT_EQ(Events::count(), 217u);
|
||||
ASSERT_EQ(Events::catalog(Profile::kGa, Direction::kClientEvent).size(), 12u);
|
||||
ASSERT_EQ(Events::catalog(Profile::kTranslation, Direction::kClientEvent).size(), 3u);
|
||||
ASSERT_TRUE(Events::find(Profile::kGa, Direction::kClientEvent, "session.update") != nullptr);
|
||||
ASSERT_TRUE(Events::find(Profile::kGa, Direction::kClientEvent, "response.create") != nullptr);
|
||||
ASSERT_TRUE(Events::find(Profile::kGa, Direction::kServerEvent, "response.output_audio.delta") != nullptr);
|
||||
ASSERT_TRUE(Events::find(Profile::kBeta, Direction::kServerEvent, "response.output_audio.delta") != nullptr);
|
||||
ASSERT_TRUE(Events::find(Profile::kGa, Direction::kClientEvent, "response.audio.delta") == nullptr);
|
||||
ASSERT_TRUE(Events::find(Profile::kLive, Direction::kClientEvent, "session.start") != nullptr);
|
||||
ASSERT_TRUE(Events::find(Profile::kLiveSideband, Direction::kClientEvent, "session.start") == nullptr);
|
||||
Profile p;
|
||||
ASSERT_TRUE(Events::parse_profile("live_sideband", p));
|
||||
ASSERT_TRUE(p == Profile::kLiveSideband);
|
||||
ASSERT_FALSE(Events::parse_profile("nope", p));
|
||||
}
|
||||
|
||||
TEST(oai_events, every_client_event_builds_from_its_required_fields) {
|
||||
for (std::size_t i = 0; i < Events::count(); ++i) {
|
||||
const EventSpec& e = Events::at(i);
|
||||
if (e.direction != Direction::kClientEvent) continue;
|
||||
Json f = Json::object();
|
||||
for (const std::string& r : Calls::list(e.required))
|
||||
if (r != "type") f.set(r, Json("x"));
|
||||
Json ev = Events::make_client(e.profile, e.type, f);
|
||||
ASSERT_EQ(ev.find("type")->as_string(), std::string(e.type));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(oai_events, client_event_validation) {
|
||||
Json f = Json::object();
|
||||
f.set("session", Json::object());
|
||||
Json ev = Events::make_client(Profile::kGa, "session.update", f, "evt_1");
|
||||
ASSERT_EQ(ev.dump(), "{\"event_id\":\"evt_1\",\"session\":{},\"type\":\"session.update\"}");
|
||||
ASSERT_THROW(Events::make_client(Profile::kGa, "session.update"), ApiError);
|
||||
ASSERT_THROW(Events::make_client(Profile::kGa, "session.nope", f), ApiError);
|
||||
ASSERT_THROW(Events::make_client(Profile::kGa, "response.audio.delta", f), ApiError);
|
||||
Json extra = f;
|
||||
extra.set("surprise", Json(1));
|
||||
ASSERT_THROW(Events::make_client(Profile::kGa, "session.update", extra), ApiError);
|
||||
Json typed = f;
|
||||
typed.set("type", Json("x"));
|
||||
ASSERT_THROW(Events::make_client(Profile::kGa, "session.update", typed), ApiError);
|
||||
ASSERT_THROW(Events::make_client(Profile::kGa, "session.update", f, std::string(513, 'e')), ApiError);
|
||||
|
||||
Json a = Events::audio_append(Profile::kGa, "abc");
|
||||
ASSERT_EQ(a.dump(), "{\"audio\":\"YWJj\",\"type\":\"input_audio_buffer.append\"}");
|
||||
ASSERT_EQ(Events::audio_append(Profile::kTranslation, "abc").find("type")->as_string(),
|
||||
"session.input_audio_buffer.append");
|
||||
ASSERT_EQ(Events::audio_append(Profile::kLive, "abc").find("type")->as_string(), "session.input_audio.append");
|
||||
ASSERT_THROW(Events::audio_append(Profile::kLiveSideband, "abc"), ApiError);
|
||||
ASSERT_THROW(Events::audio_append(Profile::kGa, ""), ApiError);
|
||||
ASSERT_THROW(Events::audio_append(Profile::kGa, std::string(Caps::kMaxAudioChunkBytes + 1, 'a')), ApiError);
|
||||
}
|
||||
|
||||
TEST(oai_events, server_event_parsing) {
|
||||
auto ok = Events::parse_server(Profile::kGa, "{\"type\":\"session.created\",\"event_id\":\"event_1\",\"session\":{}}");
|
||||
ASSERT_TRUE(ok.known);
|
||||
ASSERT_EQ(ok.event_id, "event_1");
|
||||
ASSERT_FALSE(ok.is_error);
|
||||
auto err = Events::parse_server(
|
||||
Profile::kGa, "{\"type\":\"error\",\"event_id\":\"e\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"nope\"}}");
|
||||
ASSERT_TRUE(err.is_error);
|
||||
ASSERT_EQ(err.error_message, "nope");
|
||||
auto unk = Events::parse_server(Profile::kGa, "{\"type\":\"future.event\"}");
|
||||
ASSERT_FALSE(unk.known);
|
||||
ASSERT_THROW(Events::parse_server(Profile::kGa, "not json"), ApiError);
|
||||
ASSERT_THROW(Events::parse_server(Profile::kGa, "{\"no\":\"type\"}"), ApiError);
|
||||
ASSERT_THROW(Events::parse_server(Profile::kGa, "[1]"), ApiError);
|
||||
}
|
||||
|
||||
TEST(oai_ws, urls_per_endpoint) {
|
||||
WsParams p;
|
||||
p.model = "gpt-realtime";
|
||||
ASSERT_EQ(EventSocket::url_for(WsKind::kRealtime, "wss://api.openai.com/v1", p),
|
||||
"wss://api.openai.com/v1/realtime?model=gpt-realtime");
|
||||
WsParams c;
|
||||
c.call_id = "rtc_1";
|
||||
ASSERT_EQ(EventSocket::url_for(WsKind::kRealtime, "wss://h/v1", c), "wss://h/v1/realtime?call_id=rtc_1");
|
||||
ASSERT_EQ(EventSocket::url_for(WsKind::kRealtimeTranslation, "wss://h/v1", p),
|
||||
"wss://h/v1/realtime/translations?model=gpt-realtime");
|
||||
ASSERT_EQ(EventSocket::url_for(WsKind::kLiveSession, "wss://h/v1", WsParams{}), "wss://h/v1/live/sessions");
|
||||
WsParams s;
|
||||
s.session_id = "live_1";
|
||||
s.graceful_close = true;
|
||||
ASSERT_EQ(EventSocket::url_for(WsKind::kLiveSideband, "wss://h/v1", s),
|
||||
"wss://h/v1/live/sessions/live_1/attach?graceful_close=true");
|
||||
WsParams fork;
|
||||
fork.session_id = "live_1";
|
||||
ASSERT_EQ(EventSocket::url_for(WsKind::kLiveFork, "wss://h/v1", fork), "wss://h/v1/live/sessions/live_1/fork");
|
||||
ASSERT_THROW(EventSocket::url_for(WsKind::kRealtime, "wss://h/v1", WsParams{}), ApiError);
|
||||
ASSERT_THROW(EventSocket::url_for(WsKind::kLiveSession, "wss://h/v1", p), ApiError);
|
||||
ASSERT_THROW(EventSocket::url_for(WsKind::kLiveSideband, "wss://h/v1", WsParams{}), ApiError);
|
||||
WsParams evil;
|
||||
evil.model = "a&b=c";
|
||||
ASSERT_THROW(EventSocket::url_for(WsKind::kRealtime, "wss://h/v1", evil), ApiError);
|
||||
ASSERT_THROW(EventSocket::url_for(WsKind::kRealtime, "https://h/v1", p), ApiError);
|
||||
ASSERT_THROW(EventSocket::url_for(WsKind::kRealtime, "wss://h/v1/", p), ApiError);
|
||||
ASSERT_EQ(EventSocket::endpoint_count(), 6u);
|
||||
}
|
||||
|
||||
TEST(oai_ws, no_key_means_503_before_any_traffic) {
|
||||
setenv("MOLO_ENV_FILES", "off", 1);
|
||||
unsetenv("OPENAI_API_KEY");
|
||||
EventSocketOptions o;
|
||||
EventSocket s(WsKind::kRealtime, o);
|
||||
WsParams p;
|
||||
p.model = "gpt-realtime";
|
||||
try {
|
||||
s.connect(p);
|
||||
FAIL("expected NotConfiguredError");
|
||||
} catch (const NotConfiguredError& e) {
|
||||
ASSERT_EQ(e.http_status(), 503);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(oai_ws, session_over_a_loopback_server) {
|
||||
// The server side is test-owned and speaks event shapes from the spec.
|
||||
std::string seen_type;
|
||||
LoopbackServer server([&](ServerConn& c) {
|
||||
c.send_text("{\"type\":\"session.created\",\"event_id\":\"event_1\",\"session\":{\"id\":\"sess_1\"}}");
|
||||
if (auto m = c.read_message()) seen_type = *m;
|
||||
c.send_text("{\"type\":\"response.output_audio.delta\",\"event_id\":\"event_2\",\"response_id\":\"r\","
|
||||
"\"item_id\":\"i\",\"output_index\":0,\"content_index\":0,\"delta\":\"YWJj\"}");
|
||||
c.send_text("{\"type\":\"a.future.event\"}");
|
||||
c.send_close(1000, "bye");
|
||||
});
|
||||
EventSocket sock(WsKind::kRealtime, local(server));
|
||||
WsParams p;
|
||||
p.model = "gpt-realtime";
|
||||
sock.connect(p);
|
||||
ASSERT_TRUE(server.request().find("GET /realtime?model=gpt-realtime ") != std::string::npos);
|
||||
ASSERT_TRUE(server.request().find("Authorization: Bearer sk-test-KEY") != std::string::npos);
|
||||
ASSERT_TRUE(server.request().find("OpenAI-Beta") == std::string::npos);
|
||||
|
||||
Json f = Json::object();
|
||||
f.set("session", Json::object());
|
||||
sock.send(Events::make_client(Profile::kGa, "session.update", f));
|
||||
ASSERT_THROW(sock.send(Json::object()), ApiError); // no type
|
||||
Json wrong = Json::object();
|
||||
wrong.set("type", Json("response.audio.delta"));
|
||||
ASSERT_THROW(sock.send(wrong), ApiError); // a server event is not sendable
|
||||
|
||||
std::vector<std::string> types;
|
||||
RunStats st = sock.run(
|
||||
[&](const ParsedEvent& e) {
|
||||
types.push_back(e.type);
|
||||
return true;
|
||||
},
|
||||
10);
|
||||
ASSERT_EQ(types.size(), 3u);
|
||||
ASSERT_EQ(types[0], "session.created");
|
||||
ASSERT_EQ(types[1], "response.output_audio.delta");
|
||||
ASSERT_EQ(st.unknown, 1u);
|
||||
ASSERT_EQ(st.stop_reason, "closed");
|
||||
ASSERT_EQ(st.close_code, 1000);
|
||||
ASSERT_TRUE(seen_type.find("\"type\":\"session.update\"") != std::string::npos);
|
||||
}
|
||||
|
||||
TEST(oai_ws, beta_header_and_sink_stop) {
|
||||
LoopbackServer server([&](ServerConn& c) {
|
||||
for (int i = 0; i < 5; ++i) c.send_text("{\"type\":\"session.created\"}");
|
||||
c.read_message();
|
||||
});
|
||||
EventSocket sock(WsKind::kRealtimeBeta, local(server));
|
||||
WsParams p;
|
||||
p.model = "gpt-4o-realtime-preview";
|
||||
sock.connect(p);
|
||||
ASSERT_TRUE(server.request().find("OpenAI-Beta: realtime=v1") != std::string::npos);
|
||||
int n = 0;
|
||||
RunStats st = sock.run([&](const ParsedEvent&) { return ++n < 2; }, 5);
|
||||
ASSERT_EQ(st.stop_reason, "sink_stopped");
|
||||
ASSERT_EQ(st.events, 2u);
|
||||
sock.close();
|
||||
ASSERT_FALSE(sock.connected());
|
||||
}
|
||||
|
||||
TEST(oai_ws, refused_upgrade_maps_to_http_status_and_hides_the_key) {
|
||||
LoopbackServer server([](ServerConn&) {}, LoopbackServer::Handshake::Status403);
|
||||
EventSocket sock(WsKind::kRealtime, local(server, "sk-secret-123"));
|
||||
WsParams p;
|
||||
p.model = "gpt-realtime";
|
||||
try {
|
||||
sock.connect(p);
|
||||
FAIL("expected refusal");
|
||||
} catch (const ApiError& e) {
|
||||
ASSERT_EQ(e.http_status(), 403);
|
||||
ASSERT_EQ(std::string(e.what()).find("sk-secret"), std::string::npos);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(oai_ws, run_times_out_without_events) {
|
||||
LoopbackServer server([](ServerConn& c) { c.read_message(); });
|
||||
EventSocket sock(WsKind::kLiveSession, local(server));
|
||||
sock.connect(WsParams{});
|
||||
RunStats st = sock.run([](const ParsedEvent&) { return true; }, 1);
|
||||
ASSERT_EQ(st.stop_reason, "timeout");
|
||||
ASSERT_THROW(sock.run([](const ParsedEvent&) { return true; }, 0), ApiError);
|
||||
ASSERT_THROW(sock.run([](const ParsedEvent&) { return true; }, 3601), ApiError);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// Opt-in live checks against api.openai.com. Run with MOLO_LIVE_NET=1 and
|
||||
// OPENAI_API_KEY; otherwise each test prints a loud SKIP. Read-only lists
|
||||
// with limit=1 only: nothing is created, updated or deleted.
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
|
||||
#include "framework.h"
|
||||
#include "llm/config.h"
|
||||
#include "llm/error.h"
|
||||
#include "openai/platform/platform.h"
|
||||
|
||||
using namespace molo::openai::platform;
|
||||
using molo::clients::body::Json;
|
||||
|
||||
namespace {
|
||||
bool live(const char* name) {
|
||||
if (!std::getenv("MOLO_LIVE_NET") || molo::llm::Config::get("OPENAI_API_KEY").empty()) {
|
||||
std::cout << "SKIP " << name << " (set MOLO_LIVE_NET=1 and OPENAI_API_KEY)\n";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Json one() {
|
||||
Json q = Json::object();
|
||||
q.set("limit", Json(1));
|
||||
return q;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST(oai_live, read_only_lists) {
|
||||
if (!live("oai_live.read_only_lists")) return;
|
||||
auto p = Platform::from_env();
|
||||
for (Json r : {p->files.list(one()), p->batches.list(one()), p->fine_tuning_jobs.list(one()),
|
||||
p->vector_stores.list(one()), p->evals.list(one()), p->containers.list(one())}) {
|
||||
ASSERT_TRUE(r.is_object());
|
||||
ASSERT_TRUE(r.has("data"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(oai_live, client_secret_is_not_created_here) {
|
||||
// Creating a client secret is a write on the account; it is left to the
|
||||
// demo route with confirm:true. This test only documents that decision.
|
||||
if (!live("oai_live.client_secret_is_not_created_here")) return;
|
||||
}
|
||||
Reference in New Issue
Block a user