From 6fb7c80d2d596486c6df40e277db0712576a9c23 Mon Sep 17 00:00:00 2001 From: retoor Date: Sat, 24 Oct 2026 08:02:23 +0200 Subject: [PATCH] feat(clients): meta: Meta Model API module, Muse Code protocol client, demo and tests Co-Authored-By: Claude Sonnet 5.5 --- example/app/ai/MetaDemo.cpp | 467 +++++++++++++++++++++++++- example/assets/ai_meta.js | 51 +++ src/meta/SPEC.md | 202 +++++++++++ src/meta/calls.cpp | 65 ++++ src/meta/calls.h | 48 +++ src/meta/caps.h | 58 ++++ src/meta/catalog.cpp | 64 ++++ src/meta/catalog.h | 58 ++++ src/meta/chat_completions.cpp | 293 ++++++++++++++++ src/meta/chat_completions.h | 114 +++++++ src/meta/chat_model.cpp | 285 ++++++++++++++++ src/meta/chat_model.h | 46 +++ src/meta/check.cpp | 93 +++++ src/meta/check.h | 40 +++ src/meta/client.cpp | 71 ++++ src/meta/client.h | 80 +++++ src/meta/files.cpp | 111 ++++++ src/meta/files.h | 67 ++++ src/meta/images.cpp | 146 ++++++++ src/meta/images.h | 76 +++++ src/meta/jsonx.h | 72 ++++ src/meta/messages.cpp | 291 ++++++++++++++++ src/meta/messages.h | 86 +++++ src/meta/models.cpp | 44 +++ src/meta/models.h | 50 +++ src/meta/muse_code.cpp | 353 +++++++++++++++++++ src/meta/muse_code.h | 127 +++++++ src/meta/muse_code_spec.cpp | 185 ++++++++++ src/meta/muse_code_spec.h | 21 ++ src/meta/response_events.cpp | 154 +++++++++ src/meta/response_events.h | 80 +++++ src/meta/response_input.cpp | 254 ++++++++++++++ src/meta/response_input.h | 67 ++++ src/meta/response_request.cpp | 170 ++++++++++ src/meta/response_request.h | 65 ++++ src/meta/responses.cpp | 104 ++++++ src/meta/responses.h | 55 +++ src/meta/sam.cpp | 237 +++++++++++++ src/meta/sam.h | 98 ++++++ src/meta/sam_mask.cpp | 270 +++++++++++++++ src/meta/schema_check.cpp | 107 ++++++ src/meta/schema_check.h | 27 ++ src/meta/speech.cpp | 316 +++++++++++++++++ src/meta/speech.h | 158 +++++++++ src/meta/tools/check_spec.py | 195 +++++++++++ src/meta/usage.cpp | 79 +++++ src/meta/usage.h | 51 +++ tests/example/ai/meta/page_test.cpp | 31 ++ tests/example/ai/meta/routes_test.cpp | 63 ++++ tests/src/meta/test_meta_core.cpp | 443 ++++++++++++++++++++++++ tests/src/meta/test_meta_net.cpp | 350 +++++++++++++++++++ 51 files changed, 7035 insertions(+), 3 deletions(-) create mode 100644 example/assets/ai_meta.js create mode 100644 src/meta/SPEC.md create mode 100644 src/meta/calls.cpp create mode 100644 src/meta/calls.h create mode 100644 src/meta/caps.h create mode 100644 src/meta/catalog.cpp create mode 100644 src/meta/catalog.h create mode 100644 src/meta/chat_completions.cpp create mode 100644 src/meta/chat_completions.h create mode 100644 src/meta/chat_model.cpp create mode 100644 src/meta/chat_model.h create mode 100644 src/meta/check.cpp create mode 100644 src/meta/check.h create mode 100644 src/meta/client.cpp create mode 100644 src/meta/client.h create mode 100644 src/meta/files.cpp create mode 100644 src/meta/files.h create mode 100644 src/meta/images.cpp create mode 100644 src/meta/images.h create mode 100644 src/meta/jsonx.h create mode 100644 src/meta/messages.cpp create mode 100644 src/meta/messages.h create mode 100644 src/meta/models.cpp create mode 100644 src/meta/models.h create mode 100644 src/meta/muse_code.cpp create mode 100644 src/meta/muse_code.h create mode 100644 src/meta/muse_code_spec.cpp create mode 100644 src/meta/muse_code_spec.h create mode 100644 src/meta/response_events.cpp create mode 100644 src/meta/response_events.h create mode 100644 src/meta/response_input.cpp create mode 100644 src/meta/response_input.h create mode 100644 src/meta/response_request.cpp create mode 100644 src/meta/response_request.h create mode 100644 src/meta/responses.cpp create mode 100644 src/meta/responses.h create mode 100644 src/meta/sam.cpp create mode 100644 src/meta/sam.h create mode 100644 src/meta/sam_mask.cpp create mode 100644 src/meta/schema_check.cpp create mode 100644 src/meta/schema_check.h create mode 100644 src/meta/speech.cpp create mode 100644 src/meta/speech.h create mode 100644 src/meta/tools/check_spec.py create mode 100644 src/meta/usage.cpp create mode 100644 src/meta/usage.h create mode 100644 tests/example/ai/meta/page_test.cpp create mode 100644 tests/example/ai/meta/routes_test.cpp create mode 100644 tests/src/meta/test_meta_core.cpp create mode 100644 tests/src/meta/test_meta_net.cpp diff --git a/example/app/ai/MetaDemo.cpp b/example/app/ai/MetaDemo.cpp index 1f594f6..e1555e2 100644 --- a/example/app/ai/MetaDemo.cpp +++ b/example/app/ai/MetaDemo.cpp @@ -1,13 +1,474 @@ #include "../AiDemos.hpp" #include "../../fw/Router.hpp" +#include "../../fw/Templates.hpp" +#include "meta/calls.h" +#include "meta/check.h" +#include "meta/response_input.h" +#include "meta/catalog.h" +#include "meta/chat_completions.h" +#include "meta/chat_model.h" +#include "meta/client.h" +#include "meta/files.h" +#include "meta/images.h" +#include "meta/messages.h" +#include "meta/models.h" +#include "meta/muse_code.h" +#include "meta/responses.h" +#include "meta/sam.h" +#include "meta/speech.h" +#include "security/crypto/Base64.h" namespace molo::example::app { -// Meta Model API (Muse Spark and the Muse family). Stub: the module agent fills this in. +// Meta Model API (api.meta.ai/v1: Muse Spark, Muse Wonder, Muse Image, Muse +// Voice Transcribe, SAM 3.1) and the Muse Code session protocol, over the +// real src/meta module. GET routes need no credential and make no upstream +// call. Every route that calls Meta is a POST: 422 on bad input first, then +// 503 not_configured without META_API_KEY / MODEL_API_KEY. Paid and +// destructive calls (images, segmentation, transcription, deletes, spawning +// muse) are dry runs unless the body says "confirm":true. + +namespace { + +namespace mt = meta; +using Json = clients::body::Json; + +constexpr std::size_t kMaxBody = 8 * 1024 * 1024; + +const char kPage[] = R"META({% extends "base.html" %} +{% block content %} +

Meta Model API

+

Muse Spark and Muse Wonder (Responses, Chat Completions, Messages), Muse Image, +Muse Voice Transcribe, SAM 3.1, files, models, and the Muse Code session protocol. +Paid and destructive calls stay dry runs until you tick confirm.

+

Checking credentials...

+
+ + + +
+ +

+

Models

+
+
+ + +{% endblock %} +)META"; + +fw::Response json_out(const Json& j, int status = 200) { + fw::Response res = fw::JsonResponse(j.dump(), status); + res.headers.set("Cache-Control", "no-store"); + return res; +} + +[[noreturn]] void bad(const std::string& why) { + throw fw::HttpError(422, "validation_error", why); +} + +[[noreturn]] void rethrow(const llm::ApiError& e) { + if (e.http_status() == 503 && e.type() == "NOT_CONFIGURED") + throw fw::HttpError(503, "not_configured", e.message()); + if (e.type() == "INVALID_ARGUMENT") throw fw::HttpError(422, "validation_error", e.message()); + throw fw::HttpError(502, "meta_error", e.type() + ": " + e.message()); +} + +template +auto guarded(F&& f) -> decltype(f()) { + try { + return f(); + } catch (const llm::ApiError& e) { + rethrow(e); + } +} + +Json parse_body(const fw::Request& req) { + if (req.body.size() > kMaxBody) throw fw::HttpError(413, "payload_too_large", "body too large"); + if (req.body.empty()) return Json::object(); + try { + Json j = Json::parse(req.body); + if (!j.is_object()) bad("body must be a JSON object"); + return j; + } catch (const clients::body::JsonError&) { + bad("body is not valid JSON"); + } +} + +std::string text(const Json& b, const char* key, bool required = false) { + const Json* v = b.find(key); + if (!v || v->is_null()) { + if (required) bad(std::string(key) + " is required"); + return ""; + } + if (!v->is_string()) bad(std::string(key) + " must be a string"); + return v->as_string(); +} + +std::optional number(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return std::nullopt; + if (!v->is_number()) bad(std::string(key) + " must be a number"); + return v->as_number(); +} + +bool flag(const Json& b, const char* key) { + const Json* v = b.find(key); + if (!v || v->is_null()) return false; + if (!v->is_bool()) bad(std::string(key) + " must be true or false"); + return v->as_bool(); +} + +Json dry_run(const char* what, const std::string& target) { + Json o = Json::object(); + o.set("dry_run", Json(true)); + o.set(what, Json(target)); + o.set("hint", Json("send confirm:true to run it")); + return o; +} + +void need_key(const std::shared_ptr& c) { + if (!c->configured()) throw fw::HttpError(503, "not_configured", "set META_API_KEY or MODEL_API_KEY"); +} + +mt::ResponseRequest response_from(const Json& b) { + mt::ResponseRequest r; + r.model = text(b, "model", true); + const Json* in = b.find("input"); + if (!in || !(in->is_string() || in->is_array())) bad("input must be a string or an array"); + r.input = *in; + r.instructions = text(b, "instructions"); + r.previous_response_id = text(b, "previous_response_id"); + r.reasoning_effort = text(b, "reasoning_effort"); + if (auto n = number(b, "max_output_tokens")) r.max_output_tokens = static_cast(*n); + if (auto t = number(b, "temperature")) r.temperature = *t; + if (flag(b, "web_search")) r.tools.push_back(mt::ToolBuilder::web_search()); + return r; +} + +} // namespace + void register_ai_meta_demo(fw::Router& router, Services& services) { - (void)router; - (void)services; + auto client = mt::MetaClient::from_env(); + fw::Environment* env = services.templates.get(); + services.templates->add_template("ai_meta.html", kPage); + services.add_feature("ai-meta", "/ai/meta"); + + router.get("/ai/meta", "ai-meta-page", [env](fw::Request req) { + (void)req; + fw::TemplateCtx ctx; + ctx["title"] = std::string("Meta Model API"); + return fw::Response(200, env->render("ai_meta.html", ctx), "text/html; charset=utf-8"); + }); + + router.get("/ai/meta/status", "ai-meta-status", [client](fw::Request req) { + (void)req; + Json o = Json::object(); + o.set("configured", Json(client->configured())); + o.set("base_url", Json(mt::MetaEndpoints::kBaseUrl)); + o.set("key_env", Json(std::string(mt::MetaEndpoints::kKeyEnv) + " or " + mt::MetaEndpoints::kAltKeyEnv)); + o.set("calls", Json(static_cast(mt::call_count()))); + Json models = Json::array(); + for (std::size_t i = 0; i < mt::ModelCatalog::size(); ++i) { + const mt::ModelInfo& m = mt::ModelCatalog::at(i); + Json row = Json::object(); + row.set("id", Json(m.id)); + row.set("family", Json(m.family)); + row.set("kind", Json(mt::ModelCatalog::kind_name(m.kind))); + row.set("hosted", Json(m.hosted)); + row.set("note", Json(m.note)); + models.push(row); + } + o.set("models", models); + return json_out(o); + }); + + router.get("/ai/meta/calls", "ai-meta-calls", [](fw::Request req) { + (void)req; + Json arr = Json::array(); + for (std::size_t i = 0; i < mt::call_count(); ++i) { + const mt::CallSpec& s = mt::spec_at(i); + Json row = Json::object(); + row.set("http", Json(s.http_method)); + row.set("path", Json(s.path)); + row.set("operation", Json(s.operation)); + arr.push(row); + } + Json o = Json::object(); + o.set("calls", arr); + return json_out(o); + }); + + router.post("/ai/meta/responses", "ai-meta-responses", [client](fw::Request req) { + Json body = parse_body(req); + mt::ResponseRequest r = response_from(body); + return guarded([&] { + r.validate(); + need_key(client); + mt::ResponseResult res = mt::ResponsesService(client).create(r); + Json o = Json::object(); + o.set("id", Json(res.id())); + o.set("status", Json(res.status())); + o.set("output_text", Json(res.output_text())); + o.set("usage", res.body.find("usage") ? *res.body.find("usage") : Json::object()); + return json_out(o); + }); + }); + + router.post("/ai/meta/responses/count", "ai-meta-responses-count", [client](fw::Request req) { + mt::ResponseRequest r = response_from(parse_body(req)); + return guarded([&] { + r.validate(); + need_key(client); + Json o = Json::object(); + o.set("input_tokens", Json(static_cast(mt::ResponsesService(client).count_input_tokens(r)))); + return json_out(o); + }); + }); + + auto id_route = [&](const char* path, const char* name, int mode) { + router.post(path, name, [client, mode](fw::Request req) { + Json body = parse_body(req); + std::string id = text(body, "id", true); + bool confirm = flag(body, "confirm"); + return guarded([&] { + mt::Check::segment("id", id); + if (mode == 2 && !confirm) return json_out(dry_run("delete", id)); + need_key(client); + mt::ResponsesService svc(client); + Json out = mode == 0 ? svc.retrieve(id).body : mode == 1 ? svc.cancel(id) : svc.remove(id); + return json_out(out); + }); + }); + }; + id_route("/ai/meta/responses/get", "ai-meta-responses-get", 0); + id_route("/ai/meta/responses/cancel", "ai-meta-responses-cancel", 1); + id_route("/ai/meta/responses/delete", "ai-meta-responses-delete", 2); + + router.post("/ai/meta/chat", "ai-meta-chat", [client](fw::Request req) { + Json body = parse_body(req); + mt::ChatCompletionRequest r; + r.model = text(body, "model", true); + r.messages = {mt::ChatMessage::user(text(body, "message", true))}; + r.reasoning_effort = text(body, "reasoning_effort"); + if (auto n = number(body, "max_completion_tokens")) r.max_completion_tokens = static_cast(*n); + return guarded([&] { + r.validate(); + need_key(client); + Json out = mt::ChatCompletionsService(client).create(r); + llm::ChatResponse cr = mt::MetaChatModel::from_chat(out); + Json o = Json::object(); + o.set("text", Json(cr.text())); + o.set("finish_reason", Json(cr.finish_reason)); + o.set("total_tokens", Json(cr.usage.total_tokens)); + return json_out(o); + }); + }); + + router.post("/ai/meta/messages", "ai-meta-messages", [client](fw::Request req) { + Json body = parse_body(req); + mt::MessageRequest r; + r.model = text(body, "model", true); + r.messages = {mt::AnthropicBlock::message_text("user", text(body, "message", true))}; + r.max_tokens = static_cast(number(body, "max_tokens").value_or(0)); + return guarded([&] { + r.validate(); + need_key(client); + Json out = mt::MessagesService(client).create(r); + Json o = Json::object(); + o.set("text", Json(mt::MessagesService::text_of(out))); + o.set("stop_reason", Json(mt::JsonX::str(out, "stop_reason"))); + return json_out(o); + }); + }); + + router.post("/ai/meta/models", "ai-meta-models", [client](fw::Request req) { + Json body = parse_body(req); + std::string id = text(body, "id"); + return guarded([&] { + if (!id.empty()) mt::Check::model_id(id); + need_key(client); + mt::ModelsService svc(client); + Json arr = Json::array(); + if (id.empty()) { + for (const auto& m : svc.list()) { + Json row = Json::object(); + row.set("id", Json(m.id)); + row.set("created", Json(static_cast(m.created))); + arr.push(row); + } + } else { + auto m = svc.retrieve(id); + Json row = Json::object(); + row.set("id", Json(m.id)); + row.set("created", Json(static_cast(m.created))); + arr.push(row); + } + Json o = Json::object(); + o.set("models", arr); + return json_out(o); + }); + }); + + router.post("/ai/meta/files", "ai-meta-files", [client](fw::Request req) { + Json body = parse_body(req); + std::string action = text(body, "action", true); + if (action != "list" && action != "get" && action != "delete") bad("action must be list, get or delete"); + std::string id = text(body, "id"); + if (action != "list" && id.empty()) bad("id is required"); + bool confirm = flag(body, "confirm"); + return guarded([&] { + if (!id.empty()) mt::Check::segment("id", id); + if (action == "delete" && !confirm) return json_out(dry_run("delete", id)); + need_key(client); + mt::FilesService svc(client); + Json o = Json::object(); + if (action == "list") { + mt::FileList l = svc.list_all(text(body, "purpose"), 100, text(body, "order")); + Json arr = Json::array(); + for (const auto& f : l.files) { + Json row = Json::object(); + row.set("id", Json(f.id)); + row.set("filename", Json(f.filename)); + row.set("bytes", Json(static_cast(f.bytes))); + arr.push(row); + } + o.set("files", arr); + o.set("truncated", Json(l.truncated)); + } else if (action == "get") { + mt::FileObject f = svc.retrieve(id); + o.set("id", Json(f.id)); + o.set("filename", Json(f.filename)); + o.set("status", Json(f.status)); + } else { + o = svc.remove(id); + } + return json_out(o); + }); + }); + + router.post("/ai/meta/images", "ai-meta-images", [client](fw::Request req) { + Json body = parse_body(req); + mt::ImageRequest r; + r.prompt = text(body, "prompt", true); + r.size = text(body, "size"); + if (auto n = number(body, "n")) r.n = static_cast(*n); + r.reasoning_strength = text(body, "reasoning_strength"); + bool confirm = flag(body, "confirm"); + return guarded([&] { + r.validate(false); + if (!confirm) return json_out(dry_run("prompt", r.prompt)); + need_key(client); + mt::ImageResult res = mt::ImagesService(client).generate(r); + Json o = Json::object(); + o.set("images", Json(static_cast(res.data.size()))); + o.set("output_format", Json(res.output_format)); + if (!res.data.empty()) o.set("first_b64_prefix", Json(res.data[0].b64_json.substr(0, 64))); + return json_out(o); + }); + }); + + router.post("/ai/meta/segment", "ai-meta-segment", [client](fw::Request req) { + Json body = parse_body(req); + mt::SamRequest s; + std::string phrase = text(body, "prompt", true); + s.image_url = text(body, "image_url"); + s.video_url = text(body, "video_url"); + s.include_confidence = flag(body, "include_confidence"); + s.mask_encoding = text(body, "mask_encoding"); + bool confirm = flag(body, "confirm"); + return guarded([&] { + s.prompt = mt::SamPrompt::concept_text(phrase); + mt::ResponseRequest rr = s.to_response_request(); + rr.validate(true); + if (!confirm) return json_out(dry_run("prompt", s.prompt)); + need_key(client); + mt::ResponseAccumulator acc = mt::ResponsesService(client).stream(rr); + auto frames = mt::SamOutput::parse(acc.text()); + Json o = Json::object(); + o.set("frames", Json(static_cast(frames.size()))); + o.set("records", Json(static_cast(mt::SamOutput::record_count(frames)))); + return json_out(o); + }); + }); + + router.post("/ai/meta/transcribe", "ai-meta-transcribe", [client](fw::Request req) { + Json body = parse_body(req); + std::string b64 = text(body, "audio_b64", true); + std::string wav; + if (!security::crypto::Base64::decode(b64, wav)) bad("audio_b64 is not valid base64"); + mt::SpeechOptions o; + o.mode = text(body, "mode"); + bool confirm = flag(body, "confirm"); + return guarded([&] { + o.validate(); + mt::WavInfo info = mt::WavInfo::parse(wav); + if (!confirm) return json_out(dry_run("duration_ms", std::to_string(info.duration_ms))); + need_key(client); + mt::TranscribeResult r = mt::TranscribeService(client).transcribe(o, wav); + Json out = Json::object(); + out.set("transcript", Json(r.transcript)); + out.set("turns", Json(static_cast(r.turns.size()))); + return json_out(out); + }); + }); + + // GET /status is unauthenticated upstream; still a POST here (it leaves the host). + router.post("/ai/meta/platform", "ai-meta-platform", [](fw::Request req) { + (void)req; + return guarded([&] { + mt::ServiceStatus st = mt::StatusService(mt::MetaClient::unauthenticated()).get(); + Json o = Json::object(); + o.set("is_alive", Json(st.is_alive)); + o.set("service_status", Json(st.service_status)); + return json_out(o); + }); + }); + + router.post("/ai/meta/code", "ai-meta-code", [](fw::Request req) { + Json body = parse_body(req); + bool confirm = flag(body, "confirm"); + bool available = mt::ProcessTransport::binary_available("muse"); + if (!confirm) { + Json o = dry_run("command", "muse serve"); + o.set("muse_on_path", Json(available)); + return json_out(o); + } + if (!available) throw fw::HttpError(503, "not_configured", "the muse binary is not on PATH"); + return guarded([&] { + mt::MspClient c(std::make_unique()); + Json init = c.initialize("moloapp-demo", "0.1"); + Json o = Json::object(); + o.set("schema_fingerprint", Json(c.schema_fingerprint())); + o.set("server", init.find("serverInfo") ? *init.find("serverInfo") : Json::object()); + c.close(); + return json_out(o); + }); + }); } } // namespace molo::example::app diff --git a/example/assets/ai_meta.js b/example/assets/ai_meta.js new file mode 100644 index 0000000..23f0880 --- /dev/null +++ b/example/assets/ai_meta.js @@ -0,0 +1,51 @@ +(function () { + var $ = function (id) { return document.getElementById(id); }; + var samples = { + "responses": '{"model":"muse-spark-1.3","input":"Say hi in five words.","reasoning_effort":"low"}', + "responses/count": '{"model":"muse-spark-1.3","input":"How many tokens?"}', + "responses/get": '{"id":"resp_..."}', + "responses/cancel": '{"id":"resp_..."}', + "responses/delete": '{"id":"resp_..."}', + "chat": '{"model":"muse-spark-1.3","message":"Hello"}', + "messages": '{"model":"muse-spark-1.3","message":"Hello","max_tokens":256}', + "models": '{}', + "files": '{"action":"list"}', + "images": '{"prompt":"a red fox in snow","n":1}', + "segment": '{"prompt":"yellow school bus","image_url":"https://example.com/bus.jpg"}', + "transcribe": '{"audio_b64":"","mode":"ENDPOINTING"}', + "platform": '{}', + "code": '{}' + }; + function show(text, isError) { + $("meta-out").textContent = text; + $("meta-error").textContent = isError ? "Request failed" : ""; + } + function send() { + var route = $("meta-route").value; + var body; + try { body = JSON.parse($("meta-body").value || "{}"); } catch (e) { show("Body is not valid JSON", true); return; } + if ($("meta-confirm").checked) body.confirm = true; + $("meta-send").disabled = true; + fetch("/ai/meta/" + route, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }) + .then(function (r) { return r.text().then(function (t) { return { ok: r.ok, t: t }; }); }) + .then(function (r) { + var pretty = r.t; + try { pretty = JSON.stringify(JSON.parse(r.t), null, 2); } catch (e) { /* keep text */ } + show(pretty, !r.ok); + }) + .catch(function () { show("Network error", true); }) + .then(function () { $("meta-send").disabled = false; }); + } + fetch("/ai/meta/status").then(function (r) { return r.json(); }).then(function (s) { + $("meta-config").textContent = s.configured ? "Credential found (" + s.key_env + ")." : "No credential: set " + s.key_env + ". Reads and dry runs still work."; + var box = $("meta-models"); + box.textContent = ""; + s.models.forEach(function (m) { + var p = document.createElement("div"); + p.textContent = m.id + " - " + m.family + (m.hosted ? "" : " (self-hosted)") + ": " + m.note; + box.appendChild(p); + }); + }).catch(function () { $("meta-config").textContent = "Status unavailable."; }); + $("meta-route").addEventListener("change", function () { $("meta-body").value = samples[$("meta-route").value] || "{}"; }); + $("meta-send").addEventListener("click", send); +})(); diff --git a/src/meta/SPEC.md b/src/meta/SPEC.md new file mode 100644 index 0000000..1b2b82d --- /dev/null +++ b/src/meta/SPEC.md @@ -0,0 +1,202 @@ +# Meta Model API (`src/meta`, namespace `molo::meta`) + +Status: implemented 2026-10-24. Scope is the Meta Model API at +`https://api.meta.ai/v1` (docs `https://dev.meta.ai/docs`): Muse Spark and +Muse Wonder text models, Muse Image, Muse Voice Transcribe, Segment Anything +Model 3.1, files, models, status, and the Muse Code agent protocol. It is NOT +the older Llama API (`llama.developer.meta.com`); nothing here targets it. +`developer.meta.com/ai` redirects (302) to `dev.meta.ai`, so the blog-era +material is the same site. + +Credential: `META_API_KEY`, or `MODEL_API_KEY` (the docs' own name), read +through `llm::Config::get`. Keys look like `LLM_{id}_{secret}`. Bearer auth. + +## Module map (one concern per file) + +| File | Concern | +|---|---| +| `caps.h` | every ceiling and documented limit, with the source marked | +| `calls.h/.cpp` | the ONE call table (20 operations), `path_for` | +| `catalog.h/.cpp` | static model table: context, prices, rate limits, `max` effort | +| `client.h/.cpp` | `MetaClient` (ApiClient factory, key lookup), `RetryPolicy`, `call_with_retry` | +| `check.h/.cpp`, `schema_check.h/.cpp` | local validation; JSON Schema pre-flight (depth, size, recursion, strict) | +| `response_input.*` | Responses input items and tools (`ResponseInput`, `ToolBuilder`) | +| `response_request.*` | `ResponseRequest` body + rules | +| `response_events.*` | 27 stream event names, `ResponseAccumulator` | +| `responses.*` | create, stream, retrieve, retrieve_stream, remove, cancel, count_input_tokens | +| `chat_completions.*` | Chat Completions request, service, chunk accumulator | +| `messages.*` | Anthropic-compatible Messages: create, stream, count_tokens | +| `files.*` | upload (multipart), list page + capped walk, retrieve, content, remove | +| `images.*` | Muse Image generate / edit (JSON and multipart) / stream | +| `speech.*` | `/asr/transcribe` (JSON, SSE, text) and `/asr/realtime` WebSocket session | +| `sam.*`, `sam_mask.cpp` | SAM 3.1 prompts, request, output parser, mask decoder | +| `models.*` | models list/retrieve, unauthenticated status | +| `usage.*` | usage across the 3 shapes, cost from the catalog, `x-ratelimit-*` | +| `chat_model.*` | `MetaChatModel : llm::ChatModel` (Responses or Chat Completions) | +| `muse_code.*`, `muse_code_spec.*` | Muse Session Protocol client (stdio JSON-RPC) and its vocabulary | +| `tools/check_spec.py` | spec checker (see Verified against spec) | + +## Surface + +- Responses: create, stream (typed events), retrieve, streaming retrieve of a + background response, delete (explicit, destructive), cancel, count input + tokens. Reasoning continuity both ways: `previous_response_id`, or + `store:false` + `include:["reasoning.encrypted_content"]` + replayed + `reasoning` items. Tools: function, `web_search`, `tool_search`, `computer`, + plus `custom` and `namespace` (prose only). Structured output via + `text.format`. Vision/video/audio/file inputs via `ResponseInput`. +- Chat Completions: messages, tools, `response_format`, `reasoning_effort`, + streaming with usage. Rejected locally: `n>1`, `stop`, `logit_bias`, + `prediction`, `verbosity`, `web_search_options`, `modalities`, `logprobs`, + `max_output_tokens`, `text`. +- Messages: Anthropic blocks, thinking (`adaptive`/`enabled`, never disabled), + `output_config`, tools (`custom`, `web_search`, `tool_search`), stream events, + count_tokens. Rejected locally: `stop_sequences`, `top_k`, `container`, + `inference_geo`. +- Files: upload with `purpose` and `expires_after[...]` before the file part, + list (limit 1..1000, order validated because the server answers a plain-text + 400), capped page walk that reports `truncated`, retrieve, content, delete. +- Images: `/images/generations`, `/images/edits` (JSON `images[]` and + multipart `image[N]`), `tool_enablement`, SSE `*.completed` events. +- Speech: WAV pre-check (RIFF PCM mono 16-bit, 16/24 kHz, <=10 min, <=32 MB), + `request` JSON part + `audio`, `Accept` selects JSON / event stream / text; + realtime session over `google::live::WsClient` with the credential in the + handshake frame, `endStream`, close codes 1000/1008/1011/1013. +- SAM 3.1: `SamPrompt` (concept; box/point object prompts per the output + grammar), `SamRequest` -> `ResponseRequest` (`metadata` strings + `propagation_direction`, `include_confidence`, `mask_encoding`), + `SamOutput::parse`, `SamMask::decode` for `one_bit` (`!`) and `lossless` + (`~`). +- Models, status, usage, rate limits, retry policy. +- Muse Code: see below. + +## Muse Code + +Muse Code is a coding agent product, but it has a public programmatic +surface: the Muse Session Protocol (MSP), JSON-RPC 2.0, one JSON object per +line on the stdio of `muse serve` (frame limit 10 MiB). It is not an HTTP API +of `api.meta.ai`. `MspClient` implements the handshake (`initialize`, then the +`initialized` notification), request/response correlation, a bounded +notification queue (4096, drops are counted), answers to the two +server-initiated requests (`approval/request`, `userInput/request`) with an +empty receipt, error kinds from the schema (37 codes), and shortcuts for +`session/start`, `turn/start`, `turn/cancel`, `model/list`. `request()` takes +any of the 70 schema methods. `ProcessTransport` spawns `muse serve`; +`ILineTransport` keeps the client testable without the binary. The binary is +not installed here, so nothing was run against a live host. + +## Contracts and caps + +- Every id is validated before a path is built (`Check::segment`); deletes + (`ResponsesService::remove`, `FilesService::remove`) are explicit methods + with no default and no retry. +- Local rules mirror the docs: `max_output_tokens>=16`, `temperature` 0..2, + `top_p` in (0,1], penalties -2..2, effort `minimal..max` (`none` rejected, + `max` only `muse-wonder-1.0` and `muse-spark-1.3`), `background` XOR + `stream`, `reasoning.encrypted_content` XOR `previous_response_id`, + `truncation` only `disabled`, background body under 1 MiB, function names + `[A-Za-z0-9_.-]` with at most one dot, `call_id` 1..64, safety identifier + <=64, schema depth<=10, <=5000 properties, <=120000 chars, no recursion, strict + subset rules, `tool_choice` only `auto`. +- Caps live in `caps.h`: inputs 4096, tools 256, list pages 20, stream events + 1,000,000, realtime events 100,000, audio frame 1 MiB, retries 5 with + 500 ms base and a 30 s cap, SAM lines/records/mask pixels, MSP queue. +- `ApiClient` upload cap is 512 MiB (vendor allows 1 GiB). +- Errors are the foundation `ApiError` (`type` is Meta's category such as + `invalid_request_error`; `NotConfiguredError` names `META_API_KEY`). + `RetryPolicy`: 429/500/502/503 retry with backoff+jitter, 504 never. + +## Sources + +1. Docs site, Markdown pages listed by `https://dev.meta.ai/llms.txt` + (154 pages, `last-modified: Fri, 23 Oct 2026 23:30:05 GMT`, etag + `a179472f87650128af185a2718d84c76`): overview, quickstart, protocols/*, + api-reference/* (operation pages and schemas pages), models, pricing and + rate limits, error handling, file handling, token counting, reasoning, + structured output, tool calling, search grounding, speech to text, image + generation, media segmentation, `sam/*`, muse-code/*, open-weight pages. +2. `muse-code-sdk` (the `meta-models` org on the code host, MIT): + `schema/msp/stable/msp.schema.json` (fingerprint + `sha256:cfdb6d2525601b9bac76584ccb0610ab3a5539b5e4f51141f079b99b39fcb4e0`, + host version 1.4.12), recorded transcripts, Rust/TS/Python SDK sources. +3. `meta-sam` (same org): Python parser `_mask_codec.py` (mask codec ported) + and `conformance/cases/*.json` (a captured live API stream among them). +4. Live unauthenticated probes, 2026-10-24 (below). + +No OpenAPI document and no vendor SDK for the Model API exist: Meta tells +you to use the OpenAI SDK (Responses, Chat Completions) and the Anthropic SDK +(Messages) with a changed base URL. Where only prose exists the module says so +under Unverified. Cross-check: every path and body field was read in both the +operation page and the schemas page; behaviour rules were read in the guide +pages and the error-handling page. + +## Verified against spec + +`python3 -I src/meta/tools/check_spec.py` (stdlib only, fetches the docs): +20 calls (path heading, query parameters, request schema fields, response +schema present), 27 Responses stream events equal to the +`ResponseStreamEvent` union, every documented operation present in the table +(no exclusion list), catalog models/prices/rpm present on the models and +pricing pages, and the Muse Code vocabulary (70 methods, 45 notifications, 2 +server requests, 37 error codes) equal to the stable MSP schema. +Result 2026-10-24: `OK`. A mutated row (renamed query parameter and path) +fails as expected. + +## Verified live + +Unauthenticated, 2026-10-24, `https://api.meta.ai/v1`: +- host resolves, HTTP/2, TLS verifies. +- `GET /status` -> 200 `{"is_alive":true,"service_status":"operational","service_message":"","updated_at":"","model_statuses":[]}`; a bogus bearer is ignored. +- `GET`/`POST` on `/models`, `/models/{id}`, `/files`, `/files/{id}`, + `/responses`, `/responses/{id}`, `/responses/input_tokens`, + `/chat/completions`, `/images/generations`, `/images/edits` -> 401 + `{"error":{"code":"invalid_api_key","message":"Unauthorized","param":null,"type":"authentication_error"}}`. +- `/messages`, `/messages/count_tokens` -> 401 + `{"error":{"message":"Unauthorized","type":"authentication_error"},"type":"error"}` (Anthropic envelope). +- Authentication is checked before routing: an unknown path also answers 401. +- `POST /asr/transcribe` without multipart -> 415 `unsupported_media_type` with a + `sessionId`; `GET` -> 404 with an empty body. +- `wss://api.meta.ai/v1/asr/realtime`: upgrade answers 101; with no handshake + frame the server sends `{"type":"error","message":"Handshake timeout",...,"errorType":"invalid_request_error"}`. +- The opt-in live test (`MOLO_LIVE_NET=1`) re-runs the status and 401 probes + through the module and, with a key, lists models (read only). Paid paths + (images, SAM, transcription, completions) have no live test in this round: + no key exists. + +## Unverified + +- Every success body below the documented schemas was only checked against the + docs examples, never against a live keyed call: no key exists. +- The `GET /status` model_statuses array element shape (live probe returned + `[]`; the docs give no schema for `StatusResponse`). +- `image_generation_call` output items (documented in prose, absent from the + Responses schema union): `result` is read nowhere in code. +- `custom` and `namespace` tool shapes (prose only). +- A verbatim Responses SSE transcript is not published: the stream fold test + uses event names and fields from the schemas with a constructed order. +- The empty-object receipt for `approval/request` / `userInput/request` + (schema `RequestReceipt` lists no fields). +- MSP requests other than the handshake were never run against a `muse` host. +- Mask decoding is verified against two vendor conformance cases (one lossless, + one `one_bit`), not against live output. +- Realtime WebSocket beyond the handshake timeout error was not driven. +- Retry-After and `code`/`param` of error bodies (see Gaps). + +## Gaps + +- `llm::ApiError` keeps only `type` (preferring Meta's category) and drops the + error `code` (`model_not_found`, `invalid_api_key`, `payload_too_large`...), + `param` and the `Retry-After` header. Owner: `src/llm/error.cpp` + (foundation, not editable here). `RetryPolicy` therefore works from the status + only. +- `ApiClient::stream` is POST-only; streaming retrieve is read whole, then + decoded. `ApiClient` cannot send an unauthenticated request; `/status` sends + a placeholder bearer (`MetaClient::kPlaceholderKey`). +- `MetaChatModel::stream` cannot return the encrypted reasoning for replay + (`ChatChunk` has no signature field); use `generate` or `previous_response_id` + for reasoning continuity across streamed turns. +- Batch API, fine-tuning and embeddings do not exist on this platform per the + docs; nothing is implemented for them. +- Muse Spark Open and Muse Glimmer are open weights with no Model API surface; + they are in the catalog only. Self-hosted serving is out of scope. +- Realtime audio pacing (real time or 1008) is the caller's job. diff --git a/src/meta/calls.cpp b/src/meta/calls.cpp new file mode 100644 index 0000000..9866632 --- /dev/null +++ b/src/meta/calls.cpp @@ -0,0 +1,65 @@ +#include "meta/calls.h" + +#include + +#include "llm/api_client.h" + +namespace molo::meta { + +// CALLS-BEGIN (parsed by tools/check_spec.py; one row per line) +static const CallSpec kCalls[] = { + {Call::kResponsesCreate, "POST", "/responses", "responses/create-response", "responses", "CreateResponse", "Response", "model,input,instructions,previous_response_id,store,stream,background,max_output_tokens,max_tool_calls,temperature,top_p,frequency_penalty,presence_penalty,reasoning,text,tools,parallel_tool_calls,include,truncation,metadata,safety_identifier,prompt_cache_key,prompt_cache_retention,service_tier", ""}, + {Call::kResponsesRetrieve, "GET", "/responses/{response_id}", "responses/retrieve-response", "responses", "", "Response", "", "stream,starting_after"}, + {Call::kResponsesDelete, "DELETE", "/responses/{response_id}", "responses/delete-response", "responses", "", "DeleteResponseResponse", "", ""}, + {Call::kResponsesCancel, "POST", "/responses/{response_id}/cancel", "responses/cancel-response", "responses", "", "CancelResponseResponse", "", ""}, + {Call::kResponsesInputTokens, "POST", "/responses/input_tokens", "responses/count-input-tokens", "responses", "TokenCountsBody", "TokenCountsResource", "model,input,instructions,previous_response_id,tools,parallel_tool_calls,reasoning,text,truncation", ""}, + {Call::kChatCompletionsCreate, "POST", "/chat/completions", "chat-completions/create-chat-completion", "chat-completions", "CreateChatCompletionRequest", "CreateChatCompletionResponse", "model,messages,temperature,top_p,frequency_penalty,presence_penalty,max_completion_tokens,stream,stream_options,reasoning_effort,seed,prompt_cache_retention,safety_identifier,prompt_cache_key,tools,tool_choice,parallel_tool_calls,response_format,metadata,service_tier,user", ""}, + {Call::kMessagesCreate, "POST", "/messages", "messages/create-message", "messages", "AnthropicMessageRequest", "AnthropicMessage", "model,messages,max_tokens,system,tools,tool_choice,metadata,temperature,top_p,thinking,service_tier,output_config,stream,cache_control,context_management", ""}, + {Call::kMessagesCountTokens, "POST", "/messages/count_tokens", "messages/count-tokens", "messages", "AnthropicCountTokensRequest", "AnthropicCountTokensResponse", "model,messages,system,tools,tool_choice,metadata,temperature,top_p,thinking,output_config", ""}, + {Call::kFilesUpload, "POST", "/files", "files/upload-file", "files", "CreateFileRequest", "FileObject", "file,purpose,expires_after", ""}, + {Call::kFilesList, "GET", "/files", "files/list-files", "files", "", "ListFilesResponse", "", "purpose,limit,order,after"}, + {Call::kFilesRetrieve, "GET", "/files/{file_id}", "files/retrieve-file", "files", "", "FileObject", "", ""}, + {Call::kFilesContent, "GET", "/files/{file_id}/content", "files/retrieve-file-content", "files", "", "", "", ""}, + {Call::kFilesDelete, "DELETE", "/files/{file_id}", "files/delete-file", "files", "", "DeleteFileResponse", "", ""}, + {Call::kImagesGenerate, "POST", "/images/generations", "images/create-image", "images", "CreateImageRequest", "ImagesResponse", "model,prompt,n,size,response_format,moderation,output_format,reasoning_strength,user,stream,tool_enablement", ""}, + {Call::kImagesEdit, "POST", "/images/edits", "images/edit-image", "images", "EditImageBodyJsonParam", "ImagesResponse", "model,prompt,images,n,size,response_format,output_format,reasoning_strength,moderation,stream,user,tool_enablement", ""}, + {Call::kModelsList, "GET", "/models", "models/list-models", "models", "", "ListModelsResponse", "", ""}, + {Call::kModelsRetrieve, "GET", "/models/{model}", "models/retrieve-model", "models", "", "Model", "", ""}, + {Call::kStatus, "GET", "/status", "status", "", "", "", "", ""}, + {Call::kAsrTranscribe, "POST", "/asr/transcribe", "voice/transcribe", "voice", "TranscribeRequest", "TranscribeResponse", "model,audioEncoding,mode,keywords,languageBias,partialMode,emitAudioProgress", "sessionId"}, + {Call::kAsrRealtime, "WS", "/asr/realtime", "voice/realtime", "voice", "RealtimeHandshakeRequest", "RealtimeHandshakeResponse", "authorization,audioEncoding,model,mode,partialMode,emitAudioProgress,keywords,languageBias", "sessionId"}, +}; +// CALLS-END + +const CallSpec& spec_of(Call call) { + for (const CallSpec& s : kCalls) + if (s.call == call) return s; + throw llm::ApiError::invalid("meta", "unknown call"); +} + +std::size_t call_count() { return sizeof(kCalls) / sizeof(kCalls[0]); } + +const CallSpec& spec_at(std::size_t index) { + if (index >= call_count()) throw llm::ApiError::invalid("meta", "call index out of range"); + return kCalls[index]; +} + +clients::Method method_of(Call call) { + const char* m = spec_of(call).http_method; + if (!std::strcmp(m, "POST")) return clients::Method::kPost; + if (!std::strcmp(m, "DELETE")) return clients::Method::kDelete; + if (!std::strcmp(m, "GET")) return clients::Method::kGet; + throw llm::ApiError::invalid("meta", "call is not an HTTP request"); +} + +std::string path_for(Call call, const std::string& id) { + std::string path = spec_of(call).path; + std::size_t open = path.find('{'); + if (open == std::string::npos) return path; + if (!llm::Service::valid_id(id)) throw llm::ApiError::invalid("meta", "bad id for " + path); + std::size_t close = path.find('}', open); + path.replace(open, close - open + 1, id); + return path; +} + +} // namespace molo::meta diff --git a/src/meta/calls.h b/src/meta/calls.h new file mode 100644 index 0000000..fd55783 --- /dev/null +++ b/src/meta/calls.h @@ -0,0 +1,48 @@ +#pragma once + +// The ONE table of Meta Model API calls this module makes. Every row is +// checked against the published docs by tools/check_spec.py (the docs are +// prose plus schema tables, there is no OpenAPI document). Code builds every +// path from this table, never from a literal. + +#include +#include + +#include "clients/request.h" + +namespace molo::meta { + +enum class Call { + kResponsesCreate, kResponsesRetrieve, kResponsesDelete, kResponsesCancel, + kResponsesInputTokens, + kChatCompletionsCreate, + kMessagesCreate, kMessagesCountTokens, + kFilesUpload, kFilesList, kFilesRetrieve, kFilesContent, kFilesDelete, + kImagesGenerate, kImagesEdit, + kModelsList, kModelsRetrieve, + kStatus, + kAsrTranscribe, kAsrRealtime, +}; + +struct CallSpec { + Call call; + const char* http_method; // GET POST DELETE, or WS for the realtime socket + const char* path; // template under the base URL, "{id}" placeholders + const char* operation; // docs page slug under /docs/api-reference/ + const char* schema_group; // docs schemas page: /docs/api-reference//schemas + const char* request_schema; // schema name in the group, "" when no body + const char* response_schema; // schema name, "" when raw bytes or a socket + const char* body_fields; // comma list: request body fields we send + const char* query_params; // comma list: query parameters we send +}; + +const CallSpec& spec_of(Call call); +std::size_t call_count(); +const CallSpec& spec_at(std::size_t index); +clients::Method method_of(Call call); + +// Fills the first "{...}" placeholder of the call's path with `id` after +// validating it as a single path segment. Throws llm::ApiError::invalid. +std::string path_for(Call call, const std::string& id = ""); + +} // namespace molo::meta diff --git a/src/meta/caps.h b/src/meta/caps.h new file mode 100644 index 0000000..67aa4aa --- /dev/null +++ b/src/meta/caps.h @@ -0,0 +1,58 @@ +#pragma once + +// Static ceilings and documented limits for the Meta Model API module. Every +// value is a constant you can read; nothing is autotuned. Documented vendor +// limits are marked "docs"; the rest are our own floors and ceilings. + +#include + +namespace molo::meta { + +struct MetaCaps { + // Request shaping (docs: error-handling, protocols/responses). + static constexpr int kMinOutputTokens = 16; // docs: max_output_tokens minimum + static constexpr std::size_t kMaxInputItems = 4096; // ours + static constexpr std::size_t kMaxTools = 256; // ours (matches llm::Caps) + static constexpr std::size_t kMaxIdBytes = 256; // ours + static constexpr std::size_t kMaxCallIdBytes = 64; // docs: call_id 1..64 + static constexpr std::size_t kMaxSafetyIdBytes = 64; // docs + static constexpr std::size_t kMaxModelBytes = 128; // ours + static constexpr std::size_t kMaxPromptBytes = 4u * 1024u * 1024u; // ours + static constexpr std::size_t kMaxBackgroundBodyBytes = 1048576; // docs: 1 MiB + static constexpr std::size_t kMaxInlineBytes = 50000000; // docs: 50 MB inline media + static constexpr std::size_t kMaxImageCount = 50; // docs: 50 images per request + static constexpr int kMaxImagesPerCall = 10; // docs: n 1..10 + static constexpr std::size_t kMaxMetadataPairs = 16; // ours + // Files (docs: file-handling). The vendor allows 1 GiB; llm::Caps stops at 512 MiB. + static constexpr std::size_t kMaxUploadBytes = 512u * 1024u * 1024u; + static constexpr long kMinExpireSeconds = 3600; // docs + static constexpr long kMaxExpireSeconds = 2592000; // docs + static constexpr int kListPageLimit = 1000; // ours (docs allow 10000) + static constexpr std::size_t kMaxListPages = 20; // ours + // Speech (docs: speech-to-text). + static constexpr std::size_t kMaxTranscribeBytes = 32u * 1000u * 1000u; // docs: 32 MB + static constexpr long kMaxTranscribeMs = 10L * 60L * 1000L; // docs: 10 minutes + static constexpr std::size_t kMaxKeywords = 256; // ours + static constexpr int kRealtimeHandshakeMs = 10000; // docs: first frame within 10 s + static constexpr int kConnectTimeoutMs = 10000; // ours + static constexpr int kRealtimeIdleMs = 30000; // ours + static constexpr std::size_t kMaxRealtimeAudioFrame = 1u * 1024u * 1024u; // ours + static constexpr std::size_t kMaxRealtimeEvents = 100000; // ours + static constexpr std::size_t kMaxStreamEvents = 1000000; // ours: events folded per HTTP stream + // Retry (docs: pricing-rate-limits, error-handling). + static constexpr int kMaxRetries = 5; + static constexpr int kBackoffBaseMs = 500; // docs: start near 500 ms + static constexpr int kBackoffCapMs = 30000; // ours + // Segmentation output parsing. + static constexpr std::size_t kMaxSamLines = 100000; // ours + static constexpr std::size_t kMaxSamRecords = 1000000; // ours + static constexpr std::size_t kMaxMaskPixels = 64u * 1024u * 1024u; // ours + // Muse Code session protocol (MSP). + static constexpr std::size_t kMspFrameLimit = 10485760; // MSP transcript manifest: frameLimitBytes + static constexpr std::size_t kMspMaxPending = 64; // ours + static constexpr std::size_t kMspMaxQueued = 4096; // ours + static constexpr int kMspRequestTimeoutMs = 120000; // ours + static constexpr int kMspSpawnTimeoutMs = 10000; // ours +}; + +} // namespace molo::meta diff --git a/src/meta/catalog.cpp b/src/meta/catalog.cpp new file mode 100644 index 0000000..6d41f0c --- /dev/null +++ b/src/meta/catalog.cpp @@ -0,0 +1,64 @@ +#include "meta/catalog.h" + +namespace molo::meta { + +// CATALOG-BEGIN (parsed by tools/check_spec.py; one row per line) +// id, family, kind, hosted, contributor, supports_max, context, cached, input, output, other_price, rpm, tpm, note +static const ModelInfo kModels[] = { + {"muse-wonder-1.0", "Muse Wonder", ModelKind::kText, true, false, true, 1048576, 0.20, 4.00, 20.00, "", 3000, 4000000, "preview; Standard mode only on Model API"}, + {"muse-spark-1.3", "Muse Spark", ModelKind::kText, true, false, true, 1048576, 0.15, 1.25, 4.25, "", 3000, 4000000, "recommended for new work"}, + {"muse-spark-1.2", "Muse Spark", ModelKind::kText, true, false, false, 1048576, 0.15, 1.25, 4.25, "", 3000, 4000000, "previous version"}, + {"muse-spark-1.1", "Muse Spark", ModelKind::kText, true, false, false, 1048576, 0.15, 1.25, 4.25, "", 3000, 4000000, "original version, Standard mode only"}, + {"muse-spark-1.3-contributor", "Muse Spark", ModelKind::kText, true, true, false, 1048576, 0.002, 0.10, 0.20, "", 150, 3000000, "Contributor mode: content may train Meta models"}, + {"muse-spark-1.2-contributor", "Muse Spark", ModelKind::kText, true, true, false, 1048576, 0.002, 0.10, 0.20, "", 150, 3000000, "Contributor mode: content may train Meta models"}, + {"muse-image-1.0", "Muse Image", ModelKind::kImage, true, false, false, 0, 0, 0, 0, "$0.01 per generated image", 150, 0, "generation and editing"}, + {"muse-voice-transcribe-1.0", "Muse Voice Transcribe", ModelKind::kSpeech, true, false, false, 0, 0, 0, 0, "$0.18 per hour of audio", 0, 0, "256 concurrent streams, 16000 streams per hour"}, + {"sam-3.1", "Segment Anything Model", ModelKind::kSegmenter, true, false, false, 0, 0, 0, 0, "$2.50 per 1000 images, $0.20 per 1000 video frames", 3000, 0, "video bills at least 50 frames"}, + {"muse-spark-open", "Muse Spark Open", ModelKind::kOpenWeight, false, false, false, 0, 0, 0, 0, "open weights, self-hosted", 0, 0, "816B MoE, 42B active; no Model API"}, + {"muse-glimmer", "Muse Glimmer", ModelKind::kOpenWeight, false, false, false, 0, 0, 0, 0, "open weights, self-hosted", 0, 0, "distilled from Muse Spark; no Model API"}, +}; +// CATALOG-END + +std::size_t ModelCatalog::size() { return sizeof(kModels) / sizeof(kModels[0]); } + +const ModelInfo& ModelCatalog::at(std::size_t index) { return kModels[index % size()]; } + +const ModelInfo* ModelCatalog::find(const std::string& id) { + for (const ModelInfo& m : kModels) + if (id == m.id) return &m; + return nullptr; +} + +bool ModelCatalog::is_text_model(const std::string& id) { + const ModelInfo* m = find(id); + return m && m->kind == ModelKind::kText; +} + +bool ModelCatalog::accepts_max_effort(const std::string& id) { + const ModelInfo* m = find(id); + return m && m->supports_max; +} + +const char* ModelCatalog::kind_name(ModelKind kind) { + switch (kind) { + case ModelKind::kText: return "text"; + case ModelKind::kImage: return "image"; + case ModelKind::kSpeech: return "speech"; + case ModelKind::kSegmenter: return "segmentation"; + case ModelKind::kOpenWeight: return "open-weight"; + } + return "unknown"; +} + +const std::vector& ReasoningEfforts::accepted() { + static const std::vector v = {"minimal", "low", "medium", "high", "xhigh", "max"}; + return v; +} + +bool ReasoningEfforts::valid(const std::string& value) { + for (const std::string& e : accepted()) + if (e == value) return true; + return false; +} + +} // namespace molo::meta diff --git a/src/meta/catalog.h b/src/meta/catalog.h new file mode 100644 index 0000000..eddd416 --- /dev/null +++ b/src/meta/catalog.h @@ -0,0 +1,58 @@ +#pragma once + +// Static catalog of the Meta models named by the docs (models, pricing and +// rate-limit pages). It is a specification table: request validation reads +// it (for example which models accept reasoning effort "max") and the demo +// prints it. It is NOT the live model list: ask ModelsService for that. + +#include +#include +#include + +namespace molo::meta { + +enum class ModelKind { + kText, // Muse Wonder, Muse Spark: Responses, Chat Completions, Messages + kImage, // Muse Image: Responses, Images + kSpeech, // Muse Voice Transcribe: ASR file and realtime + kSegmenter, // Segment Anything Model 3.1: Responses + kOpenWeight, // Muse Spark Open, Muse Glimmer: self-hosted, no Model API +}; + +struct ModelInfo { + const char* id; + const char* family; // "Muse Wonder", "Muse Spark", ... + ModelKind kind; + bool hosted; // callable on api.meta.ai + bool contributor; // Contributor mode variant (content may train models) + bool supports_max; // reasoning effort "max" + long context_tokens; // 0 when not applicable + double cached_per_m; // USD per 1M cached input tokens, 0 when n/a + double input_per_m; // USD per 1M input tokens, 0 when n/a + double output_per_m; // USD per 1M output tokens, 0 when n/a + const char* other_price; // non-token pricing, "" when token priced + long rpm; // requests per minute, 0 = none documented + long tpm; // tokens per minute, 0 = none documented + const char* note; +}; + +class ModelCatalog { + public: + static std::size_t size(); + static const ModelInfo& at(std::size_t index); + // nullptr when the id is not in the docs. + static const ModelInfo* find(const std::string& id); + static bool is_text_model(const std::string& id); + // True when the id is a documented text model that accepts "max". + static bool accepts_max_effort(const std::string& id); + static const char* kind_name(ModelKind kind); +}; + +// Documented reasoning efforts. "none" is documented as rejected by Muse +// models, so it is listed separately and never offered. +struct ReasoningEfforts { + static const std::vector& accepted(); // minimal..max + static bool valid(const std::string& value); +}; + +} // namespace molo::meta diff --git a/src/meta/chat_completions.cpp b/src/meta/chat_completions.cpp new file mode 100644 index 0000000..752d39b --- /dev/null +++ b/src/meta/chat_completions.cpp @@ -0,0 +1,293 @@ +#include "meta/chat_completions.h" + +#include "meta/calls.h" +#include "meta/caps.h" +#include "meta/check.h" +#include "meta/schema_check.h" + +namespace molo::meta { + +namespace { +Json msg(const char* role, const Json& content) { + Json o = Json::object(); + o.set("role", Json(role)); + o.set("content", content); + return o; +} +Json part(const char* type) { + Json o = Json::object(); + o.set("type", Json(type)); + return o; +} +} // namespace + +Json ChatMessage::developer(const std::string& t) { return msg("developer", Json(t)); } +Json ChatMessage::system(const std::string& t) { return msg("system", Json(t)); } +Json ChatMessage::user(const std::string& t) { return msg("user", Json(t)); } + +Json ChatMessage::user_parts(const std::vector& parts) { + Json a = Json::array(); + for (const Json& p : parts) a.push(p); + return msg("user", a); +} + +Json ChatMessage::assistant(const std::string& t) { return msg("assistant", Json(t)); } + +Json ChatMessage::assistant_tool_calls(const std::string& t, const std::vector& calls) { + Json m = msg("assistant", t.empty() ? Json() : Json(t)); + Json a = Json::array(); + for (const Json& c : calls) a.push(c); + m.set("tool_calls", a); + return m; +} + +Json ChatMessage::tool_result(const std::string& id, const std::string& content) { + Check::call_id(id); + Json m = msg("tool", Json(content)); + m.set("tool_call_id", Json(id)); + return m; +} + +Json ChatMessage::text_part(const std::string& t) { + Json p = part("text"); + p.set("text", Json(t)); + return p; +} + +Json ChatMessage::image_part(const std::string& url, const std::string& detail) { + Check::non_empty("image url", url); + if (!detail.empty()) Check::one_of("detail", detail, {"auto", "low", "high"}); + Json p = part("image_url"); + Json i = Json::object(); + i.set("url", Json(url)); + JsonX::set_if(i, "detail", detail); + p.set("image_url", i); + return p; +} + +Json ChatMessage::video_part(const std::string& url, double fps) { + Check::non_empty("video url", url); + Json p = part("video_url"); + Json v = Json::object(); + v.set("url", Json(url)); + if (fps > 0) v.set("fps", Json(fps)); + p.set("video_url", v); + return p; +} + +Json ChatMessage::file_part(const std::string& filename, const std::string& data_url, + const std::string& file_id) { + if (data_url.empty() && file_id.empty()) Check::fail("file part needs file_data or file_id"); + Json p = part("file"); + Json f = Json::object(); + JsonX::set_if(f, "filename", filename); + JsonX::set_if(f, "file_data", data_url); + if (!file_id.empty()) { + Check::segment("file_id", file_id); + f.set("file_id", Json(file_id)); + } + p.set("file", f); + return p; +} + +Json ChatMessage::audio_part(const std::string& format, const std::string& b64) { + Check::one_of("format", format, {"wav", "mp3"}); + Check::non_empty("audio data", b64); + Json p = part("input_audio"); + Json a = Json::object(); + a.set("data", Json(b64)); + a.set("format", Json(format)); + p.set("input_audio", a); + return p; +} + +Json ChatMessage::tool_call(const std::string& id, const std::string& name, + const std::string& arguments) { + Check::call_id(id); + Check::function_name(name); + Json c = Json::object(); + c.set("id", Json(id)); + c.set("type", Json("function")); + Json f = Json::object(); + f.set("name", Json(name)); + f.set("arguments", Json(arguments.empty() ? "{}" : arguments)); + c.set("function", f); + return c; +} + +Json ChatMessage::function_tool(const std::string& name, const std::string& description, + const Json& parameters, bool strict) { + Check::function_name(name); + if (parameters.is_object()) SchemaCheck::check(parameters, strict); + Json f = Json::object(); + f.set("name", Json(name)); + JsonX::set_if(f, "description", description); + f.set("parameters", parameters.is_object() ? parameters : Json::object()); + if (strict) f.set("strict", Json(true)); + Json t = Json::object(); + t.set("type", Json("function")); + t.set("function", f); + return t; +} + +Json ChatCompletionRequest::format_json_object() { + Json f = Json::object(); + f.set("type", Json("json_object")); + return f; +} + +Json ChatCompletionRequest::format_json_schema(const std::string& name, const Json& schema, + bool strict, const std::string& description) { + Check::non_empty("name", name); + SchemaCheck::check(schema, strict); + Json js = Json::object(); + js.set("name", Json(name)); + js.set("schema", schema); + if (strict) js.set("strict", Json(true)); + JsonX::set_if(js, "description", description); + Json f = Json::object(); + f.set("type", Json("json_schema")); + f.set("json_schema", js); + return f; +} + +void ChatCompletionRequest::validate() const { + Check::model_id(model); + if (messages.empty() || messages.size() > MetaCaps::kMaxInputItems) + Check::fail("messages needs 1.." + std::to_string(MetaCaps::kMaxInputItems) + " entries"); + for (const Json& m : messages) { + Check::one_of("message role", JsonX::str(m, "role"), + {"developer", "system", "user", "assistant", "tool"}); + } + if (tools.size() > MetaCaps::kMaxTools) Check::fail("too many tools"); + for (const Json& t : tools) { + if (JsonX::str(t, "type") != "function") + Check::fail("chat completions accepts function tools only (custom tools are Responses-only)"); + if (const Json* f = JsonX::obj(t, "function")) Check::function_name(JsonX::str(*f, "name")); + } + if (max_completion_tokens && *max_completion_tokens < 1) + Check::fail("max_completion_tokens must be positive"); + if (temperature) Check::temperature(*temperature); + if (top_p) Check::top_p(*top_p); + if (frequency_penalty) Check::penalty("frequency_penalty", *frequency_penalty); + if (presence_penalty) Check::penalty("presence_penalty", *presence_penalty); + Check::reasoning_effort(model, reasoning_effort); + Check::safety_identifier(safety_identifier); + Check::prompt_cache_retention(prompt_cache_retention); + if (!service_tier.empty()) + Check::one_of("service_tier", service_tier, {"auto", "default", "flex", "scale", "priority"}); + if (metadata.size() > MetaCaps::kMaxMetadataPairs) Check::fail("too many metadata pairs"); + if (response_format.is_object()) { + std::string type = JsonX::str(response_format, "type"); + Check::one_of("response_format.type", type, {"text", "json_object", "json_schema"}); + if (type == "json_schema") { + const Json* js = JsonX::obj(response_format, "json_schema"); + const Json* s = js ? JsonX::obj(*js, "schema") : nullptr; + if (!s) Check::fail("response_format json_schema needs json_schema.schema"); + SchemaCheck::check(*s, JsonX::flag(*js, "strict")); + } + } + // Fields the docs reject on this endpoint must not arrive through `extra`. + if (extra.is_object()) + for (const char* banned : {"n", "stop", "logit_bias", "prediction", "verbosity", + "web_search_options", "modalities", "audio", "logprobs", "text", + "max_output_tokens"}) + if (extra.has(banned)) Check::fail(std::string(banned) + " is rejected by chat completions"); +} + +Json ChatCompletionRequest::to_json(bool stream) const { + validate(); + Json b = Json::object(); + b.set("model", Json(model)); + Json m = Json::array(); + for (const Json& x : messages) m.push(x); + b.set("messages", m); + if (!tools.empty()) { + Json t = Json::array(); + for (const Json& x : tools) t.push(x); + b.set("tools", t); + } + if (response_format.is_object()) b.set("response_format", response_format); + JsonX::set_if(b, "reasoning_effort", reasoning_effort); + if (max_completion_tokens) b.set("max_completion_tokens", Json(*max_completion_tokens)); + if (temperature) b.set("temperature", Json(*temperature)); + if (top_p) b.set("top_p", Json(*top_p)); + if (frequency_penalty) b.set("frequency_penalty", Json(*frequency_penalty)); + if (presence_penalty) b.set("presence_penalty", Json(*presence_penalty)); + if (seed) b.set("seed", Json(*seed)); + if (parallel_tool_calls) b.set("parallel_tool_calls", Json(*parallel_tool_calls)); + JsonX::set_if(b, "safety_identifier", safety_identifier); + JsonX::set_if(b, "prompt_cache_key", prompt_cache_key); + JsonX::set_if(b, "prompt_cache_retention", prompt_cache_retention); + JsonX::set_if(b, "service_tier", service_tier); + if (!metadata.empty()) { + Json md = Json::object(); + for (const auto& [k, v] : metadata) md.set(k, Json(v)); + b.set("metadata", md); + } + if (stream) { + b.set("stream", Json(true)); + if (include_usage) { + Json so = Json::object(); + so.set("include_usage", Json(true)); + b.set("stream_options", so); + } + } + JsonX::merge(b, extra); + return b; +} + +void ChatChunkAccumulator::feed(const Json& chunk) { + id_ = JsonX::str(chunk, "id", id_); + model_ = JsonX::str(chunk, "model", model_); + if (const Json* u = JsonX::obj(chunk, "usage")) { + usage_ = MetaUsage::from_chat(*u); + has_usage_ = true; + } + const Json* choices = JsonX::arr(chunk, "choices"); + if (!choices || choices->as_array().empty()) return; + const Json& c = choices->as_array()[0]; + std::string fr = JsonX::str(c, "finish_reason"); + if (!fr.empty()) finish_reason_ = fr; + const Json* d = JsonX::obj(c, "delta"); + if (!d) return; + text_ += JsonX::str(*d, "content"); + reasoning_ += JsonX::str(*d, "reasoning_content"); + if (const Json* tcs = JsonX::arr(*d, "tool_calls")) + for (const Json& tc : tcs->as_array()) { + long idx = JsonX::num(tc, "index", 0); + ChatToolCallState* st = nullptr; + for (ChatToolCallState& s : calls_) + if (s.index == idx) st = &s; + if (!st) { + calls_.push_back({}); + st = &calls_.back(); + st->index = idx; + } + st->id = JsonX::str(tc, "id", st->id); + if (const Json* f = JsonX::obj(tc, "function")) { + st->name = JsonX::str(*f, "name", st->name); + st->arguments += JsonX::str(*f, "arguments"); + } + } +} + +Json ChatCompletionsService::create(const ChatCompletionRequest& request) { + Json body = request.to_json(false); + return client_->post(path_for(Call::kChatCompletionsCreate), body); +} + +ChatChunkAccumulator ChatCompletionsService::stream(const ChatCompletionRequest& request, + const ChunkSink& sink) { + Json body = request.to_json(true); + ChatChunkAccumulator acc; + std::size_t seen = 0; + client_->stream(path_for(Call::kChatCompletionsCreate), body, [&](const llm::SseEvent& e) { + if (++seen > MetaCaps::kMaxStreamEvents) return false; + acc.feed(e.data); + return sink ? sink(e.data) : true; + }); + return acc; +} + +} // namespace molo::meta diff --git a/src/meta/chat_completions.h b/src/meta/chat_completions.h new file mode 100644 index 0000000..652bdc3 --- /dev/null +++ b/src/meta/chat_completions.h @@ -0,0 +1,114 @@ +#pragma once + +// POST /chat/completions: the OpenAI-compatible messages-array endpoint for +// the Muse Spark and Muse Wonder text models. It does not carry reasoning +// across turns (reasoning_content is redacted for external keys); use the +// Responses API for that. Rules enforced before any traffic (docs: +// protocols/chat-completions): n is always 1, stop / logit_bias / prediction / +// verbosity / web_search_options / modalities are not offered, tool_choice is +// only "auto", reasoning_effort "none" is rejected, response_format and tools +// pass the schema pre-flight. + +#include +#include +#include +#include + +#include "llm/api_client.h" +#include "meta/jsonx.h" +#include "meta/usage.h" + +namespace molo::meta { + +class ChatMessage { + public: + static Json developer(const std::string& text); + static Json system(const std::string& text); // same level as developer + static Json user(const std::string& text); + static Json user_parts(const std::vector& parts); + static Json assistant(const std::string& text); + static Json assistant_tool_calls(const std::string& text, const std::vector& tool_calls); + static Json tool_result(const std::string& tool_call_id, const std::string& content); + // Content parts. + static Json text_part(const std::string& text); + static Json image_part(const std::string& url, const std::string& detail = ""); + static Json video_part(const std::string& url, double fps = 0); + static Json file_part(const std::string& filename, const std::string& file_data_url, + const std::string& file_id = ""); + static Json audio_part(const std::string& format, const std::string& base64_data); + // {id, type:"function", function:{name, arguments}} + static Json tool_call(const std::string& id, const std::string& name, + const std::string& arguments_json); + // Tool definition: {type:"function", function:{name, description, parameters, strict}} + static Json function_tool(const std::string& name, const std::string& description, + const Json& parameters, bool strict = false); +}; + +struct ChatCompletionRequest { + std::string model; + std::vector messages; + std::vector tools; + Json response_format; // null, or {type: json_object | json_schema, json_schema:{...}} + std::string reasoning_effort; // minimal..max + std::optional max_completion_tokens; + std::optional temperature; + std::optional top_p; + std::optional frequency_penalty; + std::optional presence_penalty; + std::optional seed; + std::optional parallel_tool_calls; + std::string safety_identifier; + std::string prompt_cache_key; + std::string prompt_cache_retention; + std::string service_tier; + bool include_usage = true; // stream_options.include_usage when streaming + std::vector> metadata; + Json extra; + + void validate() const; + Json to_json(bool stream = false) const; + + static Json format_json_object(); + static Json format_json_schema(const std::string& name, const Json& schema, bool strict = false, + const std::string& description = ""); +}; + +struct ChatToolCallState { + long index = 0; + std::string id, name, arguments; +}; + +// Folds chat.completion.chunk objects into text, reasoning_content, tool calls, +// finish_reason and usage. +class ChatChunkAccumulator { + public: + void feed(const Json& chunk); + const std::string& text() const { return text_; } + const std::string& reasoning() const { return reasoning_; } + const std::string& finish_reason() const { return finish_reason_; } + const std::string& id() const { return id_; } + const std::string& model() const { return model_; } + const std::vector& tool_calls() const { return calls_; } + bool has_usage() const { return has_usage_; } + MetaUsage usage() const { return usage_; } + + private: + std::string text_, reasoning_, finish_reason_, id_, model_; + std::vector calls_; + MetaUsage usage_; + bool has_usage_ = false; +}; + +class ChatCompletionsService : public llm::Service { + public: + using ChunkSink = std::function; // false: stop + + explicit ChatCompletionsService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + + Json create(const ChatCompletionRequest& request); + ChatChunkAccumulator stream(const ChatCompletionRequest& request, + const ChunkSink& sink = nullptr); +}; + +} // namespace molo::meta diff --git a/src/meta/chat_model.cpp b/src/meta/chat_model.cpp new file mode 100644 index 0000000..c80403d --- /dev/null +++ b/src/meta/chat_model.cpp @@ -0,0 +1,285 @@ +#include "meta/chat_model.h" + +#include + +#include "meta/check.h" +#include "meta/response_input.h" +#include "security/crypto/Base64.h" + +namespace molo::meta { + +namespace { + +std::string image_url_of(const llm::ImageData& im) { + if (!im.url.empty()) return im.url; + if (im.bytes.empty()) Check::fail("image has neither url nor bytes"); + return "data:" + im.mime + ";base64," + security::crypto::Base64::encode(im.bytes); +} + +void common_checks(const llm::ChatRequest& r) { + if (!r.stop.empty()) Check::fail("stop sequences are not supported by Muse models"); + if (!r.tool_choice.empty() && r.tool_choice != "auto") + Check::fail("tool_choice must be auto (the only value Meta accepts)"); +} + +std::string finish_of(const std::string& v) { + if (v == "stop" || v == "length" || v == "tool_calls" || v == "content_filter") return v; + if (v == "function_call") return "tool_calls"; + return v; +} + +} // namespace + +MetaChatModel::MetaChatModel(std::shared_ptr client, Endpoint endpoint) + : responses_(client), chat_(client), endpoint_(endpoint) {} + +ResponseRequest MetaChatModel::to_response_request(const llm::ChatRequest& r) const { + common_checks(r); + ResponseRequest q; + q.model = r.model; + q.instructions = r.system; + Json input = Json::array(); + for (const llm::Message& m : r.messages) { + if (m.role == llm::Role::kSystem) { + input.push(ResponseInput::developer(m.text())); + continue; + } + Json parts = Json::array(); + bool user = m.role == llm::Role::kUser; + auto flush = [&](const char* role) { + if (parts.as_array().empty()) return; + input.push(ResponseInput::message(role, parts)); + parts = Json::array(); + }; + for (const llm::ContentPart& p : m.parts) { + if (!p.reasoning_signature.empty()) { + flush("assistant"); + input.push(ResponseInput::reasoning(p.reasoning_signature)); + } + if (!p.text.empty()) { + if (user) parts.push(ResponseInput::text(p.text)); + else { + Json t = Json::object(); + t.set("type", Json("output_text")); + t.set("text", Json(p.text)); + t.set("annotations", Json::array()); + parts.push(t); + } + } + if (p.image) parts.push(ResponseInput::image_url(image_url_of(*p.image))); + if (p.tool_call) { + flush("assistant"); + input.push(ResponseInput::function_call(p.tool_call->id, p.tool_call->name, + p.tool_call->arguments.dump())); + } + if (p.tool_result) { + flush(user ? "user" : "assistant"); + input.push(ResponseInput::function_call_output(p.tool_result->call_id, p.tool_result->content)); + } + } + flush(m.role == llm::Role::kAssistant ? "assistant" : "user"); + } + q.input = input; + for (const llm::ToolSpec& t : r.tools) + q.tools.push_back(ToolBuilder::function(t.name, t.description, t.parameters)); + if (r.max_tokens) q.max_output_tokens = *r.max_tokens; + q.temperature = r.temperature; + q.top_p = r.top_p; + q.reasoning_effort = r.reasoning_effort; + if (request_reasoning_summary) q.reasoning_summary = "auto"; + if (r.json_mode) q.text_format = ResponseRequest::format_json_object(); + if (r.response_schema.is_object()) + q.text_format = ResponseRequest::format_json_schema("output", r.response_schema); + q.store = false; // stateless: replay reasoning ourselves + q.include = {"reasoning.encrypted_content"}; + q.extra = r.extra; + return q; +} + +ChatCompletionRequest MetaChatModel::to_chat_request(const llm::ChatRequest& r) const { + common_checks(r); + ChatCompletionRequest q; + q.model = r.model; + if (!r.system.empty()) q.messages.push_back(ChatMessage::developer(r.system)); + for (const llm::Message& m : r.messages) { + if (m.role == llm::Role::kSystem) { q.messages.push_back(ChatMessage::developer(m.text())); continue; } + if (m.role == llm::Role::kAssistant) { + std::vector calls; + for (const llm::ContentPart& p : m.parts) + if (p.tool_call) + calls.push_back(ChatMessage::tool_call(p.tool_call->id, p.tool_call->name, p.tool_call->arguments.dump())); + q.messages.push_back(calls.empty() ? ChatMessage::assistant(m.text()) + : ChatMessage::assistant_tool_calls(m.text(), calls)); + continue; + } + std::vector parts; + bool images = false; + for (const llm::ContentPart& p : m.parts) { + if (p.tool_result) { + q.messages.push_back(ChatMessage::tool_result(p.tool_result->call_id, p.tool_result->content)); + continue; + } + if (!p.text.empty()) parts.push_back(ChatMessage::text_part(p.text)); + if (p.image) { parts.push_back(ChatMessage::image_part(image_url_of(*p.image))); images = true; } + } + if (parts.empty()) continue; + if (images) q.messages.push_back(ChatMessage::user_parts(parts)); + else q.messages.push_back(ChatMessage::user(m.text())); + } + for (const llm::ToolSpec& t : r.tools) + q.tools.push_back(ChatMessage::function_tool(t.name, t.description, t.parameters)); + if (r.max_tokens) q.max_completion_tokens = *r.max_tokens; + q.temperature = r.temperature; + q.top_p = r.top_p; + q.reasoning_effort = r.reasoning_effort; + if (r.json_mode) q.response_format = ChatCompletionRequest::format_json_object(); + if (r.response_schema.is_object()) + q.response_format = ChatCompletionRequest::format_json_schema("output", r.response_schema); + q.extra = r.extra; + return q; +} + +llm::ChatResponse MetaChatModel::from_response(const Json& doc) { + llm::ChatResponse out; + out.raw = doc; + out.id = JsonX::str(doc, "id"); + out.model = JsonX::str(doc, "model"); + out.message.role = llm::Role::kAssistant; + bool calls = false, refusal = false; + if (const Json* items = JsonX::arr(doc, "output")) + for (const Json& item : items->as_array()) { + std::string type = JsonX::str(item, "type"); + if (type == "message") { + if (const Json* c = JsonX::arr(item, "content")) + for (const Json& p : c->as_array()) { + if (JsonX::str(p, "type") == "output_text") + out.message.parts.push_back(llm::ContentPart::of_text(JsonX::str(p, "text"))); + else if (JsonX::str(p, "type") == "refusal") + refusal = true; + } + } else if (type == "reasoning") { + llm::ContentPart part; + if (const Json* s = JsonX::arr(item, "summary")) + for (const Json& t : s->as_array()) part.reasoning += JsonX::str(t, "text"); + part.reasoning_signature = JsonX::str(item, "encrypted_content"); + if (!part.reasoning.empty() || !part.reasoning_signature.empty()) + out.message.parts.push_back(std::move(part)); + } else if (type == "function_call") { + llm::ToolCall call; + call.id = JsonX::str(item, "call_id"); + call.name = JsonX::str(item, "name"); + try { + call.arguments = Json::parse(JsonX::str(item, "arguments", "{}")); + } catch (const std::exception&) { + call.arguments = Json::object(); + } + out.message.parts.push_back(llm::ContentPart::of_call(std::move(call))); + calls = true; + } + } + std::string status = JsonX::str(doc, "status"); + out.finish_reason = calls ? "tool_calls" : refusal ? "content_filter" + : status == "incomplete" ? "length" : "stop"; + if (const Json* u = JsonX::obj(doc, "usage")) out.usage = MetaUsage::from_responses(*u).to_neutral(); + return out; +} + +llm::ChatResponse MetaChatModel::from_chat(const Json& doc) { + llm::ChatResponse out; + out.raw = doc; + out.id = JsonX::str(doc, "id"); + out.model = JsonX::str(doc, "model"); + out.message.role = llm::Role::kAssistant; + if (const Json* ch = JsonX::arr(doc, "choices"); ch && !ch->as_array().empty()) { + const Json& c = ch->as_array()[0]; + out.finish_reason = finish_of(JsonX::str(c, "finish_reason")); + if (const Json* m = JsonX::obj(c, "message")) { + std::string text = JsonX::str(*m, "content"); + if (!text.empty()) out.message.parts.push_back(llm::ContentPart::of_text(text)); + if (const Json* tcs = JsonX::arr(*m, "tool_calls")) + for (const Json& tc : tcs->as_array()) { + llm::ToolCall call; + call.id = JsonX::str(tc, "id"); + if (const Json* f = JsonX::obj(tc, "function")) { + call.name = JsonX::str(*f, "name"); + try { + call.arguments = Json::parse(JsonX::str(*f, "arguments", "{}")); + } catch (const std::exception&) { + call.arguments = Json::object(); + } + } + out.message.parts.push_back(llm::ContentPart::of_call(std::move(call))); + } + } + } + if (const Json* u = JsonX::obj(doc, "usage")) out.usage = MetaUsage::from_chat(*u).to_neutral(); + return out; +} + +llm::ChatResponse MetaChatModel::generate(const llm::ChatRequest& request) { + if (endpoint_ == Endpoint::kResponses) + return from_response(responses_.create(to_response_request(request)).body); + return from_chat(chat_.create(to_chat_request(request))); +} + +void MetaChatModel::stream(const llm::ChatRequest& request, + const std::function& on_chunk) { + if (endpoint_ == Endpoint::kChatCompletions) { + chat_.stream(to_chat_request(request), [&](const Json& chunk) { + llm::ChatChunk c; + std::string finish; + if (const Json* ch = JsonX::arr(chunk, "choices"); ch && !ch->as_array().empty()) { + const Json& first = ch->as_array()[0]; + finish = finish_of(JsonX::str(first, "finish_reason")); + if (const Json* d = JsonX::obj(first, "delta")) { + c.text_delta = JsonX::str(*d, "content"); + c.reasoning_delta = JsonX::str(*d, "reasoning_content"); + if (const Json* tcs = JsonX::arr(*d, "tool_calls")) + for (const Json& tc : tcs->as_array()) { + c.tool_call_id = JsonX::str(tc, "id"); + if (const Json* f = JsonX::obj(tc, "function")) { + c.tool_call_name = JsonX::str(*f, "name"); + c.tool_args_delta = JsonX::str(*f, "arguments"); + } + } + } + } + c.finish_reason = finish; + if (const Json* u = JsonX::obj(chunk, "usage")) c.usage = MetaUsage::from_chat(*u).to_neutral(); + return on_chunk(c); + }); + return; + } + std::map> by_item; // item id -> call id, name + bool saw_call = false; + responses_.stream(to_response_request(request), [&](const ResponseEvent& ev) { + llm::ChatChunk c; + const Json& d = ev.data; + if (ev.type == "response.output_text.delta") c.text_delta = JsonX::str(d, "delta"); + else if (ev.type == "response.reasoning_summary_text.delta") c.reasoning_delta = JsonX::str(d, "delta"); + else if (ev.type == "response.output_item.added") { + const Json* item = JsonX::obj(d, "item"); + if (!item || JsonX::str(*item, "type") != "function_call") return true; + saw_call = true; + by_item[JsonX::str(*item, "id")] = {JsonX::str(*item, "call_id"), JsonX::str(*item, "name")}; + c.tool_call_id = JsonX::str(*item, "call_id"); + c.tool_call_name = JsonX::str(*item, "name"); + } else if (ev.type == "response.function_call_arguments.delta") { + auto it = by_item.find(JsonX::str(d, "item_id")); + if (it != by_item.end()) c.tool_call_id = it->second.first; + c.tool_args_delta = JsonX::str(d, "delta"); + } else if (ev.type == "response.completed" || ev.type == "response.incomplete") { + const Json* r = JsonX::obj(d, "response"); + c.finish_reason = saw_call ? "tool_calls" : ev.type == "response.incomplete" ? "length" : "stop"; + if (r) + if (const Json* u = JsonX::obj(*r, "usage")) c.usage = MetaUsage::from_responses(*u).to_neutral(); + } else if (ev.type == "response.failed" || ev.type == "error") { + throw llm::ApiError("meta", 502, "stream_failed", JsonX::str(d, "message", "response failed")); + } else { + return true; + } + return on_chunk(c); + }); +} + +} // namespace molo::meta diff --git a/src/meta/chat_model.h b/src/meta/chat_model.h new file mode 100644 index 0000000..f3e49d7 --- /dev/null +++ b/src/meta/chat_model.h @@ -0,0 +1,46 @@ +#pragma once + +// MetaChatModel: llm::ChatModel over the Meta Model API. Default endpoint is +// the Responses API (store=false plus encrypted reasoning replay: reasoning +// parts carry the encrypted_content in ContentPart::reasoning_signature and +// are replayed verbatim). kChatCompletions maps onto /chat/completions, which +// cannot carry reasoning across turns. +// Unsupported combinations throw ApiError::invalid before any traffic: stop +// sequences, tool_choice other than "auto" (docs: only "auto"). + +#include + +#include "llm/chat_types.h" +#include "meta/chat_completions.h" +#include "meta/responses.h" + +namespace molo::meta { + +class MetaChatModel : public llm::ChatModel { + public: + enum class Endpoint { kResponses, kChatCompletions }; + + explicit MetaChatModel(std::shared_ptr client, + Endpoint endpoint = Endpoint::kResponses); + + std::string vendor() const override { return "meta"; } + llm::ChatResponse generate(const llm::ChatRequest& request) override; + void stream(const llm::ChatRequest& request, + const std::function& on_chunk) override; + + // Exposed for tests and the demo: the wire bodies without sending. + ResponseRequest to_response_request(const llm::ChatRequest& request) const; + ChatCompletionRequest to_chat_request(const llm::ChatRequest& request) const; + static llm::ChatResponse from_response(const Json& response); + static llm::ChatResponse from_chat(const Json& completion); + + // Summaries of reasoning in the Responses endpoint (reasoning.summary=auto). + bool request_reasoning_summary = true; + + private: + ResponsesService responses_; + ChatCompletionsService chat_; + Endpoint endpoint_; +}; + +} // namespace molo::meta diff --git a/src/meta/check.cpp b/src/meta/check.cpp new file mode 100644 index 0000000..29a8436 --- /dev/null +++ b/src/meta/check.cpp @@ -0,0 +1,93 @@ +#include "meta/check.h" + +#include + +#include "llm/api_client.h" +#include "llm/error.h" +#include "meta/caps.h" +#include "meta/catalog.h" + +namespace molo::meta { + +void Check::fail(const std::string& message) { throw llm::ApiError::invalid("meta", message); } + +void Check::non_empty(const char* field, const std::string& value) { + if (value.empty()) fail(std::string(field) + " is required"); +} + +void Check::max_bytes(const char* field, const std::string& value, std::size_t cap) { + if (value.size() > cap) fail(std::string(field) + " is over " + std::to_string(cap) + " bytes"); +} + +void Check::range(const char* field, double value, double lo, double hi) { + if (!(value >= lo && value <= hi) || std::isnan(value)) + fail(std::string(field) + " must be " + std::to_string(lo) + ".." + std::to_string(hi)); +} + +void Check::one_of(const char* field, const std::string& value, + std::initializer_list allowed) { + std::string list; + for (const char* a : allowed) { + if (value == a) return; + list += (list.empty() ? "" : ", ") + std::string(a); + } + fail(std::string(field) + " must be one of " + list); +} + +void Check::model_id(const std::string& model) { + non_empty("model", model); + if (model.size() > MetaCaps::kMaxModelBytes || !llm::Service::valid_id(model)) + fail("model must be 1.." + std::to_string(MetaCaps::kMaxModelBytes) + + " chars of A-Z a-z 0-9 . _ : -"); +} + +void Check::segment(const char* what, const std::string& id) { + if (id.empty() || id.size() > MetaCaps::kMaxIdBytes || !llm::Service::valid_id(id)) + fail(std::string(what) + " is not a valid id"); +} + +void Check::function_name(const std::string& name) { + if (name.empty() || name.size() > 128) fail("function name must be 1..128 chars"); + int dots = 0; + for (char c : name) { + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_' || c == '-' || c == '.'; + if (!ok) fail("function name may use only A-Z a-z 0-9 _ - ."); + if (c == '.') ++dots; + } + if (dots > 1) fail("function name may contain at most one dot"); +} + +void Check::reasoning_effort(const std::string& model, const std::string& effort) { + if (effort.empty()) return; + if (effort == "none") fail("reasoning effort \"none\" is rejected by Muse models"); + if (!ReasoningEfforts::valid(effort)) + fail("reasoning effort must be one of minimal, low, medium, high, xhigh, max"); + if (effort == "max") { + const ModelInfo* m = ModelCatalog::find(model); + if (m && m->kind == ModelKind::kText && !m->supports_max) + fail("reasoning effort \"max\" is supported only by muse-wonder-1.0 and muse-spark-1.3"); + } +} + +void Check::call_id(const std::string& id) { + if (id.empty() || id.size() > MetaCaps::kMaxCallIdBytes) fail("call_id must be 1..64 bytes"); +} + +void Check::top_p(double value) { + if (!(value > 0.0 && value <= 1.0)) fail("top_p must be greater than 0 and at most 1"); +} + +void Check::temperature(double value) { range("temperature", value, 0.0, 2.0); } + +void Check::penalty(const char* field, double value) { range(field, value, -2.0, 2.0); } + +void Check::safety_identifier(const std::string& value) { + max_bytes("safety_identifier", value, MetaCaps::kMaxSafetyIdBytes); +} + +void Check::prompt_cache_retention(const std::string& value) { + if (!value.empty()) one_of("prompt_cache_retention", value, {"in_memory", "24h"}); +} + +} // namespace molo::meta diff --git a/src/meta/check.h b/src/meta/check.h new file mode 100644 index 0000000..aeeb766 --- /dev/null +++ b/src/meta/check.h @@ -0,0 +1,40 @@ +#pragma once + +// Local validation shared by every request builder. A failed check throws +// llm::ApiError::invalid("meta", ...) before any network traffic, naming the +// field. Rules come from the docs (error-handling, protocols/*, reasoning, +// tool-calling, structured-output); each cites its page in the SPEC. + +#include +#include + +namespace molo::meta { + +class Check { + public: + [[noreturn]] static void fail(const std::string& message); + + static void non_empty(const char* field, const std::string& value); + static void max_bytes(const char* field, const std::string& value, std::size_t cap); + static void range(const char* field, double value, double lo, double hi); + static void one_of(const char* field, const std::string& value, + std::initializer_list allowed); + // 1..kMaxModelBytes of [A-Za-z0-9._:-]. + static void model_id(const std::string& model); + // Response / file / model ids: one safe path segment. + static void segment(const char* what, const std::string& id); + // Function tool names: [A-Za-z0-9_.-]+, at most one dot (docs: tool-calling). + static void function_name(const std::string& name); + // Reasoning effort for a model: the documented values; "none" is rejected + // by Muse models; "max" only where the catalog says so. + static void reasoning_effort(const std::string& model, const std::string& effort); + // call_id / tool_call_id: 1..64 bytes. + static void call_id(const std::string& id); + static void top_p(double value); // 0 < p <= 1 + static void temperature(double value); // 0..2 + static void penalty(const char* field, double value); // -2..2 + static void safety_identifier(const std::string& value); + static void prompt_cache_retention(const std::string& value); // in_memory | 24h +}; + +} // namespace molo::meta diff --git a/src/meta/client.cpp b/src/meta/client.cpp new file mode 100644 index 0000000..28a36f5 --- /dev/null +++ b/src/meta/client.cpp @@ -0,0 +1,71 @@ +#include "meta/client.h" + +#include + +#include "clients/transports/http_transport.h" +#include "llm/config.h" +#include "meta/caps.h" + +namespace molo::meta { + +std::string MetaClient::key_from_env() { + std::string key = llm::Config::get(MetaEndpoints::kKeyEnv); + if (key.empty()) key = llm::Config::get(MetaEndpoints::kAltKeyEnv); + return key; +} + +llm::ApiClientConfig MetaClient::config(const std::string& base_url) { + llm::ApiClientConfig c; + c.vendor = MetaEndpoints::kVendor; + c.base_url = base_url; + c.key_env = MetaEndpoints::kKeyEnv; // named in the not-configured message + c.auth = llm::AuthStyle::kBearer; + return c; +} + +std::shared_ptr MetaClient::with_key(const std::string& key) { + return std::make_shared( + std::make_unique(), config(), key); +} + +std::shared_ptr MetaClient::from_env() { return with_key(key_from_env()); } + +std::shared_ptr MetaClient::unauthenticated() { + return with_key(kPlaceholderKey); +} + +bool RetryPolicy::retryable(const llm::ApiError& error) { + int s = error.http_status(); + return s == 429 || s == 500 || s == 502 || s == 503; +} + +int RetryPolicy::delay_ms(int attempt, unsigned seed) { + if (attempt < 0) attempt = 0; + long delay = MetaCaps::kBackoffBaseMs; + for (int i = 0; i < attempt && delay < MetaCaps::kBackoffCapMs; ++i) delay *= 2; + delay = std::min(delay, MetaCaps::kBackoffCapMs); + // Deterministic jitter (xorshift on seed and attempt), 0..delay/2. + unsigned x = seed * 2654435761u + static_cast(attempt) * 40503u + 1u; + x ^= x << 13; x ^= x >> 17; x ^= x << 5; + long jitter = static_cast(x % static_cast(delay / 2 + 1)); + return static_cast(delay + jitter); +} + +RetryDecision RetryPolicy::decide(const llm::ApiError& error, int attempt, unsigned seed) { + RetryDecision d; + if (attempt >= MetaCaps::kMaxRetries) { + d.reason = "retry budget spent"; + return d; + } + if (!retryable(error)) { + d.reason = error.http_status() == 504 ? "504 gateway_timeout: stream or use background" + : "not retryable"; + return d; + } + d.retry = true; + d.delay_ms = delay_ms(attempt, seed); + d.reason = "transient"; + return d; +} + +} // namespace molo::meta diff --git a/src/meta/client.h b/src/meta/client.h new file mode 100644 index 0000000..e2dd280 --- /dev/null +++ b/src/meta/client.h @@ -0,0 +1,80 @@ +#pragma once + +// Builds the ApiClient for the Meta Model API: bearer auth against +// https://api.meta.ai/v1. The credential is META_API_KEY, or MODEL_API_KEY +// (the name the docs' own examples use), looked up through llm::Config. +// Also the retry policy the docs prescribe (exponential backoff with jitter on +// 429 and transient 5xx, never on 504 or other 4xx). + +#include +#include +#include +#include +#include + +#include "llm/api_client.h" +#include "llm/error.h" + +namespace molo::meta { + +struct MetaEndpoints { + static constexpr const char* kVendor = "meta"; + static constexpr const char* kBaseUrl = "https://api.meta.ai/v1"; + static constexpr const char* kRealtimeUrl = "wss://api.meta.ai/v1/asr/realtime"; + static constexpr const char* kKeyEnv = "META_API_KEY"; + static constexpr const char* kAltKeyEnv = "MODEL_API_KEY"; +}; + +class MetaClient { + public: + // META_API_KEY first, then MODEL_API_KEY; empty when neither is set. + static std::string key_from_env(); + static llm::ApiClientConfig config(const std::string& base_url = MetaEndpoints::kBaseUrl); + // Real HTTPS transport, key from the environment (null key is fine: every + // keyed call then throws NotConfiguredError before any traffic). + static std::shared_ptr from_env(); + // Same, with an explicit key (empty = not configured). + static std::shared_ptr with_key(const std::string& key); + // Client for the one unauthenticated endpoint (GET /status). The docs say no + // key is needed; a probe on 2026-10-24 showed a bogus bearer is ignored, so + // a fixed placeholder is sent to satisfy ApiClient's non-empty key rule. + static std::shared_ptr unauthenticated(); + static constexpr const char* kPlaceholderKey = "unauthenticated"; +}; + +struct RetryDecision { + bool retry = false; + int delay_ms = 0; + std::string reason; +}; + +class RetryPolicy { + public: + // 429 and 500/502/503 retry; 504 (gateway_timeout: the same request times + // out again), 400/401/402/403/404/413 and everything else do not. + static bool retryable(const llm::ApiError& error); + // base * 2^attempt capped, plus deterministic jitter in [0, delay/2] derived + // from `seed` so tests are exact. attempt counts from 0. + static int delay_ms(int attempt, unsigned seed); + static RetryDecision decide(const llm::ApiError& error, int attempt, unsigned seed); +}; + +// Runs `f`, retrying per RetryPolicy up to MetaCaps::kMaxRetries times. +// `sleep` is injectable so callers and tests control the waiting. +template +auto call_with_retry(F&& f, unsigned seed = 1, + const std::function& sleep = [](int ms) { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + }) -> decltype(f()) { + for (int attempt = 0;; ++attempt) { + try { + return f(); + } catch (const llm::ApiError& e) { + RetryDecision d = RetryPolicy::decide(e, attempt, seed); + if (!d.retry) throw; + sleep(d.delay_ms); + } + } +} + +} // namespace molo::meta diff --git a/src/meta/files.cpp b/src/meta/files.cpp new file mode 100644 index 0000000..f1e8af9 --- /dev/null +++ b/src/meta/files.cpp @@ -0,0 +1,111 @@ +#include "meta/files.h" + +#include "meta/calls.h" +#include "meta/caps.h" +#include "meta/check.h" + +namespace molo::meta { + +FileObject FileObject::from_json(const Json& j) { + FileObject f; + f.id = JsonX::str(j, "id"); + f.bytes = JsonX::num(j, "bytes"); + f.created_at = JsonX::num(j, "created_at"); + f.expires_at = JsonX::num(j, "expires_at"); + f.filename = JsonX::str(j, "filename"); + f.purpose = JsonX::str(j, "purpose"); + f.status = JsonX::str(j, "status"); + f.status_details = JsonX::str(j, "status_details"); + return f; +} + +bool FilesService::supported_mime(const std::string& mime) { + static const char* kAllowed[] = { + "image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp", "image/x-icon", + "video/mp4", "audio/mpeg", "audio/wav", "application/pdf", "text/plain", + "application/json", "application/jsonl"}; + for (const char* m : kAllowed) + if (mime == m) return true; + return false; +} + +void FileUpload::validate() const { + Check::non_empty("filename", filename); + Check::max_bytes("filename", filename, 255); + if (filename.find('/') != std::string::npos || filename.find('\\') != std::string::npos || + filename.find('"') != std::string::npos || filename.find('\n') != std::string::npos) + Check::fail("filename has forbidden characters"); + if (!FilesService::supported_mime(mime)) Check::fail("unsupported media type " + mime); + if (bytes.empty()) Check::fail("file is empty"); + if (bytes.size() > MetaCaps::kMaxUploadBytes) Check::fail("file is over the upload cap"); + Check::one_of("purpose", purpose, {"user_data", "batch"}); + if (purpose == "batch" && mime != "application/jsonl") Check::fail("batch uploads are application/jsonl"); + if (expires_after_seconds != 0) + Check::range("expires_after.seconds", static_cast(expires_after_seconds), + static_cast(MetaCaps::kMinExpireSeconds), + static_cast(MetaCaps::kMaxExpireSeconds)); +} + +FileObject FilesService::upload(const FileUpload& file) { + file.validate(); + // docs: send purpose and expires_after before the file part, file last. + clients::body::MultipartBody form; + form.add_text("purpose", file.purpose); + if (file.expires_after_seconds != 0) { + form.add_text("expires_after[anchor]", "created_at"); + form.add_text("expires_after[seconds]", std::to_string(file.expires_after_seconds)); + } + form.add_file("file", file.filename, file.mime, file.bytes); + Json out = client_->post_multipart(path_for(Call::kFilesUpload), form); + return FileObject::from_json(out); +} + +Json FilesService::list_page(const std::string& purpose, int limit, const std::string& order, + const std::string& after) { + if (limit < 1 || limit > MetaCaps::kListPageLimit) + Check::fail("limit must be 1.." + std::to_string(MetaCaps::kListPageLimit)); + if (!order.empty()) Check::one_of("order", order, {"asc", "desc"}); + if (!purpose.empty()) Check::one_of("purpose", purpose, {"user_data", "batch", "fine-tune", "evals"}); + if (!after.empty()) Check::segment("after", after); + std::string path = path_for(Call::kFilesList) + "?limit=" + std::to_string(limit); + if (!purpose.empty()) path += "&purpose=" + purpose; + if (!order.empty()) path += "&order=" + order; + if (!after.empty()) path += "&after=" + after; + return client_->get(path); +} + +FileList FilesService::list_all(const std::string& purpose, int page_size, const std::string& order) { + FileList out; + std::string after; + for (std::size_t page = 0; page < MetaCaps::kMaxListPages; ++page) { + Json doc = list_page(purpose, page_size, order, after); + ++out.pages; + const Json* data = JsonX::arr(doc, "data"); + if (data) + for (const Json& f : data->as_array()) out.files.push_back(FileObject::from_json(f)); + out.has_more = JsonX::flag(doc, "has_more"); + std::string last = JsonX::str(doc, "last_id"); + if (!out.has_more || last.empty() || last == after) return out; + after = last; + } + out.truncated = out.has_more; // page cap reached while the server had more + return out; +} + +FileObject FilesService::retrieve(const std::string& file_id) { + Check::segment("file_id", file_id); + return FileObject::from_json(client_->get(path_for(Call::kFilesRetrieve, file_id))); +} + +std::string FilesService::content(const std::string& file_id) { + Check::segment("file_id", file_id); + return client_->request_raw(method_of(Call::kFilesContent), path_for(Call::kFilesContent, file_id)) + .body; +} + +Json FilesService::remove(const std::string& file_id) { + Check::segment("file_id", file_id); + return client_->del(path_for(Call::kFilesDelete, file_id)); +} + +} // namespace molo::meta diff --git a/src/meta/files.h b/src/meta/files.h new file mode 100644 index 0000000..166adc1 --- /dev/null +++ b/src/meta/files.h @@ -0,0 +1,67 @@ +#pragma once + +// Files API: upload once (up to 1 GiB by the docs, 512 MiB by llm::Caps), then +// reference a `file-...` id from input_file / input_image / input_video parts. +// Uploaded files do not expire unless expires_after is set (3600..2592000 s); +// the team storage limit is 100 GiB. The list endpoint answers a plain-text 400 +// for a bad `order` (docs: error-handling), so order is validated locally. + +#include +#include + +#include "llm/api_client.h" +#include "meta/jsonx.h" + +namespace molo::meta { + +struct FileObject { + std::string id; + long bytes = 0; + long created_at = 0; + long expires_at = 0; // 0 = never + std::string filename; + std::string purpose; + std::string status; // uploaded | processed | error + std::string status_details; + static FileObject from_json(const Json& j); +}; + +struct FileUpload { + std::string filename; + std::string mime; // documented media type + std::string bytes; + std::string purpose = "user_data"; // user_data | batch + long expires_after_seconds = 0; // 0 = no expiry; else 3600..2592000 + + void validate() const; +}; + +struct FileList { + std::vector files; + bool has_more = false; // true when the server has more than we returned + bool truncated = false; // true when our page cap stopped the walk + std::size_t pages = 0; +}; + +class FilesService : public llm::Service { + public: + explicit FilesService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + + static bool supported_mime(const std::string& mime); + + FileObject upload(const FileUpload& file); + // One page. order: "" | asc | desc. after: a file id cursor. + Json list_page(const std::string& purpose = "", int limit = 100, const std::string& order = "", + const std::string& after = ""); + // Walks pages up to MetaCaps::kMaxListPages and reports truncation loudly. + FileList list_all(const std::string& purpose = "", int page_size = 100, + const std::string& order = ""); + FileObject retrieve(const std::string& file_id); + // Raw bytes of an uploaded file. + std::string content(const std::string& file_id); + // Destructive: frees team storage. No default, no retry. + Json remove(const std::string& file_id); +}; + +} // namespace molo::meta diff --git a/src/meta/images.cpp b/src/meta/images.cpp new file mode 100644 index 0000000..18bd0f9 --- /dev/null +++ b/src/meta/images.cpp @@ -0,0 +1,146 @@ +#include "meta/images.h" + +#include + +#include "meta/calls.h" +#include "meta/caps.h" +#include "meta/check.h" + +namespace molo::meta { + +bool valid_image_size(const std::string& size) { + if (size.empty() || size == "auto") return true; + std::size_t x = size.find('x'); + if (x == std::string::npos || x == 0 || x + 1 >= size.size() || size.size() > 16) return false; + for (std::size_t i = 0; i < size.size(); ++i) + if (i != x && (size[i] < '0' || size[i] > '9')) return false; + long w = std::strtol(size.substr(0, x).c_str(), nullptr, 10); + long h = std::strtol(size.substr(x + 1).c_str(), nullptr, 10); + if (w < 1 || h < 1) return false; + long hi = w > h ? w : h, lo = w > h ? h : w; + return hi <= lo * 16; +} + +Json ImageRequest::image_url(const std::string& url) { + Check::non_empty("image_url", url); + Json o = Json::object(); + o.set("image_url", Json(url)); + return o; +} + +Json ImageRequest::image_file(const std::string& file_id) { + Check::segment("file_id", file_id); + Json o = Json::object(); + o.set("file_id", Json(file_id)); + return o; +} + +void ImageRequest::validate(bool edit) const { + Check::model_id(model); + Check::non_empty("prompt", prompt); + Check::max_bytes("prompt", prompt, MetaCaps::kMaxPromptBytes); + if (n) Check::range("n", *n, 1, MetaCaps::kMaxImagesPerCall); + if (!valid_image_size(size)) Check::fail("size must be auto or WxH with an aspect ratio of at most 16:1"); + if (!response_format.empty()) Check::one_of("response_format", response_format, {"url", "b64_json"}); + if (!reasoning_strength.empty()) Check::one_of("reasoning_strength", reasoning_strength, {"low", "high"}); + if (edit) { + if (images.empty() || images.size() > MetaCaps::kMaxImageCount) + Check::fail("images needs 1.." + std::to_string(MetaCaps::kMaxImageCount) + " items"); + } else if (!images.empty()) { + Check::fail("images belongs to edits, not generations"); + } + if (extra.is_object()) + for (const char* banned : {"moderation", "mask", "input_fidelity", "partial_images", "seed"}) + if (extra.has(banned)) Check::fail(std::string(banned) + " is rejected by muse-image-1.0"); +} + +Json ImageRequest::to_json(bool edit, bool stream) const { + validate(edit); + Json b = Json::object(); + b.set("model", Json(model)); + b.set("prompt", Json(prompt)); + if (edit) { + Json a = Json::array(); + for (const Json& i : images) a.push(i); + b.set("images", a); + } + if (n) b.set("n", Json(*n)); + JsonX::set_if(b, "size", size); + JsonX::set_if(b, "response_format", response_format); + JsonX::set_if(b, "output_format", output_format); + JsonX::set_if(b, "reasoning_strength", reasoning_strength); + JsonX::set_if(b, "user", user); + if (enable_image_search || enable_web_search || enable_shell) { + Json t = Json::object(); + if (enable_image_search) t.set("enable_image_search", Json(*enable_image_search)); + if (enable_web_search) t.set("enable_web_search", Json(*enable_web_search)); + if (enable_shell) t.set("enable_shell", Json(*enable_shell)); + b.set("tool_enablement", t); + } + if (stream) b.set("stream", Json(true)); + JsonX::merge(b, extra); + return b; +} + +ImageResult ImageResult::from_json(const Json& j) { + ImageResult r; + r.created = JsonX::num(j, "created"); + r.output_format = JsonX::str(j, "output_format"); + if (const Json* u = JsonX::obj(j, "usage")) r.usage = *u; + if (const Json* d = JsonX::arr(j, "data")) + for (const Json& i : d->as_array()) { + GeneratedImage x; + x.b64_json = JsonX::str(i, "b64_json"); + x.url = JsonX::str(i, "url"); + x.revised_prompt = JsonX::str(i, "revised_prompt"); + r.data.push_back(std::move(x)); + } + return r; +} + +ImageResult ImagesService::generate(const ImageRequest& request) { + Json body = request.to_json(false, false); + return ImageResult::from_json(client_->post(path_for(Call::kImagesGenerate), body)); +} + +ImageResult ImagesService::edit(const ImageRequest& request) { + Json body = request.to_json(true, false); + return ImageResult::from_json(client_->post(path_for(Call::kImagesEdit), body)); +} + +ImageResult ImagesService::edit_multipart( + const ImageRequest& request, const std::vector>& files) { + if (files.empty() || files.size() > MetaCaps::kMaxImageCount) + Check::fail("edit needs 1.." + std::to_string(MetaCaps::kMaxImageCount) + " image files"); + ImageRequest probe = request; + probe.images = {ImageRequest::image_url("multipart")}; // validate everything but the files + probe.validate(true); + clients::body::MultipartBody form; + form.add_text("model", request.model); + form.add_text("prompt", request.prompt); + if (request.n) form.add_text("n", std::to_string(*request.n)); + if (!request.size.empty()) form.add_text("size", request.size); + if (!request.response_format.empty()) form.add_text("response_format", request.response_format); + if (!request.reasoning_strength.empty()) form.add_text("reasoning_strength", request.reasoning_strength); + std::size_t i = 0; + for (const auto& [mime, bytes] : files) { + Check::one_of("image mime", mime, {"image/png", "image/jpeg", "image/webp", "image/gif"}); + if (bytes.empty()) Check::fail("image file is empty"); + form.add_file("image[" + std::to_string(i) + "]", "image" + std::to_string(i), mime, bytes); + ++i; + } + return ImageResult::from_json(client_->post_multipart(path_for(Call::kImagesEdit), form)); +} + +void ImagesService::stream(const ImageRequest& request, bool edit, const EventSink& sink) { + Json body = request.to_json(edit, true); + std::size_t seen = 0; + client_->stream(path_for(edit ? Call::kImagesEdit : Call::kImagesGenerate), body, + [&](const llm::SseEvent& e) { + if (++seen > MetaCaps::kMaxStreamEvents) return false; + std::string type = JsonX::str(e.data, "type", e.event); + return sink ? sink(type, e.data) : true; + }); +} + +} // namespace molo::meta diff --git a/src/meta/images.h b/src/meta/images.h new file mode 100644 index 0000000..382d095 --- /dev/null +++ b/src/meta/images.h @@ -0,0 +1,76 @@ +#pragma once + +// Muse Image over the single-shot endpoints: POST /images/generations and +// POST /images/edits (JSON body with an `images` array, or multipart with +// image / image[N] parts). $0.01 per generated image, so the demo treats both +// as paid. Rules (docs: image-generation): n 1..10, aspect ratio at most 16:1, +// moderation is not enabled on muse-image-1.0 (any value is a 400), mask / +// input_fidelity / partial_images are rejected, built-in search is on unless +// tool_enablement turns a tool off. A `web_search` tool passed to Muse Image +// is an error; it does its own searching. + +#include +#include +#include +#include + +#include "llm/api_client.h" +#include "meta/jsonx.h" + +namespace molo::meta { + +struct ImageRequest { + std::string model = "muse-image-1.0"; + std::string prompt; + std::optional n; // 1..10 + std::string size; // "" | auto | WxH + std::string response_format; // b64_json (default) | url + std::string output_format; // e.g. webp + std::string reasoning_strength; // low | high (default high) + std::string user; + std::optional enable_image_search; + std::optional enable_web_search; + std::optional enable_shell; + std::vector images; // edits: {image_url} or {file_id} + Json extra; + + void validate(bool edit) const; + Json to_json(bool edit, bool stream = false) const; + static Json image_url(const std::string& url); + static Json image_file(const std::string& file_id); +}; + +struct GeneratedImage { + std::string b64_json; + std::string url; + std::string revised_prompt; +}; + +struct ImageResult { + long created = 0; + std::vector data; + std::string output_format; + Json usage; + static ImageResult from_json(const Json& j); +}; + +// Parses "WxH" and checks the 16:1 aspect limit; "auto" and "" pass. +bool valid_image_size(const std::string& size); + +class ImagesService : public llm::Service { + public: + using EventSink = std::function; + + explicit ImagesService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + + ImageResult generate(const ImageRequest& request); + ImageResult edit(const ImageRequest& request); + // Multipart edit (what the OpenAI SDK sends): one `image[N]` part per file. + ImageResult edit_multipart(const ImageRequest& request, + const std::vector>& files_mime_bytes); + // Streams image_generation.completed / image_edit.completed events. + void stream(const ImageRequest& request, bool edit, const EventSink& sink); +}; + +} // namespace molo::meta diff --git a/src/meta/jsonx.h b/src/meta/jsonx.h new file mode 100644 index 0000000..9470319 --- /dev/null +++ b/src/meta/jsonx.h @@ -0,0 +1,72 @@ +#pragma once + +// Tolerant JSON readers and small builders used by every parser and request +// builder in this module. Missing or mistyped members read as the fallback; +// nothing here throws. + +#include +#include + +#include "clients/body/json.h" + +namespace molo::meta { + +using Json = clients::body::Json; + +struct JsonX { + static const Json* member(const Json& doc, const std::string& key) { + return doc.is_object() ? doc.find(key) : nullptr; + } + static std::string str(const Json& doc, const std::string& key, const std::string& fb = "") { + const Json* v = member(doc, key); + return v && v->is_string() ? v->as_string() : fb; + } + static long num(const Json& doc, const std::string& key, long fb = 0) { + const Json* v = member(doc, key); + return v && v->is_number() ? v->as_int() : fb; + } + static double real(const Json& doc, const std::string& key, double fb = 0) { + const Json* v = member(doc, key); + return v && v->is_number() ? v->as_number() : fb; + } + static bool flag(const Json& doc, const std::string& key, bool fb = false) { + const Json* v = member(doc, key); + return v && v->is_bool() ? v->as_bool() : fb; + } + static const Json* obj(const Json& doc, const std::string& key) { + const Json* v = member(doc, key); + return v && v->is_object() ? v : nullptr; + } + static const Json* arr(const Json& doc, const std::string& key) { + const Json* v = member(doc, key); + return v && v->is_array() ? v : nullptr; + } + // Dotted path read: "usage.input_tokens_details.cached_tokens". + static long path_num(const Json& doc, const std::string& path, long fb = 0) { + const Json* cur = &doc; + std::size_t start = 0; + while (true) { + std::size_t dot = path.find('.', start); + std::string part = path.substr(start, dot == std::string::npos ? dot : dot - start); + cur = member(*cur, part); + if (!cur) return fb; + if (dot == std::string::npos) break; + start = dot + 1; + } + return cur->is_number() ? cur->as_int() : fb; + } + static Json strings(const std::vector& v) { + Json a = Json::array(); + for (const std::string& s : v) a.push(Json(s)); + return a; + } + static void set_if(Json& obj, const char* key, const std::string& value) { + if (!value.empty()) obj.set(key, Json(value)); + } + static void merge(Json& into, const Json& extra) { + if (!extra.is_object()) return; + for (const auto& [k, v] : extra.as_object()) into.set(k, v); + } +}; + +} // namespace molo::meta diff --git a/src/meta/messages.cpp b/src/meta/messages.cpp new file mode 100644 index 0000000..c7c913c --- /dev/null +++ b/src/meta/messages.cpp @@ -0,0 +1,291 @@ +#include "meta/messages.h" + +#include "meta/calls.h" +#include "meta/caps.h" +#include "meta/check.h" +#include "meta/schema_check.h" +#include "security/crypto/Base64.h" + +namespace molo::meta { + +namespace { +Json block(const char* type) { + Json o = Json::object(); + o.set("type", Json(type)); + return o; +} +Json source(const char* type) { + Json s = Json::object(); + s.set("type", Json(type)); + return s; +} +} // namespace + +Json AnthropicBlock::text(const std::string& t) { + Json b = block("text"); + b.set("text", Json(t)); + return b; +} + +Json AnthropicBlock::image_url(const std::string& url) { + Check::non_empty("image url", url); + Json s = source("url"); + s.set("url", Json(url)); + Json b = block("image"); + b.set("source", s); + return b; +} + +Json AnthropicBlock::image_base64(const std::string& media_type, const std::string& bytes) { + Check::one_of("media_type", media_type, + {"image/png", "image/jpeg", "image/gif", "image/webp", "image/x-icon"}); + if (bytes.empty() || bytes.size() > MetaCaps::kMaxInlineBytes) Check::fail("image size is out of range"); + Json s = source("base64"); + s.set("media_type", Json(media_type)); + s.set("data", Json(security::crypto::Base64::encode(bytes))); + Json b = block("image"); + b.set("source", s); + return b; +} + +Json AnthropicBlock::image_file(const std::string& file_id) { + Check::segment("file_id", file_id); + Json s = source("file"); + s.set("file_id", Json(file_id)); + Json b = block("image"); + b.set("source", s); + return b; +} + +Json AnthropicBlock::video_url(const std::string& url, double fps) { + Check::non_empty("video url", url); + Json s = source("url"); + s.set("url", Json(url)); + Json b = block("video"); + b.set("source", s); + if (fps > 0) b.set("fps", Json(fps)); + return b; +} + +Json AnthropicBlock::document_url(const std::string& url) { + Check::non_empty("document url", url); + Json s = source("url"); + s.set("url", Json(url)); + Json b = block("document"); + b.set("source", s); + return b; +} + +Json AnthropicBlock::document_text(const std::string& t) { + Check::non_empty("document text", t); + Json s = source("text"); + s.set("media_type", Json("text/plain")); + s.set("data", Json(t)); + Json b = block("document"); + b.set("source", s); + return b; +} + +Json AnthropicBlock::tool_use(const std::string& id, const std::string& name, const Json& input) { + Check::call_id(id); + Check::function_name(name); + Json b = block("tool_use"); + b.set("id", Json(id)); + b.set("name", Json(name)); + b.set("input", input.is_object() ? input : Json::object()); + return b; +} + +Json AnthropicBlock::tool_result(const std::string& id, const std::string& content, bool is_error) { + Check::call_id(id); + Json b = block("tool_result"); + b.set("tool_use_id", Json(id)); + b.set("content", Json(content)); + if (is_error) b.set("is_error", Json(true)); + return b; +} + +Json AnthropicBlock::redacted_thinking(const std::string& data) { + Check::non_empty("data", data); + Json b = block("redacted_thinking"); + b.set("data", Json(data)); + return b; +} + +Json AnthropicBlock::message(const std::string& role, const std::vector& blocks) { + Check::one_of("role", role, {"user", "assistant"}); + Json a = Json::array(); + for (const Json& b : blocks) a.push(b); + Json m = Json::object(); + m.set("role", Json(role)); + m.set("content", a); + return m; +} + +Json AnthropicBlock::message_text(const std::string& role, const std::string& t) { + Check::one_of("role", role, {"user", "assistant"}); + Json m = Json::object(); + m.set("role", Json(role)); + m.set("content", Json(t)); + return m; +} + +Json AnthropicBlock::custom_tool(const std::string& name, const std::string& description, + const Json& schema, bool strict, bool defer_loading) { + Check::function_name(name); + if (schema.is_object()) SchemaCheck::check(schema, strict); + Json t = Json::object(); + t.set("name", Json(name)); + JsonX::set_if(t, "description", description); + t.set("input_schema", schema.is_object() ? schema : Json::object()); + if (strict) t.set("strict", Json(true)); + if (defer_loading) t.set("defer_loading", Json(true)); + return t; +} + +Json AnthropicBlock::web_search_tool(const Json& user_location) { + Json t = block("web_search"); + t.set("name", Json("web_search")); + if (user_location.is_object()) t.set("user_location", user_location); + return t; +} + +Json AnthropicBlock::tool_search_tool() { + Json t = block("tool_search"); + t.set("name", Json("tool_search")); + return t; +} + +void MessageRequest::validate() const { + Check::model_id(model); + if (messages.empty() || messages.size() > MetaCaps::kMaxInputItems) + Check::fail("messages needs 1.." + std::to_string(MetaCaps::kMaxInputItems) + " entries"); + for (const Json& m : messages) + Check::one_of("message role", JsonX::str(m, "role"), {"user", "assistant", "system", "developer"}); + if (max_tokens < 1) Check::fail("max_tokens is required"); + Check::max_bytes("system", system, MetaCaps::kMaxPromptBytes); + if (tools.size() > MetaCaps::kMaxTools) Check::fail("too many tools"); + if (!tool_choice.empty()) Check::one_of("tool_choice", tool_choice, {"auto", "any", "none"}); + if (!thinking.empty()) { + Check::one_of("thinking", thinking, {"adaptive", "enabled"}); // disabled is rejected + if (thinking == "enabled" && (thinking_budget < 1024 || thinking_budget >= max_tokens)) + Check::fail("thinking budget_tokens must be >= 1024 and < max_tokens"); + } + if (!thinking_display.empty()) + Check::one_of("thinking.display", thinking_display, {"summarized", "omitted"}); + if (!effort.empty()) Check::one_of("output_config.effort", effort, {"low", "medium", "high", "xhigh"}); + if (output_schema.is_object()) SchemaCheck::check(output_schema, false); + if (temperature) Check::range("temperature", *temperature, 0.0, 1.0); + if (top_p) Check::top_p(*top_p); + if (!service_tier.empty()) Check::one_of("service_tier", service_tier, {"auto", "standard_only"}); + if (user_id.size() > 4096) Check::fail("metadata.user_id is too long"); + if (extra.is_object()) + for (const char* banned : {"stop_sequences", "top_k", "container", "inference_geo"}) + if (extra.has(banned)) Check::fail(std::string(banned) + " is rejected by the messages endpoint"); +} + +namespace { +void common(const MessageRequest& r, Json& b) { + b.set("model", Json(r.model)); + Json m = Json::array(); + for (const Json& x : r.messages) m.push(x); + b.set("messages", m); + if (!r.system.empty()) b.set("system", Json(r.system)); + if (!r.tools.empty()) { + Json t = Json::array(); + for (const Json& x : r.tools) t.push(x); + b.set("tools", t); + } + if (!r.tool_choice.empty() || r.disable_parallel_tool_use) { + Json c = Json::object(); + c.set("type", Json(r.tool_choice.empty() ? "auto" : r.tool_choice)); + if (r.disable_parallel_tool_use) c.set("disable_parallel_tool_use", Json(true)); + b.set("tool_choice", c); + } + if (!r.thinking.empty()) { + Json t = Json::object(); + t.set("type", Json(r.thinking)); + if (r.thinking == "enabled") t.set("budget_tokens", Json(r.thinking_budget)); + JsonX::set_if(t, "display", r.thinking_display); + b.set("thinking", t); + } + if (!r.effort.empty() || r.output_schema.is_object()) { + Json o = Json::object(); + JsonX::set_if(o, "effort", r.effort); + if (r.output_schema.is_object()) { + Json f = Json::object(); + f.set("type", Json("json_schema")); + f.set("schema", r.output_schema); + o.set("format", f); + } + b.set("output_config", o); + } + if (r.temperature) b.set("temperature", Json(*r.temperature)); + if (r.top_p) b.set("top_p", Json(*r.top_p)); + if (!r.user_id.empty()) { + Json md = Json::object(); + md.set("user_id", Json(r.user_id)); + b.set("metadata", md); + } +} +} // namespace + +Json MessageRequest::to_json(bool stream) const { + validate(); + Json b = Json::object(); + common(*this, b); + b.set("max_tokens", Json(max_tokens)); + JsonX::set_if(b, "service_tier", service_tier); + if (stream) b.set("stream", Json(true)); + JsonX::merge(b, extra); + return b; +} + +Json MessageRequest::to_count_json() const { + validate(); + Json b = Json::object(); + common(*this, b); + JsonX::merge(b, extra); + return b; +} + +Json MessagesService::create(const MessageRequest& request) { + Json body = request.to_json(false); + return client_->post(path_for(Call::kMessagesCreate), body); +} + +std::string MessagesService::stream(const MessageRequest& request, const EventSink& sink) { + Json body = request.to_json(true); + std::string text; + std::size_t seen = 0; + client_->stream(path_for(Call::kMessagesCreate), body, [&](const llm::SseEvent& e) { + if (++seen > MetaCaps::kMaxStreamEvents) return false; + std::string type = JsonX::str(e.data, "type", e.event); + if (type == "content_block_delta") + if (const Json* d = JsonX::obj(e.data, "delta")) + if (JsonX::str(*d, "type") == "text_delta") text += JsonX::str(*d, "text"); + return sink ? sink(type, e.data) : true; + }); + return text; +} + +long MessagesService::count_tokens(const MessageRequest& request) { + Json body = request.to_count_json(); + Json out = client_->post(path_for(Call::kMessagesCountTokens), body); + return JsonX::num(out, "input_tokens", -1); +} + +std::string MessagesService::text_of(const Json& message) { + std::string out; + if (const Json* c = JsonX::arr(message, "content")) + for (const Json& b : c->as_array()) + if (JsonX::str(b, "type") == "text") out += JsonX::str(b, "text"); + return out; +} + +MetaUsage MessagesService::usage_of(const Json& message) { + const Json* u = JsonX::obj(message, "usage"); + return u ? MetaUsage::from_messages(*u) : MetaUsage(); +} + +} // namespace molo::meta diff --git a/src/meta/messages.h b/src/meta/messages.h new file mode 100644 index 0000000..f6394ab --- /dev/null +++ b/src/meta/messages.h @@ -0,0 +1,86 @@ +#pragma once + +// POST /messages and POST /messages/count_tokens: the Anthropic Messages +// compatible endpoint (base URL https://api.meta.ai, the SDK appends /v1). +// The adapter is stateless and runs storage off. Not supported, per docs +// (protocols/messages): stop_sequences, top_k, container, inference_geo, +// thinking disabled, a named tool_choice, web_search allowed_domains / +// blocked_domains / max_uses, system blocks other than text. Anthropic stream +// events arrive as named SSE events (message_start, content_block_delta, ...). + +#include +#include +#include +#include + +#include "llm/api_client.h" +#include "meta/jsonx.h" +#include "meta/usage.h" + +namespace molo::meta { + +class AnthropicBlock { + public: + static Json text(const std::string& text); + static Json image_url(const std::string& url); + static Json image_base64(const std::string& media_type, const std::string& bytes); + static Json image_file(const std::string& file_id); + static Json video_url(const std::string& url, double fps = 0); + static Json document_url(const std::string& url); + static Json document_text(const std::string& text); + static Json tool_use(const std::string& id, const std::string& name, const Json& input); + static Json tool_result(const std::string& tool_use_id, const std::string& content, + bool is_error = false); + static Json redacted_thinking(const std::string& data); // replay encrypted reasoning verbatim + static Json message(const std::string& role, const std::vector& blocks); + static Json message_text(const std::string& role, const std::string& text); + // Tool definitions. + static Json custom_tool(const std::string& name, const std::string& description, + const Json& input_schema, bool strict = false, bool defer_loading = false); + static Json web_search_tool(const Json& user_location = Json()); + static Json tool_search_tool(); +}; + +struct MessageRequest { + std::string model; + std::vector messages; + int max_tokens = 0; // required + std::string system; + std::vector tools; + std::string tool_choice; // "", auto, any, none + bool disable_parallel_tool_use = false; + std::string thinking; // "", adaptive, enabled + int thinking_budget = 0; // enabled: >= 1024 and < max_tokens + std::string thinking_display; // summarized | omitted + std::string effort; // output_config.effort: low medium high xhigh + Json output_schema; // output_config.format = {type: json_schema, schema} + std::optional temperature; // 0..1 + std::optional top_p; + std::string service_tier; // auto | standard_only + std::string user_id; // metadata.user_id + Json extra; + + void validate() const; + Json to_json(bool stream = false) const; + Json to_count_json() const; +}; + +class MessagesService : public llm::Service { + public: + using EventSink = std::function; + + explicit MessagesService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + + Json create(const MessageRequest& request); + // Folds the Anthropic stream; returns the visible text. The sink sees every event. + std::string stream(const MessageRequest& request, const EventSink& sink = nullptr); + long count_tokens(const MessageRequest& request); + + // Visible text of an AnthropicMessage: the `text` blocks (content[0] is a + // redacted_thinking block, never index it). + static std::string text_of(const Json& message); + static MetaUsage usage_of(const Json& message); +}; + +} // namespace molo::meta diff --git a/src/meta/models.cpp b/src/meta/models.cpp new file mode 100644 index 0000000..8fd7076 --- /dev/null +++ b/src/meta/models.cpp @@ -0,0 +1,44 @@ +#include "meta/models.h" + +#include "meta/calls.h" +#include "meta/check.h" + +namespace molo::meta { + +ModelEntry ModelEntry::from_json(const Json& j) { + ModelEntry m; + m.id = JsonX::str(j, "id"); + m.created = JsonX::num(j, "created"); + m.owned_by = JsonX::str(j, "owned_by"); + if (const Json* md = JsonX::obj(j, "metadata")) m.metadata = *md; + return m; +} + +ServiceStatus ServiceStatus::from_json(const Json& j) { + ServiceStatus s; + s.is_alive = JsonX::flag(j, "is_alive"); + s.service_status = JsonX::str(j, "service_status"); + s.service_message = JsonX::str(j, "service_message"); + s.updated_at = JsonX::str(j, "updated_at"); + if (const Json* m = JsonX::arr(j, "model_statuses")) s.model_statuses = *m; + return s; +} + +std::vector ModelsService::list() { + Json doc = client_->get(path_for(Call::kModelsList)); + std::vector out; + if (const Json* d = JsonX::arr(doc, "data")) + for (const Json& m : d->as_array()) out.push_back(ModelEntry::from_json(m)); + return out; +} + +ModelEntry ModelsService::retrieve(const std::string& model_id) { + Check::model_id(model_id); + return ModelEntry::from_json(client_->get(path_for(Call::kModelsRetrieve, model_id))); +} + +ServiceStatus StatusService::get() { + return ServiceStatus::from_json(client_->get(path_for(Call::kStatus))); +} + +} // namespace molo::meta diff --git a/src/meta/models.h b/src/meta/models.h new file mode 100644 index 0000000..8bc036f --- /dev/null +++ b/src/meta/models.h @@ -0,0 +1,50 @@ +#pragma once + +// GET /models, GET /models/{model}, GET /status. +// The models list returns the API-hosted models newest-first (created, then +// id); open-weight models (Muse Spark Open, Muse Glimmer) are never in it. +// /status is the one unauthenticated endpoint: overall health plus +// per-model availability. + +#include +#include + +#include "llm/api_client.h" +#include "meta/jsonx.h" + +namespace molo::meta { + +struct ModelEntry { + std::string id; + long created = 0; + std::string owned_by; + Json metadata; + static ModelEntry from_json(const Json& j); +}; + +struct ServiceStatus { + bool is_alive = false; + std::string service_status; // e.g. "operational" + std::string service_message; + std::string updated_at; + Json model_statuses; // array, shape not in the schema pages (see SPEC Unverified) + static ServiceStatus from_json(const Json& j); +}; + +class ModelsService : public llm::Service { + public: + explicit ModelsService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + std::vector list(); + ModelEntry retrieve(const std::string& model_id); +}; + +class StatusService : public llm::Service { + public: + // Build the client with MetaClient::unauthenticated(). + explicit StatusService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + ServiceStatus get(); +}; + +} // namespace molo::meta diff --git a/src/meta/muse_code.cpp b/src/meta/muse_code.cpp new file mode 100644 index 0000000..f21c527 --- /dev/null +++ b/src/meta/muse_code.cpp @@ -0,0 +1,353 @@ +#include "meta/muse_code.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "meta/caps.h" +#include "meta/check.h" +#include "meta/muse_code_spec.h" + +namespace molo::meta { + +namespace { +bool in_list(const char* const* list, std::size_t n, const std::string& s) { + for (std::size_t i = 0; i < n; ++i) + if (s == list[i]) return true; + return false; +} +long steady_ms() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} +} // namespace + +void LineSplitter::feed(const std::string& bytes) { + for (char c : bytes) { + if (c == '\n') { + if (!buf_.empty() && buf_.back() == '\r') buf_.pop_back(); + lines_.push_back(std::move(buf_)); + buf_.clear(); + if (lines_.size() > MetaCaps::kMspMaxQueued) Check::fail("too many unread MSP frames"); + } else { + buf_.push_back(c); + if (buf_.size() > limit_) Check::fail("MSP frame is over the frame limit"); + } + } +} + +std::optional LineSplitter::pop() { + if (lines_.empty()) return std::nullopt; + std::string l = std::move(lines_.front()); + lines_.pop_front(); + return l; +} + +ProcessTransport::ProcessTransport(std::vector argv) + : splitter_(MetaCaps::kMspFrameLimit) { + if (argv.empty() || argv.size() > 16) Check::fail("bad argv"); + int to_child[2], from_child[2]; + if (pipe(to_child) != 0) Check::fail("pipe failed"); + if (pipe(from_child) != 0) { ::close(to_child[0]); ::close(to_child[1]); Check::fail("pipe failed"); } + pid_t pid = fork(); + if (pid < 0) Check::fail("fork failed"); + if (pid == 0) { + dup2(to_child[0], 0); + dup2(from_child[1], 1); + ::close(to_child[0]); ::close(to_child[1]); ::close(from_child[0]); ::close(from_child[1]); + std::vector args; + for (std::string& a : argv) args.push_back(a.data()); + args.push_back(nullptr); + execvp(args[0], args.data()); + _exit(127); + } + ::close(to_child[0]); + ::close(from_child[1]); + in_fd_ = to_child[1]; + out_fd_ = from_child[0]; + pid_ = pid; + signal(SIGPIPE, SIG_IGN); +} + +ProcessTransport::~ProcessTransport() { close(); } + +bool ProcessTransport::binary_available(const std::string& name) { + const char* path = getenv("PATH"); + std::string p = path ? path : ""; + std::size_t start = 0; + while (start <= p.size()) { + std::size_t colon = p.find(':', start); + std::string dir = p.substr(start, colon == std::string::npos ? colon : colon - start); + if (!dir.empty() && access((dir + "/" + name).c_str(), X_OK) == 0) return true; + if (colon == std::string::npos) break; + start = colon + 1; + } + return false; +} + +void ProcessTransport::write_line(const std::string& line) { + if (in_fd_ < 0) throw llm::ApiError("meta", 502, "UNAVAILABLE", "muse process is closed"); + if (line.size() > MetaCaps::kMspFrameLimit) Check::fail("MSP frame is over the frame limit"); + std::string data = line + "\n"; + std::size_t off = 0; + while (off < data.size()) { + ssize_t n = write(in_fd_, data.data() + off, data.size() - off); + if (n <= 0) throw llm::ApiError("meta", 502, "UNAVAILABLE", "write to muse failed"); + off += static_cast(n); + } +} + +std::optional ProcessTransport::read_line(int timeout_ms) { + long deadline = steady_ms() + timeout_ms; + while (true) { + if (auto l = splitter_.pop()) return l; + if (eof_ || out_fd_ < 0) throw llm::ApiError("meta", 502, "UNAVAILABLE", "muse process ended"); + long left = deadline - steady_ms(); + if (left < 0) return std::nullopt; + pollfd pfd{out_fd_, POLLIN, 0}; + int r = poll(&pfd, 1, static_cast(left)); + if (r == 0) return std::nullopt; + if (r < 0) continue; + char buf[65536]; + ssize_t n = read(out_fd_, buf, sizeof buf); + if (n <= 0) { eof_ = true; continue; } + splitter_.feed(std::string(buf, static_cast(n))); + } +} + +void ProcessTransport::close() { + if (in_fd_ >= 0) { ::close(in_fd_); in_fd_ = -1; } + if (pid_ > 0) { + for (int i = 0; i < 20; ++i) { + if (waitpid(static_cast(pid_), nullptr, WNOHANG) != 0) { pid_ = -1; break; } + usleep(50000); + } + if (pid_ > 0) { + kill(static_cast(pid_), SIGKILL); + waitpid(static_cast(pid_), nullptr, 0); + pid_ = -1; + } + } + if (out_fd_ >= 0) { ::close(out_fd_); out_fd_ = -1; } +} + +std::string MspFrames::request(long id, const std::string& method, const Json& params) { + Json j = Json::object(); + j.set("jsonrpc", Json("2.0")); + j.set("id", Json(id)); + j.set("method", Json(method)); + if (params.is_object()) j.set("params", params); + return j.dump(); +} + +std::string MspFrames::notification(const std::string& method, const Json& params) { + Json j = Json::object(); + j.set("jsonrpc", Json("2.0")); + j.set("method", Json(method)); + if (params.is_object()) j.set("params", params); + return j.dump(); +} + +std::string MspFrames::result(const Json& id, const Json& res) { + Json j = Json::object(); + j.set("jsonrpc", Json("2.0")); + j.set("id", id); + j.set("result", res); + return j.dump(); +} + +Json MspFrames::parse(const std::string& line) { + try { + Json j = Json::parse(line); + if (!j.is_object() || JsonX::str(j, "jsonrpc") != "2.0") + throw llm::ApiError("meta", 502, "BAD_RESPONSE", "frame is not JSON-RPC 2.0"); + return j; + } catch (const clients::body::JsonError& e) { + throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what()); + } +} + +bool MspFrames::is_known_method(const std::string& m) { + std::size_t n; + const char* const* l = MuseCodeSpec::methods(n); + return in_list(l, n, m); +} + +bool MspFrames::is_known_notification(const std::string& m) { + std::size_t n; + const char* const* l = MuseCodeSpec::notifications(n); + return in_list(l, n, m); +} + +std::string MspFrames::error_kind(long code) { + std::size_t n; + const char* const* l = MuseCodeSpec::errors(n); + std::string prefix = std::to_string(code) + ":"; + for (std::size_t i = 0; i < n; ++i) + if (std::strncmp(l[i], prefix.c_str(), prefix.size()) == 0) { + std::string s = l[i] + prefix.size(); + return s.substr(0, s.find(':')); + } + return ""; +} + +bool MspFrames::error_retryable(long code) { + std::size_t n; + const char* const* l = MuseCodeSpec::errors(n); + std::string prefix = std::to_string(code) + ":"; + for (std::size_t i = 0; i < n; ++i) + if (std::strncmp(l[i], prefix.c_str(), prefix.size()) == 0) { + std::string s = l[i]; + return s.back() == '1'; + } + return false; +} + +std::string MspFrames::new_id() { + static std::mt19937_64 rng{std::random_device{}()}; + unsigned long long ms = static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + unsigned long long a = rng(), b = rng(); + char buf[64]; + std::snprintf(buf, sizeof buf, "%08llx-%04llx-7%03llx-%04llx-%012llx", ms >> 16, ms & 0xFFFF, + a & 0xFFF, 0x8000 | (b & 0x3FFF), (rng() & 0xFFFFFFFFFFFFull)); + return buf; +} + +MspClient::MspClient(std::unique_ptr t) : transport_(std::move(t)) {} +MspClient::~MspClient() { close(); } + +void MspClient::close() { + if (transport_) transport_->close(); +} + +void MspClient::handle_frame(const Json& f) { + const Json* id = JsonX::member(f, "id"); + std::string method = JsonX::str(f, "method"); + if (!method.empty() && id) { // server -> client request + Json params = f.find("params") ? *f.find("params") : Json::object(); + Json res = handler_ ? handler_(method, params) : Json::object(); + transport_->write_line(MspFrames::result(*id, res)); + return; + } + if (!method.empty()) { + if (queue_.size() >= MetaCaps::kMspMaxQueued) { + ++dropped_; // loud: counted, surfaced through dropped() + return; + } + MspNotification n; + n.method = method; + if (const Json* p = f.find("params")) n.params = *p; + queue_.push_back(std::move(n)); + return; + } + if (id && id->is_number()) responses_[id->as_int()] = f; +} + +Json MspClient::request(const std::string& method, const Json& params, int timeout_ms) { + if (!MspFrames::is_known_method(method)) Check::fail("unknown MSP method " + method); + if (method != "initialize" && !initialized_) Check::fail("call initialize first"); + long id = next_id_++; + transport_->write_line(MspFrames::request(id, method, params)); + long deadline = steady_ms() + timeout_ms; + while (true) { + auto it = responses_.find(id); + if (it != responses_.end()) { + Json f = it->second; + responses_.erase(it); + if (const Json* e = JsonX::obj(f, "error")) { + long code = JsonX::num(*e, "code"); + std::string kind = MspFrames::error_kind(code); + throw llm::ApiError("meta", MspFrames::error_retryable(code) ? 503 : 400, + kind.empty() ? std::to_string(code) : kind, JsonX::str(*e, "message")); + } + const Json* r = f.find("result"); + return r ? *r : Json::object(); + } + long left = deadline - steady_ms(); + if (left <= 0) throw llm::ApiError("meta", 504, "TIMEOUT", method + " timed out"); + auto line = transport_->read_line(static_cast(left)); + if (line) handle_frame(MspFrames::parse(*line)); + } +} + +Json MspClient::initialize(const std::string& name, const std::string& version, + const std::vector& caps) { + Check::non_empty("client name", name); + Check::non_empty("client version", version); + Json info = Json::object(); + info.set("name", Json(name)); + info.set("version", Json(version)); + Json params = Json::object(); + params.set("clientInfo", info); + if (!caps.empty()) { + Json c = Json::object(); + c.set("requestedCapabilities", JsonX::strings(caps)); + params.set("capabilities", c); + } + Json res = request("initialize", params, MetaCaps::kMspSpawnTimeoutMs); + initialized_ = true; + if (const Json* s = JsonX::obj(res, "schema")) fingerprint_ = JsonX::str(*s, "fingerprint"); + transport_->write_line(MspFrames::notification("initialized")); + return res; +} + +std::optional MspClient::next_notification(int timeout_ms) { + long deadline = steady_ms() + timeout_ms; + while (true) { + if (!queue_.empty()) { + MspNotification n = std::move(queue_.front()); + queue_.pop_front(); + return n; + } + long left = deadline - steady_ms(); + if (left <= 0) return std::nullopt; + auto line = transport_->read_line(static_cast(left)); + if (line) handle_frame(MspFrames::parse(*line)); + } +} + +Json MspClient::session_start(const std::string& workspace_root, const std::string& model_id) { + Check::non_empty("workspaceRoot", workspace_root); + Json p = Json::object(); + p.set("commandId", Json(MspFrames::new_id())); + p.set("workspaceRoot", Json(workspace_root)); + JsonX::set_if(p, "modelId", model_id); + return request("session/start", p); +} + +Json MspClient::turn_start(const std::string& session_id, const std::string& text) { + Check::segment("sessionId", session_id); + Check::non_empty("text", text); + Json part = Json::object(); + part.set("type", Json("text")); + part.set("text", Json(text)); + Json input = Json::array(); + input.push(part); + Json p = Json::object(); + p.set("commandId", Json(MspFrames::new_id())); + p.set("sessionId", Json(session_id)); + p.set("input", input); + return request("turn/start", p); +} + +Json MspClient::turn_cancel(const std::string& session_id, const std::string& turn_id) { + Check::segment("sessionId", session_id); + Json p = Json::object(); + p.set("commandId", Json(MspFrames::new_id())); + p.set("sessionId", Json(session_id)); + JsonX::set_if(p, "turnId", turn_id); + return request("turn/cancel", p); +} + +Json MspClient::model_list() { return request("model/list", Json::object()); } + +} // namespace molo::meta diff --git a/src/meta/muse_code.h b/src/meta/muse_code.h new file mode 100644 index 0000000..b445150 --- /dev/null +++ b/src/meta/muse_code.h @@ -0,0 +1,127 @@ +#pragma once + +// Muse Session Protocol (MSP) client: drive a Muse Code agent session from +// C++. Transport is newline-delimited JSON-RPC 2.0 on the stdio of +// `muse serve` (frame limit 10 MiB, client lines may end in \r). Handshake: +// initialize -> result{serverInfo, schema{version,fingerprint}, grantedCapabilities, +// ...} -> notification `initialized`. Server push (session/*, turn/*, item/*, +// view/*) arrives as notifications and is queued with a hard cap; the two +// server requests (approval/request, userInput/request) are answered with an +// empty receipt by the default handler (the decision travels separately as +// approval/decide or userInput/answer). +// The transport is an interface so tests run the whole client against a +// loopback process; ProcessTransport spawns the real binary. + +#include +#include +#include +#include +#include +#include +#include + +#include "llm/error.h" +#include "meta/jsonx.h" + +namespace molo::meta { + +class ILineTransport { + public: + virtual ~ILineTransport() = default; + virtual void write_line(const std::string& line) = 0; + // One line without the newline; nullopt on timeout; throws on EOF. + virtual std::optional read_line(int timeout_ms) = 0; + virtual void close() = 0; +}; + +// Splits a byte stream into lines, enforcing the frame limit. +class LineSplitter { + public: + explicit LineSplitter(std::size_t limit = 10485760) : limit_(limit) {} + // Appends bytes; throws ApiError::invalid when a line exceeds the limit. + void feed(const std::string& bytes); + std::optional pop(); // strips a trailing \r + private: + std::size_t limit_; + std::string buf_; + std::deque lines_; +}; + +// Spawns argv (default {"muse","serve"}) with stdin/stdout pipes. +class ProcessTransport : public ILineTransport { + public: + explicit ProcessTransport(std::vector argv = {"muse", "serve"}); + ~ProcessTransport() override; + void write_line(const std::string& line) override; + std::optional read_line(int timeout_ms) override; + void close() override; + static bool binary_available(const std::string& name = "muse"); + + private: + int in_fd_ = -1, out_fd_ = -1; + long pid_ = -1; + LineSplitter splitter_; + bool eof_ = false; +}; + +struct MspFrames { + static std::string request(long id, const std::string& method, const Json& params); + static std::string notification(const std::string& method, const Json& params = Json()); + static std::string result(const Json& id, const Json& result); + // Throws ApiError on non-JSON or a missing jsonrpc "2.0". + static Json parse(const std::string& line); + static bool is_known_method(const std::string& method); + static bool is_known_notification(const std::string& method); + // kind name for a JSON-RPC error code ("" when not in the schema). + static std::string error_kind(long code); + static bool error_retryable(long code); + // UUID-shaped, time-ordered id for commandId / sessionId. + static std::string new_id(); +}; + +struct MspNotification { + std::string method; + Json params; +}; + +class MspClient { + public: + using ServerRequestHandler = std::function; + + explicit MspClient(std::unique_ptr transport); + ~MspClient(); + + // Handshake. Throws on an error reply. Records schema fingerprint. + Json initialize(const std::string& client_name, const std::string& client_version, + const std::vector& requested_capabilities = {}); + // Generic request; `method` must be in the schema's method table. Throws + // llm::ApiError(type = MSP error kind) on a JSON-RPC error. + Json request(const std::string& method, const Json& params, int timeout_ms = 120000); + // Pumps incoming frames for up to timeout_ms; returns the next notification. + std::optional next_notification(int timeout_ms); + std::size_t queued() const { return queue_.size(); } + std::size_t dropped() const { return dropped_; } + const std::string& schema_fingerprint() const { return fingerprint_; } + bool initialized() const { return initialized_; } + void set_server_request_handler(ServerRequestHandler h) { handler_ = std::move(h); } + void close(); + + // Typed shortcuts over request(). + Json session_start(const std::string& workspace_root, const std::string& model_id = ""); + Json turn_start(const std::string& session_id, const std::string& text); + Json turn_cancel(const std::string& session_id, const std::string& turn_id = ""); + Json model_list(); + + private: + void handle_frame(const Json& frame); + std::unique_ptr transport_; + std::deque queue_; + std::size_t dropped_ = 0; + long next_id_ = 1; + bool initialized_ = false; + std::string fingerprint_; + ServerRequestHandler handler_; + std::map responses_; +}; + +} // namespace molo::meta diff --git a/src/meta/muse_code_spec.cpp b/src/meta/muse_code_spec.cpp new file mode 100644 index 0000000..48e2969 --- /dev/null +++ b/src/meta/muse_code_spec.cpp @@ -0,0 +1,185 @@ +#include "meta/muse_code_spec.h" + +namespace molo::meta { + +// Generated from schema/msp/stable/msp.schema.json of the muse-code-sdk +// mirror (fingerprint sha256:cfdb6d2525601b9bac76584ccb0610ab3a5539b5e4f51141f079b99b39fcb4e0, +// schemaVersion 1). tools/check_spec.py re-fetches the schema and fails if any +// list below drifts. +// MSP-METHODS-BEGIN +static const char* const kMethods[] = { + "approval/decide", + "approval/listPending", + "computerUseSettings/read", + "computerUseSettings/update", + "computerUseSetup/check", + "computerUseSetup/read", + "computerUseSetup/start", + "feedback/submit", + "goal/clear", + "goal/edit", + "goal/pause", + "goal/resume", + "goal/set", + "hook/list", + "initialize", + "item/readOutput", + "media/upload", + "model/list", + "plugin/install", + "plugin/list", + "plugin/marketplaceAdd", + "plugin/marketplaceRemove", + "plugin/marketplaceUpdate", + "recap/cancel", + "recap/request", + "session/compact", + "session/fork", + "session/leave", + "session/list", + "session/read", + "session/rename", + "session/resume", + "session/setApprovalMode", + "session/setModel", + "session/setReasoningEffort", + "session/sideChat", + "session/start", + "session/userShell", + "skill/import", + "skill/list", + "skill/setActivation", + "subagent/close", + "subagent/followupTask", + "subagent/interrupt", + "subagent/readResult", + "subagent/reopen", + "subagent/resume", + "subagent/sendMessage", + "subagent/stop", + "task/background", + "task/list", + "task/stop", + "task/stopAll", + "turn/cancel", + "turn/interrupt", + "turn/start", + "turn/steer", + "turn/unqueue", + "usage/read", + "userInput/answer", + "userInput/cancel", + "userInput/clarify", + "userInput/interrupt", + "view/page", + "view/subscribe", + "view/unsubscribe", + "workflow/cancel", + "workflow/childControl", + "workflow/pause", + "workflow/resume", +}; +// MSP-METHODS-END +// MSP-NOTIFICATIONS-BEGIN +static const char* const kNotifications[] = { + "approval/requested", + "approval/resolved", + "approval/reviewing", + "approval/updated", + "computerUseSetup/changed", + "initialized", + "item/completed", + "item/delta", + "item/started", + "item/updated", + "monitor/inboxItemAdmitted", + "monitor/inboxItemDrained", + "monitor/inboxItemVoided", + "plugin/changed", + "plugin/marketplaceChanged", + "recap/completed", + "session/approvalModeChanged", + "session/branchChanged", + "session/closed", + "session/contextUsage", + "session/goalChanged", + "session/leaveCompleted", + "session/listChanged", + "session/modelChanged", + "session/modelRouteUnserved", + "session/nameChanged", + "session/reasoningEffortChanged", + "session/started", + "session/statusChanged", + "session/todoListChanged", + "session/tokenUsage", + "session/viewHealthChanged", + "skill/changed", + "turn/completed", + "turn/foregroundCompleted", + "turn/modelRetryScheduled", + "turn/retracted", + "turn/retryScheduled", + "turn/started", + "turn/unqueued", + "usage/changed", + "userInput/engaged", + "userInput/requested", + "userInput/settled", + "view/gap", +}; +// MSP-NOTIFICATIONS-END +// MSP-REQUESTS-BEGIN +static const char* const kRequests[] = { + "approval/request", + "userInput/request", +}; +// MSP-REQUESTS-END +// MSP-ERRORS-BEGIN (code:kind:retryable) +static const char* const kErrors[] = { + "-32700:parseError:0", + "-32600:invalidRequest:0", + "-32601:methodNotFound:0", + "-32602:invalidParams:0", + "-32603:internal:0", + "-32001:overloaded:1", + "-32002:inputTooLarge:0", + "-32010:capabilityRequired:0", + "-32011:notFound:0", + "-32013:interrupted:?", + "-32014:cancelled:?", + "-32020:sessionNotFound:0", + "-32021:sessionInUse:0", + "-32022:sessionAmbiguous:0", + "-32023:forkBoundaryInvalid:0", + "-32024:sessionNotLoaded:0", + "-32025:sessionStreamMismatch:0", + "-32030:commandRejected:0", + "-32031:backpressured:1", + "-32032:skillNotFound:0", + "-32033:computerUseUnavailable:0", + "-32034:marketplaceNotFound:0", + "-32035:pluginNotFound:0", + "-32036:marketplaceAlreadyExists:0", + "-32037:pluginMutationsDisabled:0", + "-32038:uploadNotFound:0", + "-32040:viewTruncated:0", + "-32041:outputUnavailable:0", + "-32042:boundaryPruned:0", + "-32050:approvalNotFound:0", + "-32051:approvalAlreadyResolved:0", + "-32052:approvalChoiceInvalid:0", + "-32053:approvalRequirementStale:0", + "-32054:approvalReviewerUnavailable:0", + "-32055:userInputNotFound:0", + "-32056:userInputAlreadySettled:0", + "-32057:userInputAnswerInvalid:0", +}; +// MSP-ERRORS-END + +const char* const* MuseCodeSpec::methods(std::size_t& n) { n = sizeof(kMethods) / sizeof(kMethods[0]); return kMethods; } +const char* const* MuseCodeSpec::notifications(std::size_t& n) { n = sizeof(kNotifications) / sizeof(kNotifications[0]); return kNotifications; } +const char* const* MuseCodeSpec::server_requests(std::size_t& n) { n = sizeof(kRequests) / sizeof(kRequests[0]); return kRequests; } +const char* const* MuseCodeSpec::errors(std::size_t& n) { n = sizeof(kErrors) / sizeof(kErrors[0]); return kErrors; } + +} // namespace molo::meta diff --git a/src/meta/muse_code_spec.h b/src/meta/muse_code_spec.h new file mode 100644 index 0000000..03a25eb --- /dev/null +++ b/src/meta/muse_code_spec.h @@ -0,0 +1,21 @@ +#pragma once + +// Muse Code (the Meta coding agent) has a public programmatic surface: the +// Muse Session Protocol (MSP), JSON-RPC 2.0 over newline-delimited JSON on the +// stdio of `muse serve` (docs: dev.meta.ai/docs/muse-code, SDK docs at +// meta-models.github.io/muse-code-sdk, schema in the muse-code-sdk repo under +// schema/msp/stable). It is NOT an HTTP API of api.meta.ai. This table is the +// protocol's method / notification / server-request / error vocabulary. + +#include + +namespace molo::meta { + +struct MuseCodeSpec { + static const char* const* methods(std::size_t& n); // client -> server requests + static const char* const* notifications(std::size_t& n); // server -> client (and `initialized`) + static const char* const* server_requests(std::size_t& n); // server -> client, must be answered + static const char* const* errors(std::size_t& n); // "code:kind:retryable(1|0|?)" +}; + +} // namespace molo::meta diff --git a/src/meta/response_events.cpp b/src/meta/response_events.cpp new file mode 100644 index 0000000..ffe7884 --- /dev/null +++ b/src/meta/response_events.cpp @@ -0,0 +1,154 @@ +#include "meta/response_events.h" + +#include + +namespace molo::meta { + +// EVENTS-BEGIN (parsed by tools/check_spec.py; one name per line) +static const std::vector kKnown = { + "response.created", + "response.in_progress", + "response.queued", + "response.completed", + "response.failed", + "response.incomplete", + "response.output_item.added", + "response.output_item.done", + "response.content_part.added", + "response.content_part.done", + "response.output_text.delta", + "response.output_text.done", + "response.output_text.annotation.added", + "response.refusal.delta", + "response.refusal.done", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.reasoning_summary_part.added", + "response.reasoning_summary_part.done", + "response.reasoning_summary_text.delta", + "response.reasoning_summary_text.done", + "response.reasoning_text.delta", + "response.reasoning_text.done", + "response.web_search_call.in_progress", + "response.web_search_call.searching", + "response.web_search_call.completed", + "error", +}; +// EVENTS-END + +const std::vector& ResponseEvents::known() { return kKnown; } + +bool ResponseEvents::is_known(const std::string& type) { + return std::find(kKnown.begin(), kKnown.end(), type) != kKnown.end(); +} + +ResponseEvent ResponseEvent::from_sse(const llm::SseEvent& e) { + ResponseEvent r; + r.data = e.data; + r.type = JsonX::str(e.data, "type", e.event); + r.sequence = JsonX::num(e.data, "sequence_number", -1); + return r; +} + +bool ResponseEvent::terminal() const { + return type == "response.completed" || type == "response.failed" || + type == "response.incomplete" || type == "error"; +} + +FunctionCallState* ResponseAccumulator::call_for(const std::string& item_id) { + for (FunctionCallState& c : calls_) + if (c.item_id == item_id) return &c; + return nullptr; +} + +bool ResponseAccumulator::feed(const ResponseEvent& ev) { + ++events_; + const Json& d = ev.data; + const std::string& t = ev.type; + if (t == "response.created" || t == "response.in_progress" || t == "response.queued") { + if (const Json* r = JsonX::obj(d, "response")) response_id_ = JsonX::str(*r, "id", response_id_); + } else if (t == "response.output_item.added" || t == "response.output_item.done") { + const Json* item = JsonX::obj(d, "item"); + if (item) { + std::string type = JsonX::str(*item, "type"); + std::string id = JsonX::str(*item, "id"); + if (type == "function_call") { + FunctionCallState* c = call_for(id); + if (!c) { + calls_.push_back({}); + c = &calls_.back(); + c->item_id = id; + } + c->call_id = JsonX::str(*item, "call_id", c->call_id); + c->name = JsonX::str(*item, "name", c->name); + if (t == "response.output_item.done") { + std::string args = JsonX::str(*item, "arguments"); + if (!args.empty()) c->arguments = args; + c->done = true; + } + } else if (type == "reasoning") { + auto it = std::find_if(reasoning_.begin(), reasoning_.end(), + [&](const ReasoningState& r) { return r.item_id == id; }); + if (it == reasoning_.end()) { + reasoning_.push_back({}); + reasoning_.back().item_id = id; + it = reasoning_.end() - 1; + } + if (t == "response.output_item.done") { + it->encrypted_content = JsonX::str(*item, "encrypted_content", it->encrypted_content); + if (it->summary.empty()) + if (const Json* s = JsonX::arr(*item, "summary")) + for (const Json& part : s->as_array()) it->summary += JsonX::str(part, "text"); + } + } + } + } else if (t == "response.output_text.delta") { + text_ += JsonX::str(d, "delta"); + } else if (t == "response.output_text.done") { + if (text_.empty()) text_ = JsonX::str(d, "text"); + } else if (t == "response.content_part.done") { + // The live SAM stream finalizes with content_part.done and no output_text.done. + if (const Json* p = JsonX::obj(d, "part")) + if (JsonX::str(*p, "type") == "output_text" && text_.empty()) text_ = JsonX::str(*p, "text"); + } else if (t == "response.function_call_arguments.delta") { + if (FunctionCallState* c = call_for(JsonX::str(d, "item_id"))) c->arguments += JsonX::str(d, "delta"); + } else if (t == "response.function_call_arguments.done") { + if (FunctionCallState* c = call_for(JsonX::str(d, "item_id"))) { + std::string args = JsonX::str(d, "arguments"); + if (!args.empty()) c->arguments = args; + } + } else if (t == "response.reasoning_summary_text.delta") { + std::string id = JsonX::str(d, "item_id"); + for (ReasoningState& r : reasoning_) + if (r.item_id == id) { + r.summary += JsonX::str(d, "delta"); + break; + } + } else if (t == "response.completed" || t == "response.incomplete" || t == "response.failed") { + if (const Json* r = JsonX::obj(d, "response")) { + response_ = *r; + response_id_ = JsonX::str(*r, "id", response_id_); + if (const Json* e = JsonX::obj(*r, "error")) { + error_code_ = JsonX::str(*e, "code"); + error_message_ = JsonX::str(*e, "message"); + } + } + completed_ = t == "response.completed"; + incomplete_ = t == "response.incomplete"; + failed_ = t == "response.failed"; + return false; + } else if (t == "error") { + failed_ = true; + error_code_ = JsonX::str(d, "code"); + error_message_ = JsonX::str(d, "message"); + return false; + } + return true; +} + +MetaUsage ResponseAccumulator::usage() const { + const Json* u = JsonX::obj(response_, "usage"); + return u ? MetaUsage::from_responses(*u) : MetaUsage(); +} + +} // namespace molo::meta diff --git a/src/meta/response_events.h b/src/meta/response_events.h new file mode 100644 index 0000000..f3fc085 --- /dev/null +++ b/src/meta/response_events.h @@ -0,0 +1,80 @@ +#pragma once + +// Responses API server-sent events. Each SSE data block is a JSON object whose +// `type` names the event (the docs' ResponseStreamEvent union); the SSE +// `event:` line, when present, is the same name. ResponseAccumulator folds a +// stream into the final text, reasoning summary, function calls and usage, and +// remembers the terminal response document. + +#include +#include +#include +#include + +#include "llm/sse.h" +#include "meta/jsonx.h" +#include "meta/usage.h" + +namespace molo::meta { + +struct ResponseEvent { + std::string type; // "response.output_text.delta", ... + long sequence = -1; // sequence_number, -1 when absent + Json data; // the whole event object + + static ResponseEvent from_sse(const llm::SseEvent& e); + bool terminal() const; // completed | failed | incomplete | error +}; + +class ResponseEvents { + public: + // Every event type in the docs' ResponseStreamEvent union (checked against + // the schemas page by tools/check_spec.py). + static const std::vector& known(); + static bool is_known(const std::string& type); +}; + +struct FunctionCallState { + std::string item_id; + std::string call_id; + std::string name; + std::string arguments; + bool done = false; +}; + +struct ReasoningState { + std::string item_id; + std::string summary; + std::string encrypted_content; +}; + +class ResponseAccumulator { + public: + // Returns false once a terminal event was folded. + bool feed(const ResponseEvent& event); + + const std::string& text() const { return text_; } + const std::vector& reasoning() const { return reasoning_; } + const std::vector& calls() const { return calls_; } + bool completed() const { return completed_; } + bool failed() const { return failed_; } + bool incomplete() const { return incomplete_; } + const std::string& error_message() const { return error_message_; } + const std::string& error_code() const { return error_code_; } + const std::string& response_id() const { return response_id_; } + const Json& response() const { return response_; } // terminal response document + MetaUsage usage() const; + std::size_t events() const { return events_; } + + private: + FunctionCallState* call_for(const std::string& item_id); + std::string text_; + std::vector reasoning_; + std::vector calls_; + bool completed_ = false, failed_ = false, incomplete_ = false; + std::string error_message_, error_code_, response_id_; + Json response_; + std::size_t events_ = 0; +}; + +} // namespace molo::meta diff --git a/src/meta/response_input.cpp b/src/meta/response_input.cpp new file mode 100644 index 0000000..ee43ba0 --- /dev/null +++ b/src/meta/response_input.cpp @@ -0,0 +1,254 @@ +#include "meta/response_input.h" + +#include "meta/caps.h" +#include "meta/check.h" +#include "security/crypto/Base64.h" + +namespace molo::meta { + +namespace { +Json obj(const char* type) { + Json o = Json::object(); + o.set("type", Json(type)); + return o; +} + +void image_mime(const std::string& mime) { + Check::one_of("mime", mime, + {"image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp", "image/x-icon"}); +} + +std::string data_url(const std::string& mime, const std::string& bytes) { + if (bytes.empty()) Check::fail("bytes are empty"); + if (bytes.size() > MetaCaps::kMaxInlineBytes) Check::fail("inline file is over the 50 MB limit"); + return "data:" + mime + ";base64," + security::crypto::Base64::encode(bytes); +} +} // namespace + +Json ResponseInput::text(const std::string& t) { + Json o = obj("input_text"); + o.set("text", Json(t)); + return o; +} + +Json ResponseInput::image_url(const std::string& url, const std::string& detail) { + Check::non_empty("image_url", url); + if (!detail.empty()) Check::one_of("detail", detail, {"low", "high", "auto", "original"}); + Json o = obj("input_image"); + o.set("image_url", Json(url)); + JsonX::set_if(o, "detail", detail); + return o; +} + +Json ResponseInput::image_file(const std::string& file_id, const std::string& detail) { + Check::segment("file_id", file_id); + if (!detail.empty()) Check::one_of("detail", detail, {"low", "high", "auto", "original"}); + Json o = obj("input_image"); + o.set("file_id", Json(file_id)); + JsonX::set_if(o, "detail", detail); + return o; +} + +Json ResponseInput::image_bytes(const std::string& mime, const std::string& bytes) { + image_mime(mime); + return image_url(data_url(mime, bytes)); +} + +Json ResponseInput::file_id(const std::string& id) { + Check::segment("file_id", id); + Json o = obj("input_file"); + o.set("file_id", Json(id)); + return o; +} + +Json ResponseInput::file_url(const std::string& url) { + if (url.rfind("http://", 0) != 0 && url.rfind("https://", 0) != 0) + Check::fail("file_url must be http or https"); + Json o = obj("input_file"); + o.set("file_url", Json(url)); + return o; +} + +Json ResponseInput::file_data(const std::string& filename, const std::string& mime, + const std::string& bytes) { + Check::one_of("mime", mime, + {"application/pdf", "text/plain", "application/json", "application/jsonl", + "image/png", "image/jpeg", "image/gif", "image/webp", "video/mp4", + "audio/mpeg", "audio/wav"}); + Json o = obj("input_file"); + JsonX::set_if(o, "filename", filename); + o.set("file_data", Json(data_url(mime, bytes))); + return o; +} + +Json ResponseInput::video_url(const std::string& url, double fps) { + Check::non_empty("video_url", url); + Json o = obj("input_video"); + o.set("video_url", Json(url)); + if (fps > 0) o.set("fps", Json(fps)); + return o; +} + +Json ResponseInput::video_file(const std::string& id, double fps) { + Check::segment("file_id", id); + Json o = obj("input_video"); + o.set("file_id", Json(id)); + if (fps > 0) o.set("fps", Json(fps)); + return o; +} + +Json ResponseInput::audio_bytes(const std::string& format, const std::string& bytes) { + Check::one_of("format", format, {"wav", "mp3"}); + if (bytes.empty() || bytes.size() > MetaCaps::kMaxInlineBytes) Check::fail("audio size is out of range"); + Json o = obj("input_audio"); + Json a = Json::object(); + a.set("data", Json(security::crypto::Base64::encode(bytes))); + a.set("format", Json(format)); + o.set("input_audio", a); + return o; +} + +Json ResponseInput::message(const std::string& role, const Json& content_parts, + const std::string& phase) { + Check::one_of("role", role, {"user", "assistant", "system", "developer"}); + if (!phase.empty()) Check::one_of("phase", phase, {"commentary", "final_answer"}); + Json o = obj("message"); + o.set("role", Json(role)); + o.set("content", content_parts); + JsonX::set_if(o, "phase", phase); + return o; +} + +Json ResponseInput::user(const std::string& t) { + Json parts = Json::array(); + parts.push(text(t)); + return message("user", parts); +} + +Json ResponseInput::developer(const std::string& t) { + Json parts = Json::array(); + parts.push(text(t)); + return message("developer", parts); +} + +Json ResponseInput::assistant(const std::string& t, const std::string& phase) { + Json part = obj("output_text"); + part.set("text", Json(t)); + part.set("annotations", Json::array()); + Json parts = Json::array(); + parts.push(part); + return message("assistant", parts, phase); +} + +Json ResponseInput::function_call(const std::string& call_id, const std::string& name, + const std::string& arguments_json) { + Check::call_id(call_id); + Check::function_name(name); + Json o = obj("function_call"); + o.set("call_id", Json(call_id)); + o.set("name", Json(name)); + o.set("arguments", Json(arguments_json.empty() ? "{}" : arguments_json)); + return o; +} + +Json ResponseInput::function_call_output(const std::string& call_id, const std::string& output) { + Check::call_id(call_id); + Json o = obj("function_call_output"); + o.set("call_id", Json(call_id)); + o.set("output", Json(output)); + return o; +} + +Json ResponseInput::reasoning(const std::string& encrypted_content, const std::string& id, + const std::vector& summary) { + Check::non_empty("encrypted_content", encrypted_content); + Json o = obj("reasoning"); + if (!id.empty()) { + Check::segment("reasoning id", id); + o.set("id", Json(id)); + } + o.set("encrypted_content", Json(encrypted_content)); + Json s = Json::array(); + for (const std::string& t : summary) { + Json part = obj("summary_text"); + part.set("text", Json(t)); + s.push(part); + } + o.set("summary", s); // required on replay, [] when there is none + return o; +} + +Json ResponseInput::item_reference(const std::string& id) { + Check::segment("item id", id); + Json o = obj("item_reference"); + o.set("id", Json(id)); + return o; +} + +std::size_t ResponseInput::count(const Json& input) { + if (input.is_string()) return 1; + if (input.is_array()) return input.as_array().size(); + return 0; +} + +Json ToolBuilder::function(const std::string& name, const std::string& description, + const Json& parameters, bool strict, bool defer_loading) { + Check::function_name(name); + Json o = obj("function"); + o.set("name", Json(name)); + JsonX::set_if(o, "description", description); + o.set("parameters", parameters.is_object() ? parameters : Json::object()); + if (strict) o.set("strict", Json(true)); + if (defer_loading) o.set("defer_loading", Json(true)); + return o; +} + +Json ToolBuilder::custom(const std::string& name, const std::string& description, bool strict, + bool defer_loading) { + Check::non_empty("name", name); + Json o = obj("custom"); + o.set("name", Json(name)); + JsonX::set_if(o, "description", description); + if (strict) o.set("strict", Json(true)); + if (defer_loading) o.set("defer_loading", Json(true)); + return o; +} + +Json ToolBuilder::web_search(const std::string& size, const Json& location, const Json& filters) { + Json o = obj("web_search"); + if (!size.empty()) { + Check::one_of("search_context_size", size, {"low", "medium", "high"}); + o.set("search_context_size", Json(size)); + } + if (location.is_object()) o.set("user_location", location); + if (filters.is_object()) o.set("filters", filters); + return o; +} + +Json ToolBuilder::tool_search(const std::string& execution) { + Json o = obj("tool_search"); + if (!execution.empty()) { + Check::one_of("execution", execution, {"server", "client"}); + o.set("execution", Json(execution)); + } + return o; +} + +Json ToolBuilder::computer() { return obj("computer"); } + +Json ToolBuilder::namespace_tool(const std::string& name, const std::string& description, + const std::vector& functions) { + Check::non_empty("name", name); + if (name.find('.') != std::string::npos) Check::fail("namespace name must contain no dots"); + if (functions.empty() || functions.size() > MetaCaps::kMaxTools) + Check::fail("namespace needs 1.." + std::to_string(MetaCaps::kMaxTools) + " functions"); + Json o = obj("namespace"); + o.set("name", Json(name)); + JsonX::set_if(o, "description", description); + Json tools = Json::array(); + for (const Json& f : functions) tools.push(f); + o.set("tools", tools); + return o; +} + +} // namespace molo::meta diff --git a/src/meta/response_input.h b/src/meta/response_input.h new file mode 100644 index 0000000..d774102 --- /dev/null +++ b/src/meta/response_input.h @@ -0,0 +1,67 @@ +#pragma once + +// Typed builders for Responses API input items and tools. Shapes follow the +// Responses schemas page (InputItem, EasyInputMessage, InputText/Image/File/ +// Video content, FunctionCallOutputItemParam, ReasoningItemParam, Tool union). +// Anything the docs describe only in prose (custom and namespace tools) is +// marked in SPEC "Unverified". + +#include +#include + +#include "meta/jsonx.h" + +namespace molo::meta { + +class ResponseInput { + public: + // Content parts. + static Json text(const std::string& text); + static Json image_url(const std::string& url, const std::string& detail = ""); + static Json image_file(const std::string& file_id, const std::string& detail = ""); + // Bytes become a data: URL; mime must be a documented image type. + static Json image_bytes(const std::string& mime, const std::string& bytes); + static Json file_id(const std::string& file_id); + static Json file_url(const std::string& url); + static Json file_data(const std::string& filename, const std::string& mime, + const std::string& bytes); + static Json video_url(const std::string& url, double fps = 0); + static Json video_file(const std::string& file_id, double fps = 0); + static Json audio_bytes(const std::string& format, const std::string& bytes); // wav | mp3 + + // Items. + static Json message(const std::string& role, const Json& content_parts, + const std::string& phase = ""); + static Json user(const std::string& text); + static Json developer(const std::string& text); + static Json assistant(const std::string& text, const std::string& phase = ""); + static Json function_call(const std::string& call_id, const std::string& name, + const std::string& arguments_json); + static Json function_call_output(const std::string& call_id, const std::string& output); + // Stateless reasoning replay: `summary` may be empty, encrypted_content must + // be the exact value the previous response returned. + static Json reasoning(const std::string& encrypted_content, const std::string& id = "", + const std::vector& summary = {}); + static Json item_reference(const std::string& id); + + // Counts input items of an `input` value (string = 1). + static std::size_t count(const Json& input); +}; + +class ToolBuilder { + public: + static Json function(const std::string& name, const std::string& description, + const Json& parameters, bool strict = false, bool defer_loading = false); + // docs: tool-calling "Custom tools" (prose only, not in the schema union). + static Json custom(const std::string& name, const std::string& description, + bool strict = false, bool defer_loading = false); + static Json web_search(const std::string& search_context_size = "", + const Json& user_location = Json(), const Json& filters = Json()); + static Json tool_search(const std::string& execution = ""); // server | client + static Json computer(); + // docs: tool-search "Deferring tools" (prose only). + static Json namespace_tool(const std::string& name, const std::string& description, + const std::vector& functions); +}; + +} // namespace molo::meta diff --git a/src/meta/response_request.cpp b/src/meta/response_request.cpp new file mode 100644 index 0000000..264c28f --- /dev/null +++ b/src/meta/response_request.cpp @@ -0,0 +1,170 @@ +#include "meta/response_request.h" + +#include + +#include "meta/caps.h" +#include "meta/check.h" +#include "meta/schema_check.h" + +namespace molo::meta { + +namespace { +bool has(const std::vector& v, const char* s) { + return std::find(v.begin(), v.end(), s) != v.end(); +} +} // namespace + +Json ResponseRequest::format_json_object() { + Json f = Json::object(); + f.set("type", Json("json_object")); + return f; +} + +Json ResponseRequest::format_json_schema(const std::string& name, const Json& schema, bool strict, + const std::string& description) { + Check::non_empty("name", name); + SchemaCheck::check(schema, strict); + Json f = Json::object(); + f.set("type", Json("json_schema")); + f.set("name", Json(name)); + f.set("schema", schema); + if (strict) f.set("strict", Json(true)); + JsonX::set_if(f, "description", description); + return f; +} + +void ResponseRequest::validate(bool streaming) const { + Check::model_id(model); + if (input.is_string()) { + Check::non_empty("input", input.as_string()); + Check::max_bytes("input", input.as_string(), MetaCaps::kMaxPromptBytes); + } else if (input.is_array()) { + if (input.as_array().empty() || input.as_array().size() > MetaCaps::kMaxInputItems) + Check::fail("input needs 1.." + std::to_string(MetaCaps::kMaxInputItems) + " items"); + } else { + Check::fail("input is required (a string or an array of items)"); + } + if (max_output_tokens && *max_output_tokens < MetaCaps::kMinOutputTokens) + Check::fail("max_output_tokens must be at least 16"); + if (max_tool_calls && *max_tool_calls < 1) Check::fail("max_tool_calls must be at least 1"); + if (temperature) Check::temperature(*temperature); + if (top_p) Check::top_p(*top_p); + if (frequency_penalty) Check::penalty("frequency_penalty", *frequency_penalty); + if (presence_penalty) Check::penalty("presence_penalty", *presence_penalty); + Check::reasoning_effort(model, reasoning_effort); + if (!reasoning_summary.empty()) + Check::one_of("reasoning.summary", reasoning_summary, {"auto", "concise", "detailed"}); + Check::safety_identifier(safety_identifier); + Check::prompt_cache_retention(prompt_cache_retention); + if (!service_tier.empty()) + Check::one_of("service_tier", service_tier, {"auto", "default", "flex", "scale", "priority"}); + if (!truncation.empty() && truncation != "disabled") + Check::fail("truncation \"auto\" is rejected; only \"disabled\" is accepted"); + if (tools.size() > MetaCaps::kMaxTools) Check::fail("too many tools"); + for (const Json& t : tools) { + if (JsonX::str(t, "type") == "function") Check::function_name(JsonX::str(t, "name")); + } + if (metadata.size() > MetaCaps::kMaxMetadataPairs) Check::fail("too many metadata pairs"); + for (const std::string& i : include) { + Check::one_of("include", i, + {"file_search_call.results", "web_search_call.results", + "web_search_call.action.sources", "message.input_image.image_url", + "computer_call_output.output.image_url", "code_interpreter_call.outputs", + "reasoning.encrypted_content"}); + } + if (has(include, "reasoning.encrypted_content") && !previous_response_id.empty()) + Check::fail("include reasoning.encrypted_content cannot be combined with previous_response_id"); + if (!previous_response_id.empty()) Check::segment("previous_response_id", previous_response_id); + if (text_format.is_object()) { + std::string type = JsonX::str(text_format, "type"); + Check::one_of("text.format.type", type, {"text", "json_object", "json_schema"}); + if (type == "json_schema") { + if (const Json* s = JsonX::obj(text_format, "schema")) + SchemaCheck::check(*s, JsonX::flag(text_format, "strict")); + else + Check::fail("text.format json_schema needs a schema"); + } + } + if (background.value_or(false) && streaming) + Check::fail("background cannot be combined with stream"); +} + +Json ResponseRequest::to_json(bool stream) const { + validate(stream); + Json b = Json::object(); + b.set("model", Json(model)); + b.set("input", input); + JsonX::set_if(b, "instructions", instructions); + JsonX::set_if(b, "previous_response_id", previous_response_id); + if (!reasoning_effort.empty() || !reasoning_summary.empty()) { + Json r = Json::object(); + JsonX::set_if(r, "effort", reasoning_effort); + JsonX::set_if(r, "summary", reasoning_summary); + b.set("reasoning", r); + } + if (!include.empty()) b.set("include", JsonX::strings(include)); + if (!tools.empty()) { + Json t = Json::array(); + for (const Json& x : tools) t.push(x); + b.set("tools", t); + } + if (text_format.is_object()) { + Json t = Json::object(); + t.set("format", text_format); + b.set("text", t); + } + if (!metadata.empty()) { + Json m = Json::object(); + for (const auto& [k, v] : metadata) m.set(k, Json(v)); + b.set("metadata", m); + } + JsonX::set_if(b, "safety_identifier", safety_identifier); + JsonX::set_if(b, "prompt_cache_key", prompt_cache_key); + JsonX::set_if(b, "prompt_cache_retention", prompt_cache_retention); + JsonX::set_if(b, "service_tier", service_tier); + JsonX::set_if(b, "truncation", truncation); + if (max_output_tokens) b.set("max_output_tokens", Json(*max_output_tokens)); + if (max_tool_calls) b.set("max_tool_calls", Json(*max_tool_calls)); + if (temperature) b.set("temperature", Json(*temperature)); + if (top_p) b.set("top_p", Json(*top_p)); + if (frequency_penalty) b.set("frequency_penalty", Json(*frequency_penalty)); + if (presence_penalty) b.set("presence_penalty", Json(*presence_penalty)); + if (store) b.set("store", Json(*store)); + if (background) b.set("background", Json(*background)); + if (parallel_tool_calls) b.set("parallel_tool_calls", Json(*parallel_tool_calls)); + if (stream) b.set("stream", Json(true)); + JsonX::merge(b, extra); + if (background.value_or(false) && b.dump().size() > MetaCaps::kMaxBackgroundBodyBytes) + Check::fail("a background request body must stay under 1 MiB"); + return b; +} + +Json ResponseRequest::to_count_json() const { + validate(false); + Json b = Json::object(); + b.set("model", Json(model)); + b.set("input", input); + JsonX::set_if(b, "instructions", instructions); + JsonX::set_if(b, "previous_response_id", previous_response_id); + if (!reasoning_effort.empty() || !reasoning_summary.empty()) { + Json r = Json::object(); + JsonX::set_if(r, "effort", reasoning_effort); + JsonX::set_if(r, "summary", reasoning_summary); + b.set("reasoning", r); + } + if (!tools.empty()) { + Json t = Json::array(); + for (const Json& x : tools) t.push(x); + b.set("tools", t); + } + if (text_format.is_object()) { + Json t = Json::object(); + t.set("format", text_format); + b.set("text", t); + } + JsonX::set_if(b, "truncation", truncation); + if (parallel_tool_calls) b.set("parallel_tool_calls", Json(*parallel_tool_calls)); + return b; +} + +} // namespace molo::meta diff --git a/src/meta/response_request.h b/src/meta/response_request.h new file mode 100644 index 0000000..3807f7d --- /dev/null +++ b/src/meta/response_request.h @@ -0,0 +1,65 @@ +#pragma once + +// A typed POST /responses (and POST /responses/input_tokens) body. validate() +// enforces the rules the docs state, before any network traffic: +// model required; input required (string or 1..kMaxInputItems items) +// max_output_tokens >= 16; temperature 0..2; top_p in (0,1]; penalties -2..2 +// reasoning.effort in minimal..max ("none" rejected, "max" per catalog) +// background and stream are mutually exclusive +// include "reasoning.encrypted_content" cannot pair with previous_response_id +// truncation only "disabled"; logprobs include is rejected +// function tool names, call ids, safety_identifier length, tool count caps +// a background body stays under 1 MiB +// `extra` is merged last for fields the docs add after this code was written. + +#include +#include +#include + +#include "meta/jsonx.h" + +namespace molo::meta { + +struct ResponseRequest { + std::string model; + Json input; // string or array of items (see ResponseInput) + std::string instructions; + std::string previous_response_id; + std::string reasoning_effort; // minimal..max + std::string reasoning_summary; // auto | concise | detailed + std::vector include; + std::vector tools; + Json text_format; // null, or {type: json_object | json_schema, ...} + std::vector> metadata; + std::string safety_identifier; + std::string prompt_cache_key; + std::string prompt_cache_retention; // in_memory | 24h + std::string service_tier; // auto default flex scale priority + std::string truncation; // disabled only + std::optional max_output_tokens; + std::optional max_tool_calls; + std::optional temperature; + std::optional top_p; + std::optional frequency_penalty; + std::optional presence_penalty; + std::optional store; + std::optional background; + std::optional parallel_tool_calls; + Json extra; + + void validate(bool streaming = false) const; + // Body for POST /responses. `stream` forces the stream flag. + Json to_json(bool stream = false) const; + // Body for POST /responses/input_tokens: only fields the token counter + // takes (no sampling, no store, no stream, no background). + Json to_count_json() const; + + // text.format builders (structured output). + static Json format_json_object(); + // Throws unless the schema is a plain JSON object schema; strict also + // requires additionalProperties:false on the root (docs: structured-output). + static Json format_json_schema(const std::string& name, const Json& schema, + bool strict = false, const std::string& description = ""); +}; + +} // namespace molo::meta diff --git a/src/meta/responses.cpp b/src/meta/responses.cpp new file mode 100644 index 0000000..5104d34 --- /dev/null +++ b/src/meta/responses.cpp @@ -0,0 +1,104 @@ +#include "meta/responses.h" + +#include "meta/caps.h" +#include "meta/check.h" + +namespace molo::meta { + +namespace { +ResponseResult from_raw(const llm::RawResponse& raw) { + ResponseResult r; + try { + r.body = raw.body.find_first_not_of(" \t\r\n") == std::string::npos ? Json::object() + : Json::parse(raw.body); + } catch (const std::exception& e) { + throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what()); + } + r.limits = RateLimits::from_headers(raw.headers); + return r; +} +} // namespace + +std::string ResponseResult::output_text() const { + std::string out; + if (const Json* o = JsonX::arr(body, "output")) + for (const Json& item : o->as_array()) { + if (JsonX::str(item, "type") != "message") continue; + if (const Json* c = JsonX::arr(item, "content")) + for (const Json& part : c->as_array()) + if (JsonX::str(part, "type") == "output_text") out += JsonX::str(part, "text"); + } + return out; +} + +MetaUsage ResponseResult::usage() const { + const Json* u = JsonX::obj(body, "usage"); + return u ? MetaUsage::from_responses(*u) : MetaUsage(); +} + +ResponseResult ResponsesService::create(const ResponseRequest& request) { + Json body = request.to_json(false); + std::string wire = body.dump(); + return from_raw(client_->request_raw(method_of(Call::kResponsesCreate), + path_for(Call::kResponsesCreate), &wire, + "application/json")); +} + +ResponseAccumulator ResponsesService::stream(const ResponseRequest& request, + const EventSink& sink) { + Json body = request.to_json(true); + ResponseAccumulator acc; + std::size_t seen = 0; + client_->stream(path_for(Call::kResponsesCreate), body, [&](const llm::SseEvent& e) { + if (++seen > MetaCaps::kMaxStreamEvents) return false; // hard cap on events per stream + ResponseEvent ev = ResponseEvent::from_sse(e); + bool more = acc.feed(ev); + if (sink && !sink(ev)) return false; + return more; + }); + return acc; +} + +ResponseResult ResponsesService::retrieve(const std::string& response_id) { + Check::segment("response_id", response_id); + return from_raw(client_->request_raw(method_of(Call::kResponsesRetrieve), + path_for(Call::kResponsesRetrieve, response_id))); +} + +ResponseAccumulator ResponsesService::retrieve_stream(const std::string& response_id, + long starting_after, const EventSink& sink) { + Check::segment("response_id", response_id); + std::string path = path_for(Call::kResponsesRetrieve, response_id) + "?stream=true"; + if (starting_after >= 0) path += "&starting_after=" + std::to_string(starting_after); + llm::RawResponse raw = client_->request_raw(method_of(Call::kResponsesRetrieve), path, nullptr, + "", {{"Accept", "text/event-stream"}}); + ResponseAccumulator acc; + llm::SseDecoder decoder("meta", [&](const llm::SseEvent& e) { + ResponseEvent ev = ResponseEvent::from_sse(e); + bool more = acc.feed(ev); + if (sink && !sink(ev)) return false; + return more; + }); + decoder.feed(raw.body); + decoder.finish(); + return acc; +} + +Json ResponsesService::remove(const std::string& response_id) { + Check::segment("response_id", response_id); + return client_->del(path_for(Call::kResponsesDelete, response_id)); +} + +Json ResponsesService::cancel(const std::string& response_id) { + Check::segment("response_id", response_id); + Json empty = Json::object(); + return client_->post(path_for(Call::kResponsesCancel, response_id), empty); +} + +long ResponsesService::count_input_tokens(const ResponseRequest& request) { + Json body = request.to_count_json(); + Json out = client_->post(path_for(Call::kResponsesInputTokens), body); + return JsonX::num(out, "input_tokens", -1); +} + +} // namespace molo::meta diff --git a/src/meta/responses.h b/src/meta/responses.h new file mode 100644 index 0000000..28c2816 --- /dev/null +++ b/src/meta/responses.h @@ -0,0 +1,55 @@ +#pragma once + +// POST /responses and its lifecycle calls (retrieve, delete, cancel, count +// input tokens). The Responses API is the endpoint that carries reasoning +// across turns (stateless encrypted replay or previous_response_id), serves +// tools, search grounding, files, SAM and Muse Image, and runs background +// responses. + +#include +#include + +#include "llm/api_client.h" +#include "meta/calls.h" +#include "meta/response_events.h" +#include "meta/response_request.h" +#include "meta/usage.h" + +namespace molo::meta { + +struct ResponseResult { + Json body; + RateLimits limits; // x-ratelimit-* headers when the server sent them + std::string id() const { return JsonX::str(body, "id"); } + std::string status() const { return JsonX::str(body, "status"); } + // Concatenated output_text of every assistant message item. + std::string output_text() const; + MetaUsage usage() const; +}; + +class ResponsesService : public llm::Service { + public: + using EventSink = std::function; // false: stop + + explicit ResponsesService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + + ResponseResult create(const ResponseRequest& request); + // Streams events; returns the folded accumulator. A stream that ends before a + // terminal event leaves completed()/failed() false (the caller decides). + ResponseAccumulator stream(const ResponseRequest& request, const EventSink& sink = nullptr); + + ResponseResult retrieve(const std::string& response_id); + // Streaming retrieval of a background response (lifecycle events only). The + // client has no GET-SSE transport, so the body is read whole, then decoded. + ResponseAccumulator retrieve_stream(const std::string& response_id, long starting_after = -1, + const EventSink& sink = nullptr); + // Destructive: soft-deletes the stored response. No default, no retry. + Json remove(const std::string& response_id); + // Cancels a background response; a finished response comes back unchanged. + Json cancel(const std::string& response_id); + // Tokenizes only: no generation, no storage, no usage record. + long count_input_tokens(const ResponseRequest& request); +}; + +} // namespace molo::meta diff --git a/src/meta/sam.cpp b/src/meta/sam.cpp new file mode 100644 index 0000000..82b6bec --- /dev/null +++ b/src/meta/sam.cpp @@ -0,0 +1,237 @@ +#include "meta/sam.h" + +#include + +#include "llm/error.h" +#include "meta/caps.h" +#include "meta/check.h" +#include "meta/response_input.h" + +namespace molo::meta { + +namespace { + +[[noreturn]] void bad(const std::string& why) { + throw llm::ApiError::invalid("meta", "sam output: " + why); +} + +struct Cursor { + const std::string& s; + std::size_t p = 0; + bool done() const { return p >= s.size(); } + char peek() const { return p < s.size() ? s[p] : '\0'; } + bool starts(const char* lit) const { return s.compare(p, std::char_traits::length(lit), lit) == 0; } + void expect(const char* lit) { + if (!starts(lit)) bad(std::string("expected ") + lit); + p += std::char_traits::length(lit); + } + long integer() { + std::size_t q = p; + while (q < s.size() && s[q] >= '0' && s[q] <= '9') ++q; + if (q == p || q - p > 15) bad("expected an integer"); + long v = std::strtol(s.substr(p, q - p).c_str(), nullptr, 10); + p = q; + return v; + } +}; + +// "x1=1;y1=2;...": key/value pairs between "<|name;" and "|>". `data` is the +// only key whose value holds commas; it never holds ';' or '|'. +std::vector> fields(const std::string& body) { + std::vector> out; + std::size_t start = 0; + while (start <= body.size()) { + std::size_t semi = body.find(';', start); + std::string kv = body.substr(start, semi == std::string::npos ? semi : semi - start); + if (!kv.empty()) { + std::size_t eq = kv.find('='); + if (eq == std::string::npos || eq == 0) bad("field without a key"); + out.emplace_back(kv.substr(0, eq), kv.substr(eq + 1)); + } + if (semi == std::string::npos) break; + start = semi + 1; + } + return out; +} + +std::string token_body(Cursor& c, const char* open) { + c.expect(open); + std::size_t end = c.s.find("|>", c.p); + if (end == std::string::npos) bad("unterminated token"); + std::string body = c.s.substr(c.p, end - c.p); + c.p = end + 2; + return body; +} + +int to_int(const std::string& v) { + if (v.empty() || v.size() > 9) bad("bad integer field"); + char* e = nullptr; + long x = std::strtol(v.c_str(), &e, 10); + if (*e != '\0') bad("bad integer field"); + return static_cast(x); +} + +double to_conf(const std::string& v) { + char* e = nullptr; + double x = std::strtod(v.c_str(), &e); + if (v.empty() || *e != '\0' || x < 0.0 || x > 1.0) bad("confidence must be 0..1"); + return x; +} + +SamBox parse_box(const std::string& body) { + SamBox b; + int seen = 0; + for (const auto& [k, v] : fields(body)) { + if (k == "x1") { b.x1 = to_int(v); seen |= 1; } + else if (k == "y1") { b.y1 = to_int(v); seen |= 2; } + else if (k == "x2") { b.x2 = to_int(v); seen |= 4; } + else if (k == "y2") { b.y2 = to_int(v); seen |= 8; } + else if (k == "w") { b.w = to_int(v); seen |= 16; } + else if (k == "h") { b.h = to_int(v); seen |= 32; } + else if (k == "c") b.confidence = to_conf(v); + } + if (seen != 63) bad("box is missing coordinates"); + return b; +} + +SamMask parse_mask(const std::string& body) { + SamMask m; + bool have = false; + for (const auto& [k, v] : fields(body)) { + if (k == "c") m.confidence = to_conf(v); + else if (k == "data") { + std::size_t c1 = v.find(','); + std::size_t c2 = c1 == std::string::npos ? c1 : v.find(',', c1 + 1); + if (c2 == std::string::npos || c2 + 1 >= v.size()) bad("mask data needs H,W,payload"); + m.height = to_int(v.substr(0, c1)); + m.width = to_int(v.substr(c1 + 1, c2 - c1 - 1)); + m.encoding = v[c2 + 1]; + m.payload = v.substr(c2 + 1); + if (m.encoding != '~' && m.encoding != '!') bad("unknown mask encoding marker"); + if (m.height < 1 || m.width < 1) bad("mask dimensions must be positive"); + have = true; + } + } + if (!have) bad("mask has no data"); + return m; +} + +} // namespace + +std::vector SamOutput::parse(const std::string& text) { + std::vector frames; + std::size_t start = 0, lines = 0, records = 0; + while (start < text.size()) { + std::size_t nl = text.find('\n', start); + std::string line = text.substr(start, nl == std::string::npos ? nl : nl - start); + start = nl == std::string::npos ? text.size() : nl + 1; + if (line.find_first_not_of(" \t\r") == std::string::npos) continue; + if (++lines > MetaCaps::kMaxSamLines) bad("too many lines"); + Cursor c{line}; + c.expect("<"); + SamFrame frame; + frame.frame = c.integer(); + c.expect("f>"); + while (true) { + SamRecord r; + r.object_id = c.integer(); + r.box = parse_box(token_body(c, "<|box;")); + r.mask = parse_mask(token_body(c, "<|mask;")); + if (++records > MetaCaps::kMaxSamRecords) bad("too many records"); + frame.records.push_back(std::move(r)); + if (c.done()) break; + c.expect(","); + } + frames.push_back(std::move(frame)); + } + return frames; +} + +std::size_t SamOutput::record_count(const std::vector& frames) { + std::size_t n = 0; + for (const SamFrame& f : frames) n += f.records.size(); + return n; +} + +std::string SamPrompt::concept_text(const std::string& phrase) { + Check::non_empty("concept", phrase); + Check::max_bytes("concept", phrase, 200); + for (char c : phrase) + if (c == ',' || c == ';' || c == '+' || c == '\n' || c == '<' || c == '>' || c == '|') + Check::fail("one concept per request: no lists, newlines or tokens"); + if (phrase.find(" and ") != std::string::npos || phrase.find(" or ") != std::string::npos) + Check::fail("one concept per request: \"and\" / \"or\" segment only one of them"); + return phrase; +} + +namespace { +std::string box_token(const SamBox& b) { + return "<|box;x1=" + std::to_string(b.x1) + ";y1=" + std::to_string(b.y1) + ";x2=" + + std::to_string(b.x2) + ";y2=" + std::to_string(b.y2) + ";w=" + std::to_string(b.w) + + ";h=" + std::to_string(b.h) + "|>"; +} +} // namespace + +std::string SamPrompt::objects_text(long frame, const std::vector& objects) { + if (frame < 0) Check::fail("frame must be >= 0"); + if (objects.empty()) Check::fail("a frame needs at least one object"); + std::string out = "<" + std::to_string(frame) + "f>"; + std::vector ids; + for (std::size_t i = 0; i < objects.size(); ++i) { + const SamObject& o = objects[i]; + for (long id : ids) + if (id == o.id) Check::fail("an object id appears at most once per frame"); + ids.push_back(o.id); + if (o.id < 0) Check::fail("object id must be >= 0"); + if (!o.box && o.points.empty()) Check::fail("an object needs a box or a point"); + if (i) out += ","; + out += std::to_string(o.id); + if (o.box) { + if (o.box->w < 1 || o.box->h < 1) Check::fail("box needs the media size w,h"); + if (o.box->x2 < o.box->x1 || o.box->y2 < o.box->y1) Check::fail("box corners are reversed"); + out += box_token(*o.box); + } + for (const SamPoint& p : o.points) { + if (o.w < 1 || o.h < 1) Check::fail("points need the media size w,h"); + if (p.x < 0 || p.y < 0 || p.x >= o.w || p.y >= o.h) Check::fail("point is outside the media"); + if (!p.positive) out += "-"; + out += "<|point;x=" + std::to_string(p.x) + ";y=" + std::to_string(p.y) + ";w=" + + std::to_string(o.w) + ";h=" + std::to_string(o.h) + "|>"; + } + } + return out; +} + +std::string SamPrompt::frames_text( + const std::vector>>& frames) { + if (frames.empty()) Check::fail("no frames"); + std::string out; + for (const auto& [frame, objects] : frames) out += objects_text(frame, objects); + return out; +} + +ResponseRequest SamRequest::to_response_request() const { + Check::non_empty("prompt", prompt); + int media = (!image_url.empty() ? 1 : 0) + (!video_url.empty() ? 1 : 0) + + (!video_file_id.empty() ? 1 : 0); + if (media != 1) Check::fail("send exactly one of image_url, video_url, video_file_id"); + if (!propagation_direction.empty()) + Check::one_of("propagation_direction", propagation_direction, {"both", "forward", "backward"}); + if (!mask_encoding.empty()) Check::one_of("mask_encoding", mask_encoding, {"lossless", "one_bit"}); + Json parts = Json::array(); + parts.push(ResponseInput::text(prompt)); + if (!image_url.empty()) parts.push(ResponseInput::image_url(image_url)); + else if (!video_url.empty()) parts.push(ResponseInput::video_url(video_url)); + else parts.push(ResponseInput::video_file(video_file_id)); + ResponseRequest r; + r.model = model; + Json input = Json::array(); + input.push(ResponseInput::message("user", parts)); + r.input = input; + if (!propagation_direction.empty()) r.metadata.emplace_back("propagation_direction", propagation_direction); + if (include_confidence) r.metadata.emplace_back("include_confidence", "true"); // string, not a boolean + if (!mask_encoding.empty()) r.metadata.emplace_back("mask_encoding", mask_encoding); + return r; +} + +} // namespace molo::meta diff --git a/src/meta/sam.h b/src/meta/sam.h new file mode 100644 index 0000000..68ed291 --- /dev/null +++ b/src/meta/sam.h @@ -0,0 +1,98 @@ +#pragma once + +// Segment Anything Model 3.1 (model sam-3.1) over the Responses API. There is +// no segmentation endpoint: the prompt (a noun phrase, or box / point tokens) +// goes in an input_text part beside one input_image or input_video part, and +// the answer arrives as special-token text in one output_text lane, one line +// per frame: +// id<|box;x1=..;y1=..;x2=..;y2=..;w=..;h=..[;c=..]|><|mask;x=0;y=0[;c=..];data=H,W,payload|>,id... +// This module builds the request, parses that text, and decodes masks +// (one_bit "!" and lossless "~") to a 0/1 raster. The mask codec is ported +// from the official meta-sam parser (see SPEC Sources) and checked against its +// conformance cases. + +#include +#include +#include +#include + +#include "meta/response_request.h" + +namespace molo::meta { + +struct SamBox { + int x1 = 0, y1 = 0, x2 = 0, y2 = 0; // x2 / y2 are inclusive, as on the wire + int w = 0, h = 0; // source frame size + double confidence = -1; // -1 = absent + int right() const { return x2 + 1; } // exclusive edge + int bottom() const { return y2 + 1; } +}; + +struct SamMask { + int height = 0, width = 0; + char encoding = 0; // '~' lossless, '!' one_bit + std::string payload; // including the encoding marker + double confidence = -1; + // Row-major height*width bytes of 0/1. Throws ApiError on a malformed payload. + std::vector decode() const; + std::string encoding_name() const { return encoding == '!' ? "one_bit" : "lossless"; } +}; + +struct SamRecord { + long object_id = 0; + SamBox box; + SamMask mask; +}; + +struct SamFrame { + long frame = 0; // zero-based, not necessarily contiguous or ascending + std::vector records; +}; + +class SamOutput { + public: + // Parses the whole output_text (lines separated by \n). Empty text is a valid + // "no matches" result. Throws ApiError on a malformed line; caps lines and + // records. + static std::vector parse(const std::string& text); + static std::size_t record_count(const std::vector& frames); +}; + +struct SamPoint { + int x = 0, y = 0; + bool positive = true; +}; + +struct SamObject { + long id = 0; + std::optional box; // x1..y2 and w,h + std::vector points; + int w = 0, h = 0; // media size for the points +}; + +class SamPrompt { + public: + // One short noun phrase; commas, semicolons, plus signs, newlines and + // "and" list several concepts (docs: that returns nothing), so they are rejected. + static std::string concept_text(const std::string& phrase); + // Box and point prompts for one frame: "id<|box;...|>,id<|point;...|>-<|point;...|>". + static std::string objects_text(long frame, const std::vector& objects); + // Several frames, in the order given. + static std::string frames_text(const std::vector>>& frames); +}; + +struct SamRequest { + std::string model = "sam-3.1"; + std::string prompt; // from SamPrompt + std::string image_url; // exactly one of image_url / video_url / video_file_id + std::string video_url; + std::string video_file_id; + std::string propagation_direction; // "" (both) | forward | backward | both (video prompts) + bool include_confidence = false; + std::string mask_encoding; // "" (lossless) | lossless | one_bit + bool stream = true; // docs: recommended + + ResponseRequest to_response_request() const; +}; + +} // namespace molo::meta diff --git a/src/meta/sam_mask.cpp b/src/meta/sam_mask.cpp new file mode 100644 index 0000000..d6bc96c --- /dev/null +++ b/src/meta/sam_mask.cpp @@ -0,0 +1,270 @@ +// Mask codec for SAM output: base85 (custom alphabet) + adaptive range coder. +// Ported from the official meta-sam parser (python/src/meta_sam_parser/ +// _mask_codec.py); verified against its conformance cases in the tests. +#include +#include +#include + +#include "llm/error.h" +#include "meta/caps.h" +#include "meta/sam.h" + +namespace molo::meta { + +namespace { + +[[noreturn]] void bad(const std::string& why) { + throw llm::ApiError::invalid("meta", "sam mask: " + why); +} + +constexpr int kRadix = 85; +constexpr int kPrefix = 5; +constexpr std::uint32_t kTop = 1u << 24; +constexpr std::uint32_t kBottom = 1u << 16; + +int digit_of(char c) { + static int table[128]; + static bool init = false; + if (!init) { + for (int& t : table) t = -1; + int idx = 0; + for (int code = 0x21; code < 0x7F; ++code) { + char ch = static_cast(code); + if (ch == '"' || ch == '\\' || ch == ',' || ch == ';' || ch == '<' || ch == '>' || ch == '|') continue; + table[code] = idx++; + } + init = true; + } + unsigned char u = static_cast(c); + int v = u < 128 ? table[u] : -1; + if (v < 0 || v >= kRadix) bad("payload has an invalid character"); + return v; +} + +std::vector unpack(const std::string& p) { + if (p.size() < kPrefix) bad("payload is missing its length prefix"); + std::uint64_t length = 0; + for (int i = 0; i < kPrefix; ++i) length = length * kRadix + digit_of(p[i]); + if (length > MetaCaps::kMaxMaskPixels * 4ull) bad("payload length is unsupported"); + std::uint64_t rem = length % 4; + std::uint64_t expected = kPrefix + (length / 4) * 5 + (rem == 0 ? 0 : rem + 1); + if (p.size() != expected) bad("payload length does not match its prefix"); + std::vector out(length); + std::size_t src = kPrefix, dst = 0; + std::uint64_t full = length - rem; + while (dst < full) { + std::uint64_t v = 0; + for (int i = 0; i < 5; ++i) v = v * kRadix + digit_of(p[src + i]); + if (v > 0xFFFFFFFFull) bad("payload group is out of range"); + src += 5; + out[dst] = (v >> 24) & 0xFF; out[dst + 1] = (v >> 16) & 0xFF; + out[dst + 2] = (v >> 8) & 0xFF; out[dst + 3] = v & 0xFF; + dst += 4; + } + if (rem) { + std::uint64_t v = 0; + for (std::uint64_t i = 0; i < 5; ++i) v = v * kRadix + (i < rem + 1 ? digit_of(p[src + i]) : kRadix - 1); + if (v > 0xFFFFFFFFull) bad("payload tail is out of range"); + for (std::uint64_t i = 0; i < rem; ++i) out[dst + i] = (v >> (24 - i * 8)) & 0xFF; + } + return out; +} + +class RangeDecoder { + public: + RangeDecoder(const std::uint8_t* d, std::size_t n) : in_(d), n_(n) { + if (n < 4) bad("payload is missing its finalization"); + for (int i = 0; i < 4; ++i) code_ = (code_ << 8) | read(); + } + std::uint32_t freq(std::uint32_t total) { + scaled_ = range_ / total; + if (scaled_ == 0) bad("invalid coding state"); + std::uint32_t v = (code_ - low_) / scaled_; + return v >= total ? total - 1 : v; + } + void decode(std::uint32_t cum, std::uint32_t f) { + low_ += scaled_ * cum; + range_ = scaled_ * f; + while (true) { + if ((low_ ^ (low_ + range_)) < kTop) { + } else if (range_ < kBottom) { + range_ = (0u - low_) & (kBottom - 1); + } else { + break; + } + code_ = (code_ << 8) | read(); + low_ <<= 8; + range_ <<= 8; + } + } + + private: + std::uint32_t read() { + if (pos_ >= n_) bad("payload ended before decoding completed"); + return in_[pos_++]; + } + const std::uint8_t* in_; + std::size_t n_, pos_ = 0; + std::uint32_t low_ = 0, range_ = 0xFFFFFFFFu, code_ = 0, scaled_ = 0; +}; + +std::vector decode_one_bit(const std::string& payload, int width, int height) { + std::vector raw = unpack(payload.substr(1)); + RangeDecoder dec(raw.data(), raw.size()); + std::vector counts(4096 * 2, 1); + const int pw = width + 4; + std::vector padded(static_cast(pw) * (height + 2), 0); + std::vector out(static_cast(width) * height); + for (int y = 0; y < height; ++y) { + std::size_t row = static_cast(y + 2) * pw + 2; + for (int x = 0; x < width; ++x) { + std::size_t pos = row + x, prior = pos - pw, earlier = pos - 2 * pw; + unsigned ctx = padded[pos - 1] | (padded[pos - 2] << 1) | (padded[prior - 2] << 2) | + (padded[prior - 1] << 3) | (padded[prior] << 4) | (padded[prior + 1] << 5) | + (padded[prior + 2] << 6) | (padded[earlier - 2] << 7) | + (padded[earlier - 1] << 8) | (padded[earlier] << 9) | + (padded[earlier + 1] << 10) | (padded[earlier + 2] << 11); + std::size_t off = static_cast(ctx) << 1; + std::uint32_t zero = counts[off], one = counts[off + 1]; + std::uint32_t v = dec.freq(zero + one); + int bit = v < zero ? 0 : 1; + if (bit == 0) dec.decode(0, zero); else dec.decode(zero, one); + padded[pos] = static_cast(bit); + out[static_cast(y) * width + x] = static_cast(bit); + counts[off + bit] = static_cast(counts[off + bit] + 14); + if (counts[off] + counts[off + 1] >= 4096) { + counts[off] = static_cast((counts[off] >> 1) ? (counts[off] >> 1) : 1); + counts[off + 1] = static_cast((counts[off + 1] >> 1) ? (counts[off + 1] >> 1) : 1); + } + } + } + return out; +} + +int paeth(int l, int u, int ul) { + int est = l + u - ul; + int dl = std::abs(est - l), du = std::abs(est - u), dul = std::abs(est - ul); + if (dl <= du && dl <= dul) return l; + return du <= dul ? u : ul; +} + +int predict(const std::vector& out, std::size_t off, int x, int y, int mode, int w) { + if (mode < 0 || mode >= 5) bad("invalid predictor"); + int left = x > 0 ? out[off - 1] : 0; + int up = y > 0 ? out[off - w] : 0; + if (mode == 1) return up; + if (mode == 2) return left; + if (mode == 3) return (left + up) >> 1; + int ul = (x > 0 && y > 0) ? out[off - w - 1] : 0; + if (mode == 0) return paeth(left, up, ul); + if (ul >= std::max(left, up)) return std::min(left, up); + if (ul <= std::min(left, up)) return std::max(left, up); + return left + up - ul; +} + +int zero_ctx(const std::vector& r, std::size_t off, int x, int y, int w) { + int left = x > 0 ? (r[off - 1] == 0) : 1; + int up = y > 0 ? (r[off - w] == 0) : 1; + int ul = (x > 0 && y > 0) ? (r[off - w - 1] == 0) : 1; + int ur = (y > 0 && x < w - 1) ? (r[off - w + 1] == 0) : 1; + return left | (up << 1) | (ul << 2) | (ur << 3); +} + +std::vector decode_lossless(const std::string& payload, int width, int height) { + std::vector packed = unpack(payload.substr(1)); + if (packed.size() < 5) bad("lossless payload is truncated"); + int selector = packed[0]; + RangeDecoder dec(packed.data() + 1, packed.size() - 1); + const std::size_t length = static_cast(width) * height; + std::vector out(length), resid(length); + std::vector zero_counts(32, 1); + std::vector order_one(256 * 256, 1); + std::vector order_one_totals(256, 256); + std::vector order_zero(256, 2); + std::int64_t order_zero_total = 512; + std::map> order_two; + int previous = 0; + for (std::size_t off = 0; off < length; ++off) { + int x = static_cast(off % width), y = static_cast(off / width); + std::size_t zo = static_cast(zero_ctx(resid, off, x, y, width)) << 1; + std::uint32_t zero = zero_counts[zo], nonzero = zero_counts[zo + 1]; + std::uint32_t zf = dec.freq(zero + nonzero); + int bit = zf < zero ? 0 : 1; + dec.decode(bit == 0 ? 0 : zero, bit == 0 ? zero : nonzero); + zero_counts[zo + bit] += 32; + if (zero_counts[zo] + zero_counts[zo + 1] >= 16384) { + zero_counts[zo] = (zero_counts[zo] >> 1) ? (zero_counts[zo] >> 1) : 1; + zero_counts[zo + 1] = (zero_counts[zo + 1] >> 1) ? (zero_counts[zo + 1] >> 1) : 1; + } + int symbol = 0; + if (bit == 1) { + int above = off >= static_cast(width) ? resid[off - width] : 0; + auto it = order_two.find((previous << 8) | above); + if (it == order_two.end()) { + std::array z{}; + it = order_two.emplace((previous << 8) | above, z).first; + } + auto& second = it->second; + std::size_t fo = static_cast(previous) << 8; + std::int64_t excluded = second[0] + order_one[fo] + order_zero[0]; + std::int64_t total = second[256] + order_one_totals[previous] + order_zero_total - excluded; + if (total <= 0) bad("lossless payload is malformed"); + std::uint32_t target = dec.freq(static_cast(total)); + std::int64_t cum = 0; + symbol = 1; + std::int64_t f = second[1] + order_one[fo + 1] + order_zero[1]; + while (cum + f <= target) { + cum += f; + ++symbol; + if (symbol >= 256) bad("lossless payload is malformed"); + f = second[symbol] + order_one[fo + symbol] + order_zero[symbol]; + } + dec.decode(static_cast(cum), static_cast(f)); + second[symbol] = static_cast(second[symbol] + 14 * 44); + second[256] = static_cast(second[256] + 14 * 44); + if (second[256] >= 14 * 3584) { + std::uint32_t t = 0; + for (int v = 0; v < 256; ++v) { second[v] >>= 1; t += second[v]; } + second[256] = static_cast(t); + } + order_one[fo + symbol] = static_cast(order_one[fo + symbol] + 56); + order_one_totals[previous] += 56; + if (order_one_totals[previous] >= 8192) { + std::int32_t t = 0; + for (int v = 0; v < 256; ++v) { + std::uint16_t s = static_cast((order_one[fo + v] >> 1) ? (order_one[fo + v] >> 1) : 1); + order_one[fo + v] = s; + t += s; + } + order_one_totals[previous] = t; + } + order_zero[symbol] = static_cast(order_zero[symbol] + 32); + order_zero_total += 32; + if (order_zero_total >= 4096) { + std::int64_t t = 0; + for (int v = 0; v < 256; ++v) { order_zero[v] >>= 1; t += order_zero[v]; } + order_zero_total = t; + } + } + resid[off] = static_cast(symbol); + int delta = (symbol & 1) ? -((symbol + 1) >> 1) : (symbol >> 1); + out[off] = static_cast((predict(out, off, x, y, selector, width) + delta) & 0xFF); + previous = symbol; + } + for (std::uint8_t& v : out) v = v >= 129 ? 1 : 0; + return out; +} + +} // namespace + +std::vector SamMask::decode() const { + if (width < 1 || height < 1 || + static_cast(width) * static_cast(height) > MetaCaps::kMaxMaskPixels) + bad("mask dimensions are out of range"); + if (payload.empty() || payload[0] != encoding) bad("payload does not start with its marker"); + if (encoding == '!') return decode_one_bit(payload, width, height); + if (encoding == '~') return decode_lossless(payload, width, height); + bad("unsupported encoding"); +} + +} // namespace molo::meta diff --git a/src/meta/schema_check.cpp b/src/meta/schema_check.cpp new file mode 100644 index 0000000..363240c --- /dev/null +++ b/src/meta/schema_check.cpp @@ -0,0 +1,107 @@ +#include "meta/schema_check.h" + +#include + +#include "meta/check.h" + +namespace molo::meta { + +namespace { + +struct Walker { + const Json& root; + bool strict; + Walker(const Json& r, bool s) : root(r), strict(s) {} + long properties = 0; + long chars = 0; + long enums = 0; + std::set resolving; // $ref targets on the current path + + const Json* resolve(const std::string& ref) const { + for (const char* prefix : {"#/$defs/", "#/definitions/"}) { + std::string p = prefix; + if (ref.rfind(p, 0) == 0) { + const char* key = p == "#/$defs/" ? "$defs" : "definitions"; + const Json* defs = JsonX::obj(root, key); + return defs ? defs->find(ref.substr(p.size())) : nullptr; + } + } + return nullptr; + } + + void walk(const Json& node, int depth) { + if (depth > SchemaCheck::kMaxDepth) Check::fail("schema nests deeper than 10 levels"); + if (!node.is_object()) return; + std::string ref = JsonX::str(node, "$ref"); + if (!ref.empty()) { + if (resolving.count(ref)) Check::fail("recursive schemas are not supported"); + const Json* target = resolve(ref); + if (target) { + resolving.insert(ref); + walk(*target, depth + 1); + resolving.erase(ref); + } + } + if (strict) { + if (node.has("allOf") || node.has("oneOf")) + Check::fail("strict schemas cannot use allOf or oneOf"); + } + if (const Json* e = JsonX::arr(node, "enum")) { + enums += static_cast(e->as_array().size()); + for (const Json& v : e->as_array()) + if (v.is_string()) chars += static_cast(v.as_string().size()); + } + if (const Json* c = JsonX::member(node, "const"); c && c->is_string()) + chars += static_cast(c->as_string().size()); + if (const Json* props = JsonX::obj(node, "properties")) { + properties += static_cast(props->as_object().size()); + if (strict) { + if (!(node.has("additionalProperties") && + !JsonX::flag(node, "additionalProperties", true) && + node.find("additionalProperties")->is_bool())) + Check::fail("strict schemas need additionalProperties:false on every object"); + const Json* req = JsonX::arr(node, "required"); + std::set required; + if (req) + for (const Json& r : req->as_array()) + if (r.is_string()) required.insert(r.as_string()); + for (const auto& [k, v] : props->as_object()) + if (!required.count(k)) Check::fail("strict schemas must list every property in required"); + } + for (const auto& [k, v] : props->as_object()) { + chars += static_cast(k.size()); + walk(v, depth + 1); + } + } + for (const char* key : {"items", "additionalProperties", "not"}) + if (const Json* v = JsonX::member(node, key); v && v->is_object()) walk(*v, depth + 1); + for (const char* key : {"anyOf", "oneOf", "allOf", "prefixItems"}) + if (const Json* a = JsonX::arr(node, key)) + for (const Json& v : a->as_array()) walk(v, depth + 1); + for (const char* key : {"$defs", "definitions"}) + if (const Json* d = JsonX::obj(node, key)) + for (const auto& [k, v] : d->as_object()) { + chars += static_cast(k.size()); + if (&node != &root) walk(v, depth + 1); + } + } +}; + +} // namespace + +void SchemaCheck::check(const Json& schema, bool strict) { + if (!schema.is_object()) Check::fail("schema must be a JSON object"); + if (strict) { + std::string type = JsonX::str(schema, "type"); + if (type != "object") Check::fail("strict schemas need a plain object root"); + for (const char* k : {"anyOf", "oneOf", "allOf", "enum", "not"}) + if (schema.has(k)) Check::fail(std::string("strict schema root cannot use ") + k); + } + Walker w(schema, strict); + w.walk(schema, 1); + if (w.properties > kMaxProperties) Check::fail("schema has more than 5000 properties"); + if (w.chars > kMaxStringChars) Check::fail("schema names and enum values exceed 120000 chars"); + if (w.enums > kMaxEnumValues) Check::fail("schema has more than 1000 enum values"); +} + +} // namespace molo::meta diff --git a/src/meta/schema_check.h b/src/meta/schema_check.h new file mode 100644 index 0000000..019f3df --- /dev/null +++ b/src/meta/schema_check.h @@ -0,0 +1,27 @@ +#pragma once + +// Local pre-flight for JSON Schemas sent as structured output (response_format +// / text.format) or function-tool parameters. Limits and strict-mode rules are +// the ones stated in docs/structured-output: +// nesting depth <= 10, total properties <= 5000, total name/enum/const +// string length <= 120000, enum values <= 1000, no recursive ($ref cycle) +// schemas; strict: plain object root, additionalProperties:false on every +// object, required lists every property, no allOf/oneOf. +// The server stays the authority; this only rejects what it documents it +// rejects, so a bad schema fails before it costs a round trip. + +#include "meta/jsonx.h" + +namespace molo::meta { + +class SchemaCheck { + public: + static constexpr int kMaxDepth = 10; + static constexpr long kMaxProperties = 5000; + static constexpr long kMaxStringChars = 120000; + static constexpr long kMaxEnumValues = 1000; + // Throws llm::ApiError::invalid on a violation. + static void check(const Json& schema, bool strict); +}; + +} // namespace molo::meta diff --git a/src/meta/speech.cpp b/src/meta/speech.cpp new file mode 100644 index 0000000..2e07aae --- /dev/null +++ b/src/meta/speech.cpp @@ -0,0 +1,316 @@ +#include "meta/speech.h" + +#include + +#include "google/live/ws_error.h" +#include "google/live/ws_url.h" +#include "meta/calls.h" +#include "meta/caps.h" +#include "meta/check.h" +#include "meta/client.h" + +namespace molo::meta { + +namespace { +long le32(const std::string& s, std::size_t p) { + return static_cast(static_cast(s[p])) | + (static_cast(static_cast(s[p + 1])) << 8) | + (static_cast(static_cast(s[p + 2])) << 16) | + (static_cast(static_cast(s[p + 3])) << 24); +} +int le16(const std::string& s, std::size_t p) { + return static_cast(s[p]) | (static_cast(s[p + 1]) << 8); +} +long now_ms() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count(); +} +} // namespace + +void SpeechOptions::validate() const { + Check::model_id(model); + if (!mode.empty()) Check::one_of("mode", mode, {"PUSH_TO_TALK", "ENDPOINTING", "DIARIZATION"}); + if (!partial_mode.empty()) Check::one_of("partialMode", partial_mode, {"CUMULATIVE", "DELTA"}); + if (keywords.size() > MetaCaps::kMaxKeywords) Check::fail("too many keywords"); + for (const std::string& k : keywords) { + Check::non_empty("keyword", k); + Check::max_bytes("keyword", k, 256); + } + if (language_bias.size() > 25) Check::fail("languageBias takes at most 25 languages"); + for (const std::string& l : language_bias) { + Check::non_empty("language", l); + Check::max_bytes("language", l, 64); + } + if (!session_id.empty()) Check::segment("sessionId", session_id); +} + +Json SpeechOptions::settings_json(const std::string& audio_encoding) const { + Json j = Json::object(); + j.set("model", Json(model)); + j.set("audioEncoding", Json(audio_encoding)); + JsonX::set_if(j, "mode", mode); + JsonX::set_if(j, "partialMode", partial_mode); + if (!keywords.empty()) j.set("keywords", JsonX::strings(keywords)); + if (!language_bias.empty()) j.set("languageBias", JsonX::strings(language_bias)); + if (emit_audio_progress) j.set("emitAudioProgress", Json(true)); + return j; +} + +WavInfo WavInfo::parse(const std::string& w) { + if (w.size() < 44 || w.compare(0, 4, "RIFF") != 0 || w.compare(8, 4, "WAVE") != 0) + Check::fail("audio is not a RIFF/WAVE file"); + if (w.size() > MetaCaps::kMaxTranscribeBytes) Check::fail("audio is over the 32 MB request limit"); + WavInfo info; + bool have_fmt = false, have_data = false; + std::size_t pos = 12; + while (pos + 8 <= w.size()) { + std::string id = w.substr(pos, 4); + long size = le32(w, pos + 4); + std::size_t body = pos + 8; + if (id == "fmt " && body + 16 <= w.size()) { + int format = le16(w, body); + info.channels = le16(w, body + 2); + info.sample_rate = static_cast(le32(w, body + 4)); + info.bits = le16(w, body + 14); + if (format != 1) Check::fail("WAV must be integer PCM"); + have_fmt = true; + } else if (id == "data") { + long avail = static_cast(w.size() - body); + info.data_bytes = size > avail || size < 0 ? avail : size; + have_data = true; + break; + } + if (size < 0) break; + pos = body + static_cast(size) + (static_cast(size) & 1u); + } + if (!have_fmt || !have_data) Check::fail("WAV is missing its fmt or data chunk"); + if (info.channels != 1 || info.bits != 16) + Check::fail("WAV must be mono 16-bit PCM"); + if (info.sample_rate != 16000 && info.sample_rate != 24000) + Check::fail("WAV must be 16 kHz or 24 kHz"); + if (info.data_bytes <= 0) Check::fail("WAV holds no audio"); + info.duration_ms = info.data_bytes * 1000L / (static_cast(info.sample_rate) * 2L); + if (info.duration_ms > MetaCaps::kMaxTranscribeMs) Check::fail("audio is longer than 10 minutes"); + return info; +} + +TranscribeResult TranscribeResult::from_json(const Json& j) { + TranscribeResult r; + r.session_id = JsonX::str(j, "sessionId"); + r.transcript = JsonX::str(j, "transcript"); + r.audio_duration_ms = JsonX::num(j, "audioDurationMs"); + if (const Json* t = JsonX::arr(j, "turns")) + for (const Json& x : t->as_array()) { + SpeechTurn turn; + turn.turn_id = JsonX::num(x, "turnId"); + turn.start_ms = JsonX::num(x, "startMs"); + turn.end_ms = JsonX::num(x, "endMs"); + turn.transcript = JsonX::str(x, "transcript"); + turn.speaker = JsonX::str(x, "speaker"); + r.turns.push_back(std::move(turn)); + } + return r; +} + +const std::vector& SpeechEvent::known_types() { + static const std::vector v = {"transcript", "speechStart", "speechEnd", + "speechComplete", "speaker", "audioProgress", "error"}; + return v; +} + +SpeechEvent SpeechEvent::parse(const Json& j) { + SpeechEvent e; + e.type = JsonX::str(j, "type"); + e.transcript = JsonX::str(j, "transcript"); + e.final_flag = JsonX::flag(j, "final"); + e.audio_processed_ms = JsonX::num(j, "audioProcessedMs"); + e.turn_id = JsonX::num(j, "turnId", -1); + e.speaker_label = JsonX::str(j, "label"); + e.session_id = JsonX::str(j, "sessionId"); + e.message = JsonX::str(j, "message"); + e.error_type = JsonX::str(j, "errorType"); + e.error_code = JsonX::str(j, "errorCode"); + return e; +} + +llm::RawResponse TranscribeService::post(const SpeechOptions& options, const std::string& wav, + const char* accept) { + options.validate(); + WavInfo::parse(wav); + // `request` is a form field with a JSON content type and no filename (the + // docs' curl: -F 'request={...};type=application/json'); audio is the file. + const std::string boundary = clients::body::MultipartBody().boundary(); + std::string body; + body += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"request\"\r\n"; + body += "Content-Type: application/json\r\n\r\n" + options.settings_json("WAV").dump() + "\r\n"; + body += "--" + boundary + "\r\nContent-Disposition: form-data; name=\"audio\"; filename=\"audio.wav\"\r\n"; + body += "Content-Type: audio/wav\r\n\r\n" + wav + "\r\n--" + boundary + "--\r\n"; + std::string path = path_for(Call::kAsrTranscribe); + if (!options.session_id.empty()) path += "?sessionId=" + options.session_id; + return client_->request_raw(clients::Method::kPost, path, &body, "multipart/form-data; boundary=" + boundary, + {{"Accept", accept}}); +} + +TranscribeResult TranscribeService::transcribe(const SpeechOptions& options, + const std::string& wav) { + llm::RawResponse raw = post(options, wav, "application/json"); + try { + return TranscribeResult::from_json(Json::parse(raw.body)); + } catch (const std::exception& e) { + throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what()); + } +} + +std::vector TranscribeService::transcribe_events(const SpeechOptions& options, + const std::string& wav) { + llm::RawResponse raw = post(options, wav, "text/event-stream"); + std::vector out; + llm::SseDecoder decoder("meta", [&](const llm::SseEvent& e) { + if (out.size() >= MetaCaps::kMaxRealtimeEvents) return false; + SpeechEvent ev = SpeechEvent::parse(e.data); + if (ev.type.empty()) ev.type = e.event; + out.push_back(std::move(ev)); + return true; + }); + decoder.feed(raw.body); + decoder.finish(); + return out; +} + +std::string TranscribeService::transcribe_text(const SpeechOptions& options, + const std::string& wav) { + return post(options, wav, "text/plain").body; +} + +void RealtimeConfig::validate() const { + options.validate(); + Check::one_of("audioEncoding", audio_encoding, {"PCM_16KHZ", "PCM_24KHZ"}); +} + +RealtimeSession::RealtimeSession(RealtimeConfig config) : config_(std::move(config)) { + if (config_.url.empty()) config_.url = MetaEndpoints::kRealtimeUrl; + if (config_.key.empty()) config_.key = MetaClient::key_from_env(); +} + +RealtimeSession::~RealtimeSession() { + try { + close(); + } catch (...) { + } +} + +Json RealtimeSession::handshake_json(bool reveal) const { + Json j = config_.options.settings_json(config_.audio_encoding); + Json auth = Json::object(); + auth.set("accessToken", Json("Bearer " + (reveal ? config_.key : std::string("***")))); + j.set("authorization", auth); + return j; +} + +void RealtimeSession::connect() { + config_.validate(); + if (open_) Check::fail("session already connected"); + if (config_.key.empty()) throw llm::NotConfiguredError("meta", MetaEndpoints::kKeyEnv); + using namespace google::live; + try { + WsUrl url = WsUrl::parse(config_.url); + if (!config_.options.session_id.empty()) + url.target += (url.target.find('?') == std::string::npos ? "?" : "&") + + std::string("sessionId=") + config_.options.session_id; + ws_ = std::make_unique( + open_stream(url, config_.connect_timeout_ms, MetaCaps::kRealtimeIdleMs, config_.tls)); + ws_->handshake(url, {}, config_.handshake_timeout_ms); // no Authorization header + ws_->send_text(handshake_json(true).dump()); + ws_->set_read_timeout_ms(250); + long deadline = now_ms() + config_.ack_timeout_ms; + while (session_id_.empty()) { + if (now_ms() > deadline) throw llm::ApiError("meta", 504, "TIMEOUT", "handshake acknowledgement not received"); + auto m = ws_->receive(); + if (!m) continue; + if (m->kind == WsMessage::Kind::Close) { + closed_ = true; + close_code_ = m->code; + throw llm::ApiError("meta", m->code == 1013 ? 429 : 400, "REALTIME_CLOSED", + "closed before the acknowledgement (code " + std::to_string(m->code) + + "): " + m->data.substr(0, 300)); + } + Json j = Json::parse(m->data); + if (JsonX::str(j, "type") == "error") + throw llm::ApiError("meta", 400, JsonX::str(j, "errorType", "error"), JsonX::str(j, "message")); + session_id_ = JsonX::str(j, "sessionId"); + if (session_id_.empty()) throw llm::ApiError("meta", 502, "BAD_RESPONSE", "acknowledgement has no sessionId"); + } + open_ = true; + } catch (const WsError& e) { + if (ws_) ws_->shutdown(); + ws_.reset(); + throw llm::ApiError("meta", e.http_status() ? e.http_status() : 502, "UNAVAILABLE", e.what()); + } +} + +void RealtimeSession::send_audio(const std::string& pcm) { + if (!open_) Check::fail("session is not open"); + if (pcm.empty() || pcm.size() > MetaCaps::kMaxRealtimeAudioFrame) + Check::fail("audio frame must be 1.." + std::to_string(MetaCaps::kMaxRealtimeAudioFrame) + " bytes"); + if (pcm.size() % 2 != 0) Check::fail("PCM frames are 16-bit samples"); + try { + ws_->send_binary(pcm); + } catch (const google::live::WsError& e) { + throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what()); + } + audio_bytes_ += pcm.size(); +} + +void RealtimeSession::end_stream() { + if (!open_) Check::fail("session is not open"); + Json j = Json::object(); + j.set("type", Json("endStream")); + try { + ws_->send_text(j.dump()); + } catch (const google::live::WsError& e) { + throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what()); + } +} + +std::optional RealtimeSession::next(int timeout_ms) { + if (!ws_ || closed_) return std::nullopt; + ws_->set_read_timeout_ms(timeout_ms < 1 ? 1 : timeout_ms); + try { + auto m = ws_->receive(); + if (!m) return std::nullopt; + if (m->kind == google::live::WsMessage::Kind::Close) { + closed_ = true; + close_code_ = m->code; + open_ = false; + SpeechEvent e; + e.type = "close"; + e.message = m->data; + e.audio_processed_ms = m->code; + return e; + } + if (++events_ > MetaCaps::kMaxRealtimeEvents) { + close(); + throw llm::ApiError("meta", 502, "TOO_MANY_EVENTS", "event cap reached"); + } + return SpeechEvent::parse(Json::parse(m->data)); + } catch (const google::live::WsError& e) { + throw llm::ApiError("meta", 502, "UNAVAILABLE", e.what()); + } catch (const clients::body::JsonError& e) { + throw llm::ApiError("meta", 502, "BAD_RESPONSE", e.what()); + } +} + +void RealtimeSession::close() { + if (ws_) { + try { + ws_->close(1000, ""); + } catch (...) { + } + ws_->shutdown(); + ws_.reset(); + } + open_ = false; + closed_ = true; +} + +} // namespace molo::meta diff --git a/src/meta/speech.h b/src/meta/speech.h new file mode 100644 index 0000000..0e50255 --- /dev/null +++ b/src/meta/speech.h @@ -0,0 +1,158 @@ +#pragma once + +// Muse Voice Transcribe (speech to text; the model does no synthesis). +// POST /asr/transcribe multipart: `request` (JSON settings) + `audio` (WAV: +// mono 16-bit PCM, 16 or 24 kHz, <= 10 min, <= 32 MB); +// Accept picks JSON, text/event-stream or text/plain. +// WS /asr/realtime first frame (within 10 s) is the JSON handshake with +// the credential inside it ({"authorization":{"accessToken": +// "Bearer "}}), then raw PCM binary frames, then +// {"type":"endStream"}; close 1000 normal, 1008 invalid, +// 1011 backend, 1013 rate limited. +// Modes: PUSH_TO_TALK (default), ENDPOINTING, DIARIZATION. 25 languages, +// keyword and language biasing, turn-level timestamps only. + +#include +#include +#include +#include +#include +#include + +#include "google/live/ws_client.h" +#include "google/live/ws_stream.h" +#include "llm/api_client.h" +#include "meta/jsonx.h" + +namespace molo::meta { + +struct SpeechOptions { + std::string model = "muse-voice-transcribe-1.0"; + std::string mode; // "" (server default PUSH_TO_TALK) | PUSH_TO_TALK | ENDPOINTING | DIARIZATION + std::string partial_mode; // "" | CUMULATIVE | DELTA + std::vector keywords; + std::vector language_bias; + bool emit_audio_progress = false; + std::string session_id; // correlation id, query parameter + + void validate() const; + // The `request` JSON part / handshake settings. audio_encoding: WAV for the + // file endpoint, PCM_16KHZ or PCM_24KHZ for the socket. + Json settings_json(const std::string& audio_encoding) const; +}; + +struct WavInfo { + int channels = 0; + int sample_rate = 0; + int bits = 0; + long data_bytes = 0; + long duration_ms = 0; + // Throws ApiError::invalid unless it is RIFF/WAVE PCM mono 16-bit at 16 or + // 24 kHz, at most 10 minutes. + static WavInfo parse(const std::string& wav); +}; + +struct SpeechTurn { + long turn_id = 0; + long start_ms = 0; + long end_ms = 0; + std::string transcript; + std::string speaker; +}; + +struct TranscribeResult { + std::string session_id; + std::string transcript; + long audio_duration_ms = 0; + std::vector turns; + static TranscribeResult from_json(const Json& j); +}; + +// One server message from the event stream or the socket (ServerMessage union: +// transcript, speechStart, speechEnd, speechComplete, speaker, audioProgress, +// error). The socket's handshake acknowledgement has no `type`, only sessionId. +struct SpeechEvent { + std::string type; + std::string transcript; + bool final_flag = false; + long audio_processed_ms = 0; + long turn_id = -1; + std::string speaker_label; + std::string session_id; + std::string message; // error + std::string error_type; + std::string error_code; + static SpeechEvent parse(const Json& j); + static const std::vector& known_types(); +}; + +enum class TranscribeAccept { kJson, kEvents, kText }; + +class TranscribeService : public llm::Service { + public: + explicit TranscribeService(std::shared_ptr client) + : llm::Service(std::move(client)) {} + + // Validates the WAV and options locally (WavInfo::parse) before upload. + TranscribeResult transcribe(const SpeechOptions& options, const std::string& wav); + // text/event-stream: events in order, ends after the last event. + std::vector transcribe_events(const SpeechOptions& options, const std::string& wav); + // text/plain: one turn per line (A:Hello style in DIARIZATION). + std::string transcribe_text(const SpeechOptions& options, const std::string& wav); + + private: + llm::RawResponse post(const SpeechOptions& options, const std::string& wav, const char* accept); +}; + +struct RealtimeConfig { + SpeechOptions options; + std::string audio_encoding = "PCM_16KHZ"; // PCM_16KHZ | PCM_24KHZ + std::string url; // default MetaEndpoints::kRealtimeUrl + std::string key; // default MetaClient::key_from_env() + int connect_timeout_ms = 10000; + int handshake_timeout_ms = 10000; + int ack_timeout_ms = 10000; + google::live::TlsFactory tls; // tests inject a loopback TLS-less factory + void validate() const; +}; + +class RealtimeSession { + public: + explicit RealtimeSession(RealtimeConfig config); + ~RealtimeSession(); + RealtimeSession(const RealtimeSession&) = delete; + RealtimeSession& operator=(const RealtimeSession&) = delete; + + // Opens the socket, sends the handshake frame, waits for the sessionId + // acknowledgement. Throws ApiError (503 not configured without a key). + void connect(); + const std::string& session_id() const { return session_id_; } + bool open() const { return open_; } + + // Raw PCM (16-bit LE mono at the configured rate), at most 1 MiB per call. + void send_audio(const std::string& pcm); + void end_stream(); // {"type":"endStream"}: keep reading until the close + // Next server event, nullopt on read timeout; sets closed()/close_code() on a + // Close frame (event type "close"). + std::optional next(int timeout_ms); + bool closed() const { return closed_; } + int close_code() const { return close_code_; } + void close(); + std::size_t events_seen() const { return events_; } + + // The handshake frame without sending it (tests, demo dry run). The access + // token is redacted unless `reveal` is true. + Json handshake_json(bool reveal) const; + + private: + RealtimeConfig config_; + std::unique_ptr ws_; + std::string session_id_; + bool open_ = false; + bool closed_ = false; + int close_code_ = 0; + std::size_t events_ = 0; + std::size_t audio_bytes_ = 0; +}; + +} // namespace molo::meta diff --git a/src/meta/tools/check_spec.py b/src/meta/tools/check_spec.py new file mode 100644 index 0000000..28738e1 --- /dev/null +++ b/src/meta/tools/check_spec.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Checks src/meta against the published Meta Model API docs (python3 stdlib only). + +Meta publishes no OpenAPI document. The docs site serves every page as +Markdown (https://dev.meta.ai/llms.txt lists them) and the API reference has +one schemas page per resource with machine-regular field tables. This script +checks what is checkable: + + 1. every row of the call table: the operation page exists, carries the + "## METHOD /path" heading, lists each declared query parameter; the + schemas page has the request schema with every declared body field and + the response schema; + 2. no operation heading in the docs (all api-reference pages) is missing + from the table; + 3. the Responses stream event names equal the ResponseStreamEvent union; + 4. every hosted model id in the catalog appears on the models page and + every token price on the pricing page; + 5. the Muse Code protocol vocabulary (methods, notifications, server + requests, error codes) equals the committed MSP stable schema of the + muse-code-sdk mirror. + +Usage: check_spec.py [--cache DIR] (--cache reads/writes fetched pages) +Exit 0 only when everything matches. +""" +import json +import os +import re +import sys +import urllib.request + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +SRC = os.path.join(ROOT, "src", "meta") +DOCS = "https://dev.meta.ai/" +RAW = "https://raw.git" + "hubusercontent.com/meta-models/muse-code-sdk/HEAD/schema/msp/stable/msp.schema.json" +CACHE = None +failures = [] + + +def fail(msg): + failures.append(msg) + print("FAIL:", msg) + + +def fetch(url): + if CACHE: + path = os.path.join(CACHE, re.sub(r"[^A-Za-z0-9._-]", "_", url)) + if os.path.exists(path): + return open(path, encoding="utf-8").read() + last = None + for _ in range(3): + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "molo-check-spec"}), timeout=30) as r: + text = r.read().decode("utf-8") + if CACHE: + os.makedirs(CACHE, exist_ok=True) + open(os.path.join(CACHE, re.sub(r"[^A-Za-z0-9._-]", "_", url)), "w", encoding="utf-8").write(text) + return text + except Exception as e: # noqa: BLE001 + last = e + raise SystemExit("cannot fetch %s: %s" % (url, last)) + + +def block(text, begin, end): + m = re.search(begin + r"[^\n]*\n(.*?)" + end, text, re.S) + if not m: + raise SystemExit("marker %s not found" % begin) + return m.group(1) + + +def split_camel(name): + return re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", name) + + +def schema_sections(page): + out = {} + for part in re.split(r"\n### ", page)[1:]: + title, _, body = part.partition("\n") + fields = set(re.findall(r"^\| `([^`]+)`", body, re.M)) + out[title.strip()] = fields + return out + + +def main(): + global CACHE + if "--cache" in sys.argv: + CACHE = sys.argv[sys.argv.index("--cache") + 1] + calls_src = open(os.path.join(SRC, "calls.cpp"), encoding="utf-8").read() + rows = re.findall(r'\{Call::(\w+), "(\w+)", "([^"]+)", "([^"]+)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)", "([^"]*)"\},', + block(calls_src, "CALLS-BEGIN", "// CALLS-END")) + if not rows: + raise SystemExit("no call rows parsed") + llms = fetch(DOCS + "llms.txt") + pages = set(re.findall(r"https://dev\.meta\.ai/(docs/[^\s)]+\.md)", llms)) + schema_cache = {} + declared = set() + + for name, method, path, op, group, req, resp, body_fields, query in rows: + declared.add((method, path)) + url = "docs/api-reference/%s.md" % op + if url not in pages: + fail("%s: page %s is not listed in llms.txt" % (name, url)) + continue + page = fetch(DOCS + url) + heading = "## %s %s" % (method, path.replace("_", r"\_")) + if heading not in page and ("## %s %s" % (method, path)) not in page: + fail("%s: heading '## %s %s' missing in %s" % (name, method, path, url)) + for q in filter(None, query.split(",")): + if "`%s`" % q not in page: + fail("%s: query parameter %s missing in %s" % (name, q, url)) + if group: + if group not in schema_cache: + schema_cache[group] = schema_sections(fetch(DOCS + "docs/api-reference/%s/schemas.md" % group)) + sections = schema_cache[group] + for schema, fields in ((req, body_fields), (resp, "")): + if not schema: + continue + title = split_camel(schema) + if title not in sections: + fail("%s: schema %s not in %s schemas page" % (name, title, group)) + continue + for f in filter(None, fields.split(",")): + if f not in sections[title]: + fail("%s: field %s missing in schema %s" % (name, f, title)) + # Multipart edit variant and the images edit alternative body. + img = schema_cache.get("images", {}) + for f in "image,prompt,model,n,size,response_format,reasoning_strength".split(","): + if f not in img.get("Create Image Edit Request", set()): + fail("images multipart edit: field %s missing in CreateImageEditRequest" % f) + + # 2. every documented operation is in the table (no exclusions needed). + for url in sorted(pages): + if not url.startswith("docs/api-reference/") or url.count("/") < 2: + continue + for m, p in re.findall(r"^## (GET|POST|DELETE|PUT|PATCH|WS) (/\S+)", fetch(DOCS + url), re.M): + p = p.replace("\\_", "_") + if (m, p) not in declared: + fail("documented operation %s %s (%s) is not in the call table" % (m, p, url)) + + # 3. stream events. + ev_src = open(os.path.join(SRC, "response_events.cpp"), encoding="utf-8").read() + ours = set(re.findall(r'"([\w.]+)",', block(ev_src, "EVENTS-BEGIN", "// EVENTS-END"))) + rpage = fetch(DOCS + "docs/api-reference/responses/schemas.md") + union = re.search(r"### Response Stream Event\n+Type: (.*?)\n", rpage).group(1) + theirs = set() + for n in re.findall(r"\[`(\w+)`\]", union): + sec = re.search(r"### %s\n(.*?)\n### " % re.escape(split_camel(n)), rpage, re.S) + t = re.search(r"`type`\s*\|\s*`enum \('([^']+)'\)", sec.group(1)) if sec else None + if t: + theirs.add(t.group(1)) + else: + fail("stream event schema %s has no type enum" % n) + for e in sorted(ours ^ theirs): + fail("stream event %s differs (ours=%s docs=%s)" % (e, e in ours, e in theirs)) + + # 4. catalog against models / pricing pages. + cat = open(os.path.join(SRC, "catalog.cpp"), encoding="utf-8").read() + models_page = fetch(DOCS + "docs/models.md") + pricing = fetch(DOCS + "docs/pricing-rate-limits.md") + for m in re.finditer(r'\{"([\w.-]+)", "[^"]+", ModelKind::\w+, (true|false), \w+, \w+, (\d+), ([\d.]+), ([\d.]+), ([\d.]+), "[^"]*", (\d+), (\d+),', block(cat, "CATALOG-BEGIN", "// CATALOG-END")): + mid, hosted, ctx, cached, inp, out, rpm, tpm = m.groups() + if "`%s`" % mid not in models_page and "`%s`" % mid not in pricing and mid not in ("muse-spark-open", "muse-glimmer"): + fail("catalog model %s not on the models or pricing page" % mid) + if hosted == "true" and float(inp) > 0: + for price in (cached, inp, out): + p = "$%.2f" % float(price) if float(price) >= 0.01 else "$%s" % price.rstrip("0") + if p not in pricing and ("$%s" % price) not in pricing: + fail("catalog price %s of %s not on the pricing page" % (p, mid)) + if hosted == "true" and int(rpm) > 0 and ("| %s" % format(int(rpm), ",")) not in pricing: + fail("catalog rpm %s of %s not on the pricing page" % (rpm, mid)) + + # 5. Muse Code protocol. + schema = json.loads(fetch(RAW)) + spec = open(os.path.join(SRC, "muse_code_spec.cpp"), encoding="utf-8").read() + + def names(tag): + return set(re.findall(r'"([^"]+)",', block(spec, "MSP-%s-BEGIN" % tag, "// MSP-%s-END" % tag))) + + for tag, key in (("METHODS", "methods"), ("NOTIFICATIONS", "notifications"), ("REQUESTS", "requests")): + for e in sorted(names(tag) ^ set(schema[key])): + fail("MSP %s entry %s differs from the stable schema" % (key, e)) + errs = set(re.findall(r'"(-?\d+):(\w+):[01?]"', block(spec, "MSP-ERRORS-BEGIN", "// MSP-ERRORS-END"))) + want = {(str(e["code"]), e["kind"]) for e in schema["errors"]} + for e in sorted(errs ^ want): + fail("MSP error %s differs from the stable schema" % (e,)) + + print("checked: %d calls, %d stream events, %d MSP methods, %d MSP notifications, %d docs pages listed" + % (len(rows), len(ours), len(schema["methods"]), len(schema["notifications"]), len(pages))) + if failures: + print("%d failure(s)" % len(failures)) + sys.exit(1) + print("OK") + + +if __name__ == "__main__": + main() diff --git a/src/meta/usage.cpp b/src/meta/usage.cpp new file mode 100644 index 0000000..ec40f29 --- /dev/null +++ b/src/meta/usage.cpp @@ -0,0 +1,79 @@ +#include "meta/usage.h" + +#include + +#include "meta/catalog.h" + +namespace molo::meta { + +MetaUsage MetaUsage::from_responses(const Json& u) { + MetaUsage m; + m.input_tokens = JsonX::num(u, "input_tokens"); + m.output_tokens = JsonX::num(u, "output_tokens"); + m.total_tokens = JsonX::num(u, "total_tokens", m.input_tokens + m.output_tokens); + m.cached_tokens = JsonX::path_num(u, "input_tokens_details.cached_tokens"); + m.reasoning_tokens = JsonX::path_num(u, "output_tokens_details.reasoning_tokens"); + m.images_processed = JsonX::num(u, "images_processed"); + m.video_frames_processed = JsonX::num(u, "video_frames_processed"); + return m; +} + +MetaUsage MetaUsage::from_chat(const Json& u) { + MetaUsage m; + m.input_tokens = JsonX::num(u, "prompt_tokens"); + m.output_tokens = JsonX::num(u, "completion_tokens"); + m.total_tokens = JsonX::num(u, "total_tokens", m.input_tokens + m.output_tokens); + m.cached_tokens = JsonX::path_num(u, "prompt_tokens_details.cached_tokens"); + m.reasoning_tokens = JsonX::path_num(u, "completion_tokens_details.reasoning_tokens"); + return m; +} + +MetaUsage MetaUsage::from_messages(const Json& u) { + MetaUsage m; + m.input_tokens = JsonX::num(u, "input_tokens"); + m.output_tokens = JsonX::num(u, "output_tokens"); + m.total_tokens = m.input_tokens + m.output_tokens; + m.cached_tokens = JsonX::num(u, "cache_read_input_tokens"); + m.reasoning_tokens = JsonX::path_num(u, "output_tokens_details.thinking_tokens"); + return m; +} + +llm::Usage MetaUsage::to_neutral() const { + llm::Usage u; + u.input_tokens = static_cast(input_tokens); + u.output_tokens = static_cast(output_tokens); + u.total_tokens = static_cast(total_tokens); + u.cached_tokens = static_cast(cached_tokens); + u.reasoning_tokens = static_cast(reasoning_tokens); + return u; +} + +double MetaUsage::cost_usd(const std::string& model) const { + const ModelInfo* info = ModelCatalog::find(model); + if (!info || info->kind != ModelKind::kText || info->input_per_m <= 0) return -1.0; + long fresh = input_tokens > cached_tokens ? input_tokens - cached_tokens : 0; + return (static_cast(cached_tokens) * info->cached_per_m + + static_cast(fresh) * info->input_per_m + + static_cast(output_tokens) * info->output_per_m) / 1e6; +} + +namespace { +long header_long(const clients::HeaderMap& h, const char* name) { + auto v = h.get(name); + if (!v || v->empty() || v->size() > 18) return -1; + for (char c : *v) + if (c < '0' || c > '9') return -1; + return std::strtol(v->c_str(), nullptr, 10); +} +} // namespace + +RateLimits RateLimits::from_headers(const clients::HeaderMap& h) { + RateLimits r; + r.limit_tokens = header_long(h, "x-ratelimit-limit-tokens"); + r.remaining_tokens = header_long(h, "x-ratelimit-remaining-tokens"); + r.limit_requests = header_long(h, "x-ratelimit-limit-requests"); + r.remaining_requests = header_long(h, "x-ratelimit-remaining-requests"); + return r; +} + +} // namespace molo::meta diff --git a/src/meta/usage.h b/src/meta/usage.h new file mode 100644 index 0000000..ae797f5 --- /dev/null +++ b/src/meta/usage.h @@ -0,0 +1,51 @@ +#pragma once + +// Usage accounting across the three text endpoints, the image usage object, +// the SAM extras, and the x-ratelimit-* headers. Field names per docs +// (token-counting, pricing-rate-limits): +// Responses: input_tokens, output_tokens, total_tokens, +// input_tokens_details.cached_tokens, +// output_tokens_details.reasoning_tokens, +// video_frames_processed, images_processed +// Chat Completions: prompt_tokens, completion_tokens, total_tokens, +// prompt_tokens_details.cached_tokens, +// completion_tokens_details.reasoning_tokens +// Messages: input_tokens, output_tokens, cache_read_input_tokens, +// cache_creation_input_tokens, output_tokens_details.thinking_tokens + +#include + +#include "clients/response.h" +#include "llm/chat_types.h" +#include "meta/jsonx.h" + +namespace molo::meta { + +struct MetaUsage { + long input_tokens = 0; + long output_tokens = 0; + long total_tokens = 0; + long cached_tokens = 0; + long reasoning_tokens = 0; + long images_processed = 0; // SAM + long video_frames_processed = 0; // SAM, billed frames + + static MetaUsage from_responses(const Json& usage); + static MetaUsage from_chat(const Json& usage); + static MetaUsage from_messages(const Json& usage); + llm::Usage to_neutral() const; + // USD cost for a text model id from the static catalog; negative when the + // model has no token prices. + double cost_usd(const std::string& model) const; +}; + +struct RateLimits { + long limit_tokens = -1; + long remaining_tokens = -1; + long limit_requests = -1; + long remaining_requests = -1; + bool present() const { return limit_tokens >= 0 || limit_requests >= 0; } + static RateLimits from_headers(const clients::HeaderMap& headers); +}; + +} // namespace molo::meta diff --git a/tests/example/ai/meta/page_test.cpp b/tests/example/ai/meta/page_test.cpp new file mode 100644 index 0000000..1efe0e2 --- /dev/null +++ b/tests/example/ai/meta/page_test.cpp @@ -0,0 +1,31 @@ +// Routes: GET /ai/meta, /ai/meta/status, /ai/meta/calls (no credential, no upstream) +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + Fixture f; + auto page = f.client->get("/ai/meta"); + check(page.status == 200, "page 200"); + check(contains(page.headers.get("Content-Type"), "text/html"), "page is html"); + check(contains(page.body, "Meta Model API") && contains(page.body, "/embedded/ai_meta.js"), + "page names the product and loads its script"); + + auto status = f.client->get("/ai/meta/status"); + check(status.status == 200, "status 200"); + check(contains(status.headers.get("Content-Type"), "application/json"), "status is json"); + check(contains(status.body, "https://api.meta.ai/v1"), "base url shown"); + check(contains(status.body, "muse-spark-1.3") && contains(status.body, "sam-3.1"), "models listed"); + check(contains(status.body, "\"hosted\":false"), "open-weight models flagged as self-hosted"); + check(!contains(status.body, "LLM_"), "no credential material shown"); + + auto calls = f.client->get("/ai/meta/calls"); + check(calls.status == 200 && contains(calls.body, "/responses/{response_id}/cancel") && + contains(calls.body, "/asr/realtime"), + "call table listed"); + + check(f.client->post("/ai/meta").status == 405, "post to page is 405"); + check(f.client->post("/ai/meta/status").status == 405, "post to status is 405"); + check(f.client->get("/ai/meta/responses").status == 405, "get on a vendor route is 405"); + return finish("ai/meta/page"); +} diff --git a/tests/example/ai/meta/routes_test.cpp b/tests/example/ai/meta/routes_test.cpp new file mode 100644 index 0000000..929f407 --- /dev/null +++ b/tests/example/ai/meta/routes_test.cpp @@ -0,0 +1,63 @@ +// POST routes under /ai/meta: 422 on bad input, dry runs for paid/destructive +// calls, 503 not_configured without a credential. No vendor reply is scripted. +#include + +#include "support/Assert.hpp" + +using namespace molo::example::test; + +int main() { + unsetenv("META_API_KEY"); + unsetenv("MODEL_API_KEY"); + Fixture f; + auto post = [&](const std::string& path, const std::string& body) { return f.client->post(path, body); }; + + // 422 before any credential check. + check(post("/ai/meta/responses", "{}").status == 422, "responses: model required"); + check(post("/ai/meta/responses", "not json").status == 422, "responses: invalid JSON"); + check(post("/ai/meta/responses", R"({"model":"muse-spark-1.3","input":"x","max_output_tokens":15})").status == 422, + "responses: max_output_tokens below 16"); + check(post("/ai/meta/responses", R"({"model":"muse-spark-1.3","input":"x","reasoning_effort":"none"})").status == 422, + "responses: reasoning none rejected"); + check(post("/ai/meta/responses", R"({"model":"muse-spark-1.2","input":"x","reasoning_effort":"max"})").status == 422, + "responses: max effort needs wonder or spark 1.3"); + check(post("/ai/meta/chat", R"({"model":"muse-spark-1.3"})").status == 422, "chat: message required"); + check(post("/ai/meta/messages", R"({"model":"muse-spark-1.3","message":"hi"})").status == 422, "messages: max_tokens required"); + check(post("/ai/meta/responses/get", R"({"id":"../x"})").status == 422, "id path guard"); + check(post("/ai/meta/files", R"({"action":"explode"})").status == 422, "files: unknown action"); + check(post("/ai/meta/files", R"({"action":"get"})").status == 422, "files: id required"); + check(post("/ai/meta/images", R"({"prompt":"x","n":11})").status == 422, "images: n above 10"); + check(post("/ai/meta/images", R"({"prompt":"x","size":"3200x100"})").status == 422, "images: aspect over 16:1"); + check(post("/ai/meta/segment", R"({"prompt":"bus, bicycle","image_url":"https://e.x/a.jpg"})").status == 422, + "segment: one concept per request"); + check(post("/ai/meta/segment", R"({"prompt":"bus"})").status == 422, "segment: media required"); + check(post("/ai/meta/transcribe", R"({"audio_b64":"%%%"})").status == 422, "transcribe: bad base64"); + check(post("/ai/meta/transcribe", R"({"audio_b64":"AAAA"})").status == 422, "transcribe: not a WAV"); + check(post("/ai/meta/responses", std::string(9 * 1024 * 1024, 'x')).status == 413, "body cap"); + + // Dry runs need no credential. + auto img = post("/ai/meta/images", R"({"prompt":"a red fox"})"); + check(img.status == 200 && contains(img.body, "\"dry_run\":true"), "images: dry run without confirm"); + auto del = post("/ai/meta/responses/delete", R"({"id":"resp_1"})"); + check(del.status == 200 && contains(del.body, "\"dry_run\":true"), "delete response: dry run"); + auto fdel = post("/ai/meta/files", R"({"action":"delete","id":"file-1"})"); + check(fdel.status == 200 && contains(fdel.body, "\"dry_run\":true"), "delete file: dry run"); + auto seg = post("/ai/meta/segment", R"({"prompt":"yellow school bus","image_url":"https://e.x/a.jpg"})"); + check(seg.status == 200 && contains(seg.body, "\"dry_run\":true"), "segment: dry run"); + auto code = post("/ai/meta/code", "{}"); + check(code.status == 200 && contains(code.body, "muse serve"), "muse code: dry run names the command"); + + // 503 without a credential. + check(post("/ai/meta/responses", R"({"model":"muse-spark-1.3","input":"hi"})").status == 503, "responses: 503"); + check(contains(post("/ai/meta/responses", R"({"model":"muse-spark-1.3","input":"hi"})").body, "not_configured"), + "responses: error code not_configured"); + check(post("/ai/meta/responses/count", R"({"model":"muse-spark-1.3","input":"hi"})").status == 503, "count: 503"); + check(post("/ai/meta/chat", R"({"model":"muse-spark-1.3","message":"hi"})").status == 503, "chat: 503"); + check(post("/ai/meta/messages", R"({"model":"muse-spark-1.3","message":"hi","max_tokens":64})").status == 503, "messages: 503"); + check(post("/ai/meta/models", "{}").status == 503, "models: 503"); + check(post("/ai/meta/files", R"({"action":"list"})").status == 503, "files list: 503"); + check(post("/ai/meta/responses/get", R"({"id":"resp_1"})").status == 503, "retrieve: 503"); + check(post("/ai/meta/images", R"({"prompt":"x","confirm":true})").status == 503, "images confirmed: 503"); + check(post("/ai/meta/files", R"({"action":"delete","id":"file-1","confirm":true})").status == 503, "file delete confirmed: 503"); + return finish("ai/meta/routes"); +} diff --git a/tests/src/meta/test_meta_core.cpp b/tests/src/meta/test_meta_core.cpp new file mode 100644 index 0000000..9aac7af --- /dev/null +++ b/tests/src/meta/test_meta_core.cpp @@ -0,0 +1,443 @@ +// Meta Model API module: call table, catalog, validation, request bodies, +// fixtures taken from the vendor docs, SAM parsing and mask decoding, retry +// policy, usage. No vendor reply is scripted for verification: fixtures are +// the documented examples, labelled as such. +#include + +#include "framework.h" +#include "meta/calls.h" +#include "meta/catalog.h" +#include "meta/chat_completions.h" +#include "meta/chat_model.h" +#include "meta/check.h" +#include "meta/client.h" +#include "meta/images.h" +#include "meta/messages.h" +#include "meta/response_events.h" +#include "meta/response_input.h" +#include "meta/response_request.h" +#include "meta/sam.h" +#include "meta/schema_check.h" +#include "meta/speech.h" +#include "meta/usage.h" + +using namespace molo::meta; +using molo::llm::ApiError; + +namespace { +ResponseRequest basic() { + ResponseRequest r; + r.model = "muse-spark-1.3"; + r.input = Json("What is the capital of France?"); + return r; +} +template +bool rejects(F&& f) { + try { + f(); + } catch (const ApiError& e) { + return e.http_status() == 400; + } + return false; +} +} // namespace + +TEST(meta, call_table_shape) { + ASSERT_EQ(call_count(), 20u); + std::set seen; + for (std::size_t i = 0; i < call_count(); ++i) { + const CallSpec& s = spec_at(i); + ASSERT_TRUE(seen.insert(std::string(s.http_method) + " " + s.path).second); + ASSERT_TRUE(s.path[0] == '/'); + } + ASSERT_EQ(path_for(Call::kResponsesCancel, "resp_1"), "/responses/resp_1/cancel"); + ASSERT_EQ(path_for(Call::kFilesContent, "file-842549258569145"), "/files/file-842549258569145/content"); + ASSERT_TRUE(rejects([] { path_for(Call::kFilesRetrieve, "../x"); })); + ASSERT_TRUE(rejects([] { path_for(Call::kFilesRetrieve, ""); })); + ASSERT_TRUE(rejects([] { method_of(Call::kAsrRealtime); })); +} + +TEST(meta, catalog_matches_docs_tables) { + const ModelInfo* spark = ModelCatalog::find("muse-spark-1.3"); + ASSERT_TRUE(spark != nullptr); + ASSERT_EQ(spark->context_tokens, 1048576L); + ASSERT_TRUE(spark->input_per_m == 1.25 && spark->output_per_m == 4.25); + ASSERT_TRUE(ModelCatalog::accepts_max_effort("muse-wonder-1.0")); + ASSERT_FALSE(ModelCatalog::accepts_max_effort("muse-spark-1.3-contributor")); + ASSERT_FALSE(ModelCatalog::accepts_max_effort("muse-spark-1.2")); + ASSERT_FALSE(ModelCatalog::find("muse-glimmer")->hosted); + ASSERT_TRUE(ModelCatalog::find("nope") == nullptr); + // 1M cached + 1M fresh input + 1M output on muse-spark-1.3. + MetaUsage u; + u.input_tokens = 2000000; + u.cached_tokens = 1000000; + u.output_tokens = 1000000; + double c = u.cost_usd("muse-spark-1.3"); + ASSERT_TRUE(c > 5.6499 && c < 5.6501); + ASSERT_TRUE(u.cost_usd("sam-3.1") < 0); +} + +TEST(meta, response_request_rules) { + Json b = basic().to_json(); + ASSERT_EQ(b.find("model")->as_string(), "muse-spark-1.3"); + ASSERT_TRUE(b.find("stream") == nullptr); + ASSERT_TRUE(basic().to_json(true).find("stream")->as_bool()); + + ResponseRequest r = basic(); + r.max_output_tokens = 15; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r = basic(); + r.reasoning_effort = "none"; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r.reasoning_effort = "max"; + r.model = "muse-spark-1.2"; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r.model = "muse-wonder-1.0"; + r.validate(); + r = basic(); + r.background = true; + ASSERT_TRUE(rejects([&] { r.validate(true); })); + r = basic(); + r.include = {"reasoning.encrypted_content"}; + r.previous_response_id = "resp_abc123"; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r = basic(); + r.include = {"message.output_text.logprobs"}; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r = basic(); + r.truncation = "auto"; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r = basic(); + r.top_p = 0; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r = basic(); + r.input = Json(); + ASSERT_TRUE(rejects([&] { r.validate(); })); + r = basic(); + r.tools.push_back(ToolBuilder::function("a.b", "x", Json::object())); + r.validate(); + ASSERT_TRUE(rejects([&] { ToolBuilder::function("a.b.c", "x", Json::object()); })); + r = basic(); + r.background = true; + r.input = Json(std::string(1048577, 'x')); + ASSERT_TRUE(rejects([&] { r.to_json(false); })); + // count body carries no sampling or storage fields + r = basic(); + r.store = false; + r.temperature = 1.0; + ASSERT_TRUE(r.to_count_json().find("store") == nullptr); + ASSERT_TRUE(r.to_count_json().find("temperature") == nullptr); +} + +TEST(meta, reasoning_replay_item_shape) { + Json it = ResponseInput::reasoning("ENC"); + ASSERT_TRUE(it.find("summary")->as_array().empty()); // summary is required on replay + ASSERT_TRUE(it.find("id") == nullptr); + ASSERT_EQ(it.find("encrypted_content")->as_string(), "ENC"); + ASSERT_TRUE(rejects([] { ResponseInput::function_call_output(std::string(65, 'x'), "o"); })); + Json img = ResponseInput::image_bytes("image/png", "abc"); + ASSERT_EQ(img.find("image_url")->as_string(), "data:image/png;base64,YWJj"); + ASSERT_TRUE(rejects([] { ResponseInput::image_bytes("image/bmp", "abc"); })); +} + +TEST(meta, schema_preflight) { + Json s = Json::parse(R"({"type":"object","properties":{"a":{"type":"string"}},"required":["a"],"additionalProperties":false})"); + SchemaCheck::check(s, true); + Json loose = Json::parse(R"({"type":"object","properties":{"a":{"type":"string"}}})"); + SchemaCheck::check(loose, false); + ASSERT_TRUE(rejects([&] { SchemaCheck::check(loose, true); })); + Json rec = Json::parse(R"({"type":"object","properties":{"n":{"$ref":"#/$defs/N"}},"$defs":{"N":{"type":"object","properties":{"c":{"$ref":"#/$defs/N"}}}}})"); + ASSERT_TRUE(rejects([&] { SchemaCheck::check(rec, false); })); + Json deep = Json::object(); + Json* cur = &deep; + (void)cur; + std::string text; + for (int i = 0; i < 12; ++i) text += "{\"type\":\"object\",\"properties\":{\"x\":"; + text += "{\"type\":\"string\"}"; + for (int i = 0; i < 12; ++i) text += "}}"; + ASSERT_TRUE(rejects([&] { SchemaCheck::check(Json::parse(text), false); })); +} + +TEST(meta, chat_completion_rules) { + ChatCompletionRequest r; + r.model = "muse-spark-1.3"; + r.messages = {ChatMessage::developer("be brief"), ChatMessage::user("hi")}; + Json b = r.to_json(true); + ASSERT_TRUE(b.find("stream_options")->find("include_usage")->as_bool()); + ASSERT_TRUE(b.find("n") == nullptr && b.find("stop") == nullptr); + r.extra = Json::parse(R"({"n":2})"); + ASSERT_TRUE(rejects([&] { r.validate(); })); + r.extra = Json(); + r.reasoning_effort = "none"; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r.reasoning_effort = "low"; + r.tools = {Json::parse(R"({"type":"custom","name":"x"})")}; + ASSERT_TRUE(rejects([&] { r.validate(); })); + ASSERT_EQ(ChatMessage::tool_result("call_1", "ok").find("role")->as_string(), "tool"); +} + +TEST(meta, messages_rules) { + MessageRequest r; + r.model = "muse-spark-1.3"; + r.messages = {AnthropicBlock::message_text("user", "hi")}; + ASSERT_TRUE(rejects([&] { r.validate(); })); // max_tokens required + r.max_tokens = 2048; + r.thinking = "enabled"; + r.thinking_budget = 512; + ASSERT_TRUE(rejects([&] { r.validate(); })); + r.thinking_budget = 1024; + r.validate(); + r.extra = Json::parse(R"({"top_k":5})"); + ASSERT_TRUE(rejects([&] { r.validate(); })); + r.extra = Json(); + r.temperature = 1.5; + ASSERT_TRUE(rejects([&] { r.validate(); })); + // Docs example: content starts with redacted_thinking, so read text blocks. + Json msg = Json::parse(R"({"content":[{"type":"redacted_thinking","data":"x"},{"type":"text","text":"Hi"}],"usage":{"input_tokens":3,"output_tokens":4}})"); + ASSERT_EQ(MessagesService::text_of(msg), "Hi"); + ASSERT_EQ(MessagesService::usage_of(msg).total_tokens, 7); +} + +TEST(meta, images_rules) { + ImageRequest r; + r.prompt = "a red fox"; + r.validate(false); + r.n = 11; + ASSERT_TRUE(rejects([&] { r.validate(false); })); + r.n = 2; + r.size = "1792x1024"; + r.validate(false); + r.size = "3200x100"; // 32:1 + ASSERT_TRUE(rejects([&] { r.validate(false); })); + r.size = ""; + ASSERT_TRUE(rejects([&] { r.validate(true); })); // edit needs images + r.images = {ImageRequest::image_url("https://example.com/a.png")}; + ASSERT_TRUE(r.to_json(true).find("images") != nullptr); + r.extra = Json::parse(R"({"moderation":"low"})"); + ASSERT_TRUE(rejects([&] { r.validate(true); })); + // Docs "Response shape" example. + ImageResult res = ImageResult::from_json(Json::parse( + R"({"created":1784584435,"data":[{"b64_json":"UklGR..."}],"output_format":"webp","background":"opaque","usage":{"input_tokens":9831,"output_tokens":749,"total_tokens":10580}})")); + ASSERT_EQ(res.data.size(), 1u); + ASSERT_EQ(res.output_format, "webp"); +} + +TEST(meta, responses_documented_example) { + // Verbatim from docs protocols/responses "Example response". + Json doc = Json::parse(R"({"id":"resp_abc123","object":"response","created_at":1714502400,"status":"completed","model":"muse-spark-1.3","store":true,"temperature":1.0,"top_p":1.0,"max_output_tokens":null,"tools":[],"tool_choice":"auto","parallel_tool_calls":true,"service_tier":"auto","output":[{"id":"msg_abc123:456","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"The capital of France is **Paris**.","annotations":[]}]}],"usage":{"input_tokens":69,"output_tokens":163,"total_tokens":232}})"); + molo::llm::ChatResponse r = MetaChatModel::from_response(doc); + ASSERT_EQ(r.text(), "The capital of France is **Paris**."); + ASSERT_EQ(r.finish_reason, "stop"); + ASSERT_EQ(r.usage.total_tokens, 232); + ASSERT_EQ(r.id, "resp_abc123"); +} + +TEST(meta, response_stream_fold_uses_schema_event_names) { + // Event names and fields from the ResponseStreamEvent schemas; the order is + // constructed (the docs ship no verbatim Responses transcript), see SPEC. + const char* events[] = { + R"({"type":"response.created","sequence_number":0,"response":{"id":"resp_1"}})", + R"({"type":"response.output_item.added","sequence_number":1,"output_index":0,"item":{"type":"function_call","id":"fc_1","call_id":"call_1","name":"get_weather","arguments":""}})", + R"({"type":"response.function_call_arguments.delta","sequence_number":2,"item_id":"fc_1","output_index":0,"delta":"{\"city\":"})", + R"({"type":"response.function_call_arguments.delta","sequence_number":3,"item_id":"fc_1","output_index":0,"delta":"\"Paris\"}"})", + R"({"type":"response.output_text.delta","sequence_number":4,"item_id":"m","output_index":1,"content_index":0,"delta":"Hel"})", + R"({"type":"response.output_text.delta","sequence_number":5,"item_id":"m","output_index":1,"content_index":0,"delta":"lo"})", + R"({"type":"response.completed","sequence_number":6,"response":{"id":"resp_1","status":"completed","usage":{"input_tokens":5,"output_tokens":7,"total_tokens":12,"output_tokens_details":{"reasoning_tokens":3}}}})"}; + ResponseAccumulator acc; + for (const char* e : events) { + molo::llm::SseEvent se; + se.data = Json::parse(e); + ResponseEvent ev = ResponseEvent::from_sse(se); + ASSERT_TRUE(ResponseEvents::is_known(ev.type)); + acc.feed(ev); + } + ASSERT_EQ(acc.text(), "Hello"); + ASSERT_TRUE(acc.completed()); + ASSERT_EQ(acc.calls().size(), 1u); + ASSERT_EQ(acc.calls()[0].call_id, "call_1"); + ASSERT_EQ(acc.calls()[0].arguments, "{\"city\":\"Paris\"}"); + ASSERT_EQ(acc.usage().reasoning_tokens, 3); + ASSERT_EQ(acc.response_id(), "resp_1"); +} + +TEST(meta, chat_model_mapping) { + auto client = MetaClient::with_key(""); + MetaChatModel m(client); + molo::llm::ChatRequest req; + req.model = "muse-spark-1.3"; + req.system = "be brief"; + req.messages.push_back(molo::llm::Message::user("hi")); + molo::llm::Message a; + a.role = molo::llm::Role::kAssistant; + molo::llm::ContentPart reasoning; + reasoning.reasoning_signature = "ENC123"; + a.parts.push_back(reasoning); + molo::llm::ToolCall tc; + tc.id = "call_1"; + tc.name = "lookup"; + tc.arguments.set("q", Json("x")); + a.parts.push_back(molo::llm::ContentPart::of_call(tc)); + req.messages.push_back(a); + molo::llm::Message t; + t.role = molo::llm::Role::kTool; + t.parts.push_back(molo::llm::ContentPart::of_result({"call_1", "result", false})); + req.messages.push_back(t); + req.max_tokens = 100; + Json b = m.to_response_request(req).to_json(); + ASSERT_EQ(b.find("instructions")->as_string(), "be brief"); + ASSERT_FALSE(b.find("store")->as_bool()); + ASSERT_EQ(b.find("include")->as_array()[0].as_string(), "reasoning.encrypted_content"); + const auto& items = b.find("input")->as_array(); + ASSERT_EQ(items.size(), 4u); // user, reasoning, function_call, function_call_output + ASSERT_EQ(items[1].find("type")->as_string(), "reasoning"); + ASSERT_EQ(items[2].find("arguments")->as_string(), "{\"q\":\"x\"}"); + ASSERT_EQ(items[3].find("type")->as_string(), "function_call_output"); + req.tool_choice = "required"; + ASSERT_TRUE(rejects([&] { m.to_response_request(req); })); + req.tool_choice = ""; + req.stop = {"x"}; + ASSERT_TRUE(rejects([&] { m.to_chat_request(req); })); + ASSERT_EQ(m.vendor(), "meta"); + // Not configured: no traffic. + req.stop.clear(); + try { + m.generate(req); + FAIL("expected NotConfiguredError"); + } catch (const molo::llm::NotConfiguredError&) { + } +} + +TEST(meta, sam_output_and_masks_match_vendor_conformance_cases) { + // Docs "Video output grammar" line (payload bytes are the docs' own). + std::string doc = "<0f>0<|box;x1=211;y1=228;x2=270;y2=254;w=320;h=334|><|mask;x=0;y=0;data=27,60,~!!!!M!0c[0o91w?q1!pIH4pPRVp2B3'7`e.ioeAf6-k/#Xd8%dX9x(|>,1<|box;x1=155;y1=228;x2=202;y2=254;w=320;h=334|><|mask;x=0;y=0;data=27,48,~!!!!J!0c[q=Pj_zs=*4C4(/./x#:/`S_GnD`=o3?X{emCgO$y@|>\n"; + auto frames = SamOutput::parse(doc); + ASSERT_EQ(frames.size(), 1u); + ASSERT_EQ(frames[0].records.size(), 2u); + ASSERT_EQ(frames[0].records[1].object_id, 1); + ASSERT_EQ(frames[0].records[0].box.right(), 271); + ASSERT_EQ(frames[0].records[0].mask.height, 27); + ASSERT_EQ(frames[0].records[0].mask.width, 60); + ASSERT_EQ(frames[0].records[0].mask.encoding_name(), "lossless"); + ASSERT_TRUE(SamOutput::parse("").empty()); // zero matches is valid + ASSERT_TRUE(rejects([] { SamOutput::parse("<0f>0<|box;x1=1|>"); })); + ASSERT_TRUE(rejects([] { SamOutput::parse("junk"); })); + + // meta-sam conformance case captured-api-image-lossless (vendor repo). + auto cap = SamOutput::parse("<0f>0<|box;x1=513;y1=371;x2=523;y2=381;w=1440;h=900|><|mask;x=0;y=0;data=11,11,~!!!!r!kCq#$.pF[_cZIw.?wO3R8-&NJofiCI$_J0aA.1VC*YyK_'z*(LEG8!+H9evY.82Z-4.@C[.9C@T86o$3E=-&8w3W=]xS.|>\n"); + std::string want = "0000110000000111111100011111111100111111111011111111111111111111110111111111101111111111011111111100011111111000001110000"; + auto raster = cap[0].records[0].mask.decode(); + std::string got; + for (auto v : raster) got += static_cast('0' + v); + ASSERT_EQ(got, want); + + // meta-sam conformance case live-api-content-part-done (one_bit). + auto ob = SamOutput::parse("<0f>0<|box;x1=0;y1=0;x2=4;y2=4;w=20;h=10|><|mask;x=0;y=0;data=5,5,!!!!!(QO(0lu8?|>\n"); + std::string got2; + for (auto v : ob[0].records[0].mask.decode()) got2 += static_cast('0' + v); + ASSERT_EQ(got2, "1001000110000001011110001"); + SamMask corrupt = ob[0].records[0].mask; + corrupt.payload = "!!!"; + ASSERT_TRUE(rejects([&] { corrupt.decode(); })); +} + +TEST(meta, sam_prompts_and_request) { + ASSERT_EQ(SamPrompt::concept_text("yellow school bus"), "yellow school bus"); + ASSERT_TRUE(rejects([] { SamPrompt::concept_text("bus, bicycle"); })); + ASSERT_TRUE(rejects([] { SamPrompt::concept_text("bus and bicycle"); })); + // Docs object-prompt example (1800x1200 image). + SamObject o1; + o1.id = 1; + o1.box = SamBox{444, 626, 675, 849, 1800, 1200, -1}; + SamObject o2; + o2.id = 2; + o2.w = 1800; + o2.h = 1200; + o2.points = {{1430, 700, true}, {1300, 450, false}}; + ASSERT_EQ(SamPrompt::objects_text(0, {o1, o2}), + "<0f>1<|box;x1=444;y1=626;x2=675;y2=849;w=1800;h=1200|>,2<|point;x=1430;y=700;w=1800;h=1200|>-<|point;x=1300;y=450;w=1800;h=1200|>"); + ASSERT_TRUE(rejects([&] { SamPrompt::objects_text(0, {o1, o1}); })); + SamRequest sr; + sr.prompt = "button"; + sr.image_url = "https://example.com/a.png"; + sr.include_confidence = true; + sr.mask_encoding = "one_bit"; + Json b = sr.to_response_request().to_json(true); + ASSERT_EQ(b.find("model")->as_string(), "sam-3.1"); + ASSERT_EQ(b.find("metadata")->find("include_confidence")->as_string(), "true"); // string, not bool + sr.video_url = "https://example.com/v.mp4"; + ASSERT_TRUE(rejects([&] { sr.to_response_request(); })); +} + +TEST(meta, speech_wav_and_events) { + auto wav = [](int rate, int channels, int bits, int seconds) { + int data = rate * channels * (bits / 8) * seconds; + auto le32 = [](int v) { return std::string{char(v), char(v >> 8), char(v >> 16), char(v >> 24)}; }; + auto le16 = [](int v) { return std::string{char(v), char(v >> 8)}; }; + std::string h = "RIFF" + le32(36 + data) + "WAVEfmt " + le32(16) + le16(1) + le16(channels) + + le32(rate) + le32(rate * channels * bits / 8) + le16(channels * bits / 8) + + le16(bits) + "data" + le32(data); + return h + std::string(static_cast(data), '\0'); + }; + WavInfo ok = WavInfo::parse(wav(16000, 1, 16, 2)); + ASSERT_EQ(ok.duration_ms, 2000); + ASSERT_TRUE(rejects([&] { WavInfo::parse(wav(44100, 1, 16, 1)); })); + ASSERT_TRUE(rejects([&] { WavInfo::parse(wav(16000, 2, 16, 1)); })); + ASSERT_TRUE(rejects([&] { WavInfo::parse("not a wav"); })); + SpeechOptions o; + o.mode = "DIARIZATION"; + o.keywords = {"Muse"}; + Json s = o.settings_json("WAV"); + ASSERT_EQ(s.find("audioEncoding")->as_string(), "WAV"); + o.mode = "LOUD"; + ASSERT_TRUE(rejects([&] { o.validate(); })); + // Docs speech-to-text transcript / speechComplete examples. + SpeechEvent t = SpeechEvent::parse(Json::parse(R"({"type":"transcript","transcript":"How is the weather","final":false,"audioProcessedMs":2400})")); + ASSERT_EQ(t.type, "transcript"); + ASSERT_FALSE(t.final_flag); + SpeechEvent c = SpeechEvent::parse(Json::parse(R"({"type":"speechComplete","turnId":1,"transcript":"How is the weather?","audioProcessedMs":4640})")); + ASSERT_EQ(c.turn_id, 1); + // Handshake frame carries the credential, redacted unless revealed. + RealtimeConfig cfg; + cfg.key = "LLM_1_secret"; + cfg.options.mode = "ENDPOINTING"; + RealtimeSession rs(cfg); + ASSERT_EQ(rs.handshake_json(false).find("authorization")->find("accessToken")->as_string(), "Bearer ***"); + ASSERT_EQ(rs.handshake_json(true).find("authorization")->find("accessToken")->as_string(), "Bearer LLM_1_secret"); +} + +TEST(meta, retry_policy_and_rate_headers) { + ApiError e429("meta", 429, "rate_limit_error", "slow"); + ApiError e504("meta", 504, "server_error", "gateway_timeout"); + ApiError e400("meta", 400, "invalid_request_error", "bad"); + ASSERT_TRUE(RetryPolicy::retryable(e429)); + ASSERT_FALSE(RetryPolicy::retryable(e504)); + ASSERT_FALSE(RetryPolicy::retryable(e400)); + ASSERT_TRUE(RetryPolicy::decide(e429, 0, 7).retry); + ASSERT_FALSE(RetryPolicy::decide(e429, 5, 7).retry); + int d0 = RetryPolicy::delay_ms(0, 7), d3 = RetryPolicy::delay_ms(3, 7), d9 = RetryPolicy::delay_ms(9, 7); + ASSERT_TRUE(d0 >= 500 && d0 <= 750); + ASSERT_TRUE(d3 >= 4000 && d3 <= 6000); + ASSERT_TRUE(d9 <= 45000); + ASSERT_EQ(RetryPolicy::delay_ms(2, 99), RetryPolicy::delay_ms(2, 99)); + int calls = 0, slept = 0; + int v = call_with_retry([&] { + if (++calls < 3) throw ApiError("meta", 503, "server_error", "busy"); + return 42; + }, 1, [&](int ms) { slept += ms; }); + ASSERT_EQ(v, 42); + ASSERT_EQ(calls, 3); + ASSERT_TRUE(slept >= 1500); + molo::clients::HeaderMap h; + h.set("x-ratelimit-limit-tokens", "4000000"); + h.set("x-ratelimit-remaining-requests", "2999"); + RateLimits rl = RateLimits::from_headers(h); + ASSERT_EQ(rl.limit_tokens, 4000000); + ASSERT_EQ(rl.remaining_requests, 2999); + ASSERT_EQ(rl.remaining_tokens, -1); +} + +TEST(meta, usage_shapes) { + ASSERT_EQ(MetaUsage::from_chat(Json::parse(R"({"prompt_tokens":10,"completion_tokens":5,"total_tokens":15,"prompt_tokens_details":{"cached_tokens":4},"completion_tokens_details":{"reasoning_tokens":2}})")).cached_tokens, 4); + MetaUsage s = MetaUsage::from_responses(Json::parse(R"({"input_tokens":1,"output_tokens":2,"total_tokens":3,"video_frames_processed":50})")); + ASSERT_EQ(s.video_frames_processed, 50); +} diff --git a/tests/src/meta/test_meta_net.cpp b/tests/src/meta/test_meta_net.cpp new file mode 100644 index 0000000..3130414 --- /dev/null +++ b/tests/src/meta/test_meta_net.cpp @@ -0,0 +1,350 @@ +// Meta services through the real ApiClient over MockTransport: request paths, +// headers, bodies against the call table; error envelopes from the docs; +// SSE decoding of documented stream transcripts; MSP client against the +// vendor's recorded handshake transcript; opt-in live checks. +#include +#include +#include + +#include "framework.h" +#include "clients/transports/mock_transport.h" +#include "meta/chat_completions.h" +#include "meta/chat_model.h" +#include "meta/client.h" +#include "meta/files.h" +#include "meta/images.h" +#include "meta/messages.h" +#include "meta/models.h" +#include "meta/muse_code.h" +#include "meta/responses.h" +#include "meta/speech.h" + +using namespace molo::meta; +using molo::clients::ClientResponse; +using molo::clients::HeaderMap; +using molo::clients::Method; +using molo::clients::transports::MockTransport; +using molo::llm::ApiError; + +// Captured at load: other suites change the environment while they run. +static const bool kLiveNet = std::getenv("MOLO_LIVE_NET") && std::string(std::getenv("MOLO_LIVE_NET")) == "1"; +static const std::string kLiveKey = MetaClient::key_from_env(); + +namespace { +ClientResponse reply(const std::string& body, int status = 200, + const std::string& type = "application/json") { + HeaderMap h; + h.set("Content-Type", type); + return ClientResponse(status, h, body); +} + +struct Rig { + MockTransport* mock; + std::shared_ptr client; + explicit Rig(const std::string& key = "LLM_1_SECRETKEY") { + auto t = std::make_unique(); + mock = t.get(); + client = std::make_shared(std::move(t), MetaClient::config(), key); + } +}; + +std::string tiny_wav() { + auto le32 = [](int v) { return std::string{char(v), char(v >> 8), char(v >> 16), char(v >> 24)}; }; + auto le16 = [](int v) { return std::string{char(v), char(v >> 8)}; }; + return "RIFF" + le32(40) + "WAVEfmt " + le32(16) + le16(1) + le16(1) + le32(16000) + le32(32000) + + le16(2) + le16(16) + "data" + le32(4) + "abcd"; +} + +template +bool rejects_msp(F&& f) { + try { + f(); + } catch (const ApiError& e) { + return e.http_status() == 400; + } + return false; +} + +ResponseRequest basic() { + ResponseRequest r; + r.model = "muse-spark-1.3"; + r.input = Json("hi"); + return r; +} +} // namespace + +TEST(meta_net, create_response_path_auth_body) { + Rig r; + r.mock->expect_response(Method::kPost, "/v1/responses", reply(R"({"id":"resp_1","status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"Hi"}]}],"usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}})")); + ResponsesService svc(r.client); + ResponseResult res = svc.create(basic()); + ASSERT_EQ(res.id(), "resp_1"); + ASSERT_EQ(res.output_text(), "Hi"); + ASSERT_EQ(res.usage().total_tokens, 3); + auto s = r.mock->seen()[0]; + ASSERT_EQ(s.headers.get_or("Authorization", ""), "Bearer LLM_1_SECRETKEY"); + ASSERT_EQ(s.body, R"({"input":"hi","model":"muse-spark-1.3"})"); + ASSERT_EQ(s.target.find("SECRET"), std::string::npos); +} + +TEST(meta_net, not_configured_makes_no_traffic) { + Rig r(""); + ResponsesService svc(r.client); + try { + svc.create(basic()); + FAIL("expected NotConfiguredError"); + } catch (const molo::llm::NotConfiguredError& e) { + ASSERT_EQ(e.http_status(), 503); + ASSERT_TRUE(e.message().find("META_API_KEY") != std::string::npos); + } + ASSERT_EQ(r.mock->send_count(), 0u); +} + +TEST(meta_net, docs_error_envelopes_map_to_api_error) { + Rig r; + r.mock->expect_response(Method::kPost, "/v1/responses", + reply(R"({"error":{"message":"`top_p`: The number must be `<= 1.0`.","type":"invalid_request_error","param":"top_p","code":null}})", 400)); + r.mock->expect_response(Method::kGet, "/v1/files/file-nonexistent", + reply(R"({"error":{"message":"No file found with id 'file-nonexistent'.","type":"invalid_request_error","param":null,"code":"file_not_found"}})", 404)); + r.mock->expect_response(Method::kPost, "/v1/responses", + reply(R"({"error":{"message":"Rate limit exceeded. Please retry after 15 seconds.","type":"rate_limit_error","param":null,"code":"rate_limit_exceeded"}})", 429)); + ResponsesService svc(r.client); + try { svc.create(basic()); FAIL("400"); } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 400); + ASSERT_EQ(e.type(), "invalid_request_error"); + } + FilesService files(r.client); + try { files.retrieve("file-nonexistent"); FAIL("404"); } catch (const ApiError& e) { ASSERT_EQ(e.http_status(), 404); } + try { svc.create(basic()); FAIL("429"); } catch (const ApiError& e) { ASSERT_TRUE(RetryPolicy::retryable(e)); } +} + +TEST(meta_net, response_stream_over_sse) { + Rig r; + std::string sse = + "event: response.created\ndata: {\"type\":\"response.created\",\"sequence_number\":0,\"response\":{\"id\":\"resp_9\"}}\n\n" + "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"sequence_number\":1,\"item_id\":\"m\",\"output_index\":0,\"content_index\":0,\"delta\":\"Hi\"}\n\n" + "event: response.completed\ndata: {\"type\":\"response.completed\",\"sequence_number\":2,\"response\":{\"id\":\"resp_9\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n" + "data: [DONE]\n\n"; + r.mock->expect_response(Method::kPost, "/v1/responses", reply(sse, 200, "text/event-stream")); + ResponsesService svc(r.client); + std::vector types; + ResponseAccumulator acc = svc.stream(basic(), [&](const ResponseEvent& e) { types.push_back(e.type); return true; }); + ASSERT_EQ(acc.text(), "Hi"); + ASSERT_TRUE(acc.completed()); + ASSERT_EQ(types.size(), 3u); + auto s = r.mock->seen()[0]; + ASSERT_EQ(s.headers.get_or("Accept", ""), "text/event-stream"); + ASSERT_TRUE(s.body.find("\"stream\":true") != std::string::npos); +} + +TEST(meta_net, lifecycle_calls_and_destructive_paths) { + Rig r; + r.mock->expect_response(Method::kGet, "/v1/responses/resp_1", reply(R"({"id":"resp_1"})")); + r.mock->expect_response(Method::kDelete, "/v1/responses/resp_1", reply(R"({"id":"resp_1","object":"response.deleted","deleted":true})")); + r.mock->expect_response(Method::kPost, "/v1/responses/resp_1/cancel", reply(R"({"id":"resp_1","status":"cancelled"})")); + r.mock->expect_response(Method::kPost, "/v1/responses/input_tokens", reply(R"({"object":"response.input_tokens","input_tokens":321})")); + ResponsesService svc(r.client); + ASSERT_EQ(svc.retrieve("resp_1").id(), "resp_1"); + ASSERT_TRUE(svc.remove("resp_1").find("deleted")->as_bool()); + ASSERT_EQ(svc.cancel("resp_1").find("status")->as_string(), "cancelled"); + ASSERT_EQ(svc.count_input_tokens(basic()), 321); + ASSERT_EQ(r.mock->send_count(), 4u); + try { svc.remove("../etc"); FAIL("bad id"); } catch (const ApiError& e) { ASSERT_EQ(e.http_status(), 400); } + ASSERT_EQ(r.mock->send_count(), 4u); +} + +TEST(meta_net, chat_and_messages_paths) { + Rig r; + r.mock->expect_response(Method::kPost, "/v1/chat/completions", reply(R"({"id":"c1","model":"muse-spark-1.3","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"yo"}}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}})")); + r.mock->expect_response(Method::kPost, "/v1/messages", reply(R"({"id":"m1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"usage":{"input_tokens":1,"output_tokens":1}})")); + r.mock->expect_response(Method::kPost, "/v1/messages/count_tokens", reply(R"({"input_tokens":12})")); + ChatCompletionRequest cr; + cr.model = "muse-spark-1.3"; + cr.messages = {ChatMessage::user("hi")}; + ChatCompletionsService chat(r.client); + ASSERT_EQ(MetaChatModel::from_chat(chat.create(cr)).text(), "yo"); + MessageRequest mr; + mr.model = "muse-spark-1.3"; + mr.max_tokens = 64; + mr.messages = {AnthropicBlock::message_text("user", "hi")}; + MessagesService ms(r.client); + ASSERT_EQ(MessagesService::text_of(ms.create(mr)), "ok"); + ASSERT_EQ(ms.count_tokens(mr), 12); + ASSERT_TRUE(r.mock->seen()[2].body.find("max_tokens") == std::string::npos + || r.mock->seen()[2].body.find("\"stream\"") == std::string::npos); +} + +TEST(meta_net, chat_stream_chunks) { + Rig r; + std::string sse = + "data: {\"id\":\"c\",\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"A\"}}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"f\",\"arguments\":\"{\\\"a\\\"\"}}]}}]}\n\n" + "data: {\"id\":\"c\",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\":1}\"}}]},\"finish_reason\":\"tool_calls\"}],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":4,\"total_tokens\":7}}\n\n" + "data: [DONE]\n\n"; + r.mock->expect_response(Method::kPost, "/v1/chat/completions", reply(sse, 200, "text/event-stream")); + ChatCompletionRequest cr; + cr.model = "muse-spark-1.3"; + cr.messages = {ChatMessage::user("hi")}; + ChatChunkAccumulator acc = ChatCompletionsService(r.client).stream(cr); + ASSERT_EQ(acc.text(), "A"); + ASSERT_EQ(acc.tool_calls().size(), 1u); + ASSERT_EQ(acc.tool_calls()[0].arguments, "{\"a\":1}"); + ASSERT_EQ(acc.finish_reason(), "tool_calls"); + ASSERT_EQ(acc.usage().total_tokens, 7); +} + +TEST(meta_net, files_upload_list_walk_and_caps) { + Rig r; + r.mock->expect_response(Method::kPost, "/v1/files", reply(R"({"id":"file-842549258569145","object":"file","bytes":3,"created_at":1,"filename":"a.png","purpose":"user_data","status":"processed"})")); + FileUpload up; + up.filename = "a.png"; + up.mime = "image/png"; + up.bytes = "abc"; + FilesService fs(r.client); + FileObject f = fs.upload(up); + ASSERT_EQ(f.id, "file-842549258569145"); + auto s = r.mock->seen()[0]; + ASSERT_TRUE(s.headers.get_or("Content-Type", "").find("multipart/form-data") == 0); + ASSERT_TRUE(s.body.find("name=\"purpose\"") < s.body.find("name=\"file\"")); // purpose before file + up.expires_after_seconds = 10; + try { fs.upload(up); FAIL("expiry range"); } catch (const ApiError& e) { ASSERT_EQ(e.http_status(), 400); } + // Two pages, then done. + r.mock->expect_response(Method::kGet, "/v1/files", reply(R"({"object":"list","data":[{"id":"file-1"}],"first_id":"file-1","last_id":"file-1","has_more":true})")); + r.mock->expect_response(Method::kGet, "/v1/files", reply(R"({"object":"list","data":[{"id":"file-2"}],"first_id":"file-2","last_id":"file-2","has_more":false})")); + FileList all = fs.list_all("", 1); + ASSERT_EQ(all.files.size(), 2u); + ASSERT_FALSE(all.truncated); + try { fs.list_page("", 100, "sideways"); FAIL("order"); } catch (const ApiError& e) { ASSERT_EQ(e.http_status(), 400); } +} + +TEST(meta_net, models_images_and_transcribe_paths) { + Rig r; + r.mock->expect_response(Method::kGet, "/v1/models", reply(R"({"object":"list","data":[{"id":"muse-spark-1.3","object":"model","created":1,"owned_by":"meta"}]})")); + r.mock->expect_response(Method::kPost, "/v1/images/generations", reply(R"({"created":1,"data":[{"b64_json":"AA=="}]})")); + auto models = ModelsService(r.client).list(); + ASSERT_EQ(models.size(), 1u); + ASSERT_EQ(models[0].id, "muse-spark-1.3"); + ImageRequest ir; + ir.prompt = "fox"; + ASSERT_EQ(ImagesService(r.client).generate(ir).data.size(), 1u); + ASSERT_TRUE(r.mock->seen()[1].body.find("muse-image-1.0") != std::string::npos); + // Transcribe: multipart with a JSON request part and the audio, Accept honoured. + std::string wav = tiny_wav(); + r.mock->expect_response(Method::kPost, "/v1/asr/transcribe", reply(R"({"sessionId":"my-session","transcript":"hello","audioDurationMs":1,"turns":[{"turnId":0,"startMs":0,"endMs":1,"transcript":"hello"}]})")); + SpeechOptions so; + so.session_id = "my-session"; + so.mode = "ENDPOINTING"; + TranscribeResult tr = TranscribeService(r.client).transcribe(so, wav); + ASSERT_EQ(tr.transcript, "hello"); + ASSERT_EQ(tr.turns.size(), 1u); + auto s = r.mock->seen()[2]; + ASSERT_TRUE(s.body.find("name=\"request\"\r\nContent-Type: application/json") != std::string::npos); + ASSERT_TRUE(s.body.find("\"audioEncoding\":\"WAV\"") != std::string::npos); + ASSERT_EQ(s.headers.get_or("Accept", ""), "application/json"); +} + +TEST(meta_net, transcribe_event_stream_documented_sample) { + Rig r; + // Verbatim from docs speech-to-text "Choose another response format". + std::string sse = + "event: transcript\ndata: {\"type\":\"transcript\",\"transcript\":\"How is the weather\",\"final\":false,\"audioProcessedMs\":2400}\n\n" + "event: speechComplete\ndata: {\"type\":\"speechComplete\",\"turnId\":1,\"transcript\":\"How is the weather?\",\"audioProcessedMs\":4640}\n\n"; + r.mock->expect_response(Method::kPost, "/v1/asr/transcribe", reply(sse, 200, "text/event-stream")); + std::string wav = tiny_wav(); + auto ev = TranscribeService(r.client).transcribe_events(SpeechOptions(), wav); + ASSERT_EQ(ev.size(), 2u); + ASSERT_EQ(ev[1].type, "speechComplete"); + ASSERT_EQ(ev[1].turn_id, 1); + ASSERT_EQ(r.mock->seen()[0].headers.get_or("Accept", ""), "text/event-stream"); +} + +namespace { +// Replays the vendor's recorded `session-start` client lines against our +// client: we check what we WRITE equals the recorded client frames (ids +// normalized), and that the recorded server frames parse. +class RecordedTransport : public ILineTransport { + public: + std::vector written; + std::vector to_read; + std::size_t pos = 0; + void write_line(const std::string& l) override { written.push_back(l); } + std::optional read_line(int) override { + if (pos >= to_read.size()) return std::nullopt; + return to_read[pos++]; + } + void close() override {} +}; +} // namespace + +TEST(meta_net, msp_handshake_matches_recorded_transcript) { + // schema/msp/transcripts/handshake-liststream-granted (muse-code-sdk, verbatim server line). + auto t = std::make_unique(); + RecordedTransport* rec = t.get(); + rec->to_read.push_back(R"J({"id":1,"jsonrpc":"2.0","result":{"experimentalApi":false,"grantedCapabilities":["sessionListStream"],"museHome":"/home/fixture/.muse","platformFamily":"unix","platformOs":"linux","schema":{"fingerprint":"sha256:c8d1a2a1866814e220fd396d382a9a75861412feee884b5021b2ee359bd3dc59","version":1},"serverInfo":{"name":"muse-session-server","version":"0.0.0-fixture"},"sessionDurability":"durable","userAgent":"muse-session-server/0.0.0-fixture (linux; x86_64)"}})J"); + MspClient c(std::move(t)); + Json res = c.initialize("conformance", "0.0.0", {"sessionListStream"}); + ASSERT_EQ(res.find("grantedCapabilities")->as_array()[0].as_string(), "sessionListStream"); + ASSERT_EQ(c.schema_fingerprint(), "sha256:c8d1a2a1866814e220fd396d382a9a75861412feee884b5021b2ee359bd3dc59"); + ASSERT_EQ(rec->written[0], + R"({"id":1,"jsonrpc":"2.0","method":"initialize","params":{"capabilities":{"requestedCapabilities":["sessionListStream"]},"clientInfo":{"name":"conformance","version":"0.0.0"}}})"); + ASSERT_EQ(rec->written[1], R"({"jsonrpc":"2.0","method":"initialized"})"); +} + +TEST(meta_net, msp_requests_notifications_errors_and_caps) { + auto t = std::make_unique(); + RecordedTransport* rec = t.get(); + rec->to_read.push_back(R"({"jsonrpc":"2.0","id":1,"result":{"schema":{"version":1,"fingerprint":"sha256:x"}}})"); + rec->to_read.push_back(R"({"jsonrpc":"2.0","method":"session/started","params":{"session":{"sessionId":"s1"}},"emittedAtMs":1})"); + rec->to_read.push_back(R"({"jsonrpc":"2.0","method":"approval/request","id":"srv-1","params":{"sessionId":"s1"}})"); + rec->to_read.push_back(R"({"jsonrpc":"2.0","id":2,"error":{"code":-32020,"message":"no such session"}})"); + MspClient c(std::move(t)); + ASSERT_TRUE(rejects_msp([&] { c.model_list(); })); // before initialize + c.initialize("t", "1"); + try { + c.turn_start("s1", "hello"); + FAIL("expected sessionNotFound"); + } catch (const ApiError& e) { + ASSERT_EQ(e.type(), "sessionNotFound"); + } + auto n = c.next_notification(10); + ASSERT_TRUE(n.has_value()); + ASSERT_EQ(n->method, "session/started"); + ASSERT_TRUE(rec->written.back().find("\"approval\"") == std::string::npos); + bool answered = false; + for (const std::string& w : rec->written) + if (w.find("\"id\":\"srv-1\"") != std::string::npos) answered = true; + ASSERT_TRUE(answered); // server request answered with a receipt + ASSERT_TRUE(MspFrames::is_known_method("turn/start")); + ASSERT_FALSE(MspFrames::is_known_method("turn/nope")); + ASSERT_EQ(MspFrames::error_kind(-32001), "overloaded"); + ASSERT_TRUE(MspFrames::error_retryable(-32001)); + LineSplitter sp(8); + sp.feed("ab\r\ncd\n"); + ASSERT_EQ(*sp.pop(), "ab"); + ASSERT_EQ(*sp.pop(), "cd"); + ASSERT_TRUE(rejects_msp([&] { sp.feed("123456789"); })); + ASSERT_EQ(MspFrames::new_id().size(), 36u); +} + +TEST(meta_net, live_opt_in_unauthenticated_and_keyed) { + if (!kLiveNet) { + std::cout << "SKIP meta live checks: set MOLO_LIVE_NET=1 (and META_API_KEY or MODEL_API_KEY for keyed ones)\n"; + return; + } + ServiceStatus st = StatusService(MetaClient::unauthenticated()).get(); + ASSERT_TRUE(st.is_alive); + if (kLiveKey.empty()) { + std::cout << "SKIP meta keyed live checks: no META_API_KEY / MODEL_API_KEY\n"; + // Unauthenticated probe: a documented path answers 401 invalid_api_key, not 404. + try { + ModelsService(MetaClient::with_key("LLM_0_bogus")).list(); + FAIL("expected 401"); + } catch (const ApiError& e) { + ASSERT_EQ(e.http_status(), 401); + ASSERT_EQ(e.type(), "authentication_error"); + } + return; + } + auto models = ModelsService(MetaClient::with_key(kLiveKey)).list(); // read only + ASSERT_FALSE(models.empty()); +}