feat(clients): meta: Meta Model API module, Muse Code protocol client, demo and tests

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 08:02:23 +02:00
co-authored by Claude Sonnet 5.5
parent 2e0270c043
commit 6fb7c80d2d
51 changed files with 7035 additions and 3 deletions
+98
View File
@@ -0,0 +1,98 @@
#pragma once
// Segment Anything Model 3.1 (model sam-3.1) over the Responses API. There is
// no segmentation endpoint: the prompt (a noun phrase, or box / point tokens)
// goes in an input_text part beside one input_image or input_video part, and
// the answer arrives as special-token text in one output_text lane, one line
// per frame:
// <Nf>id<|box;x1=..;y1=..;x2=..;y2=..;w=..;h=..[;c=..]|><|mask;x=0;y=0[;c=..];data=H,W,<enc>payload|>,id...
// This module builds the request, parses that text, and decodes masks
// (one_bit "!" and lossless "~") to a 0/1 raster. The mask codec is ported
// from the official meta-sam parser (see SPEC Sources) and checked against its
// conformance cases.
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
#include "meta/response_request.h"
namespace molo::meta {
struct SamBox {
int x1 = 0, y1 = 0, x2 = 0, y2 = 0; // x2 / y2 are inclusive, as on the wire
int w = 0, h = 0; // source frame size
double confidence = -1; // -1 = absent
int right() const { return x2 + 1; } // exclusive edge
int bottom() const { return y2 + 1; }
};
struct SamMask {
int height = 0, width = 0;
char encoding = 0; // '~' lossless, '!' one_bit
std::string payload; // including the encoding marker
double confidence = -1;
// Row-major height*width bytes of 0/1. Throws ApiError on a malformed payload.
std::vector<std::uint8_t> decode() const;
std::string encoding_name() const { return encoding == '!' ? "one_bit" : "lossless"; }
};
struct SamRecord {
long object_id = 0;
SamBox box;
SamMask mask;
};
struct SamFrame {
long frame = 0; // zero-based, not necessarily contiguous or ascending
std::vector<SamRecord> records;
};
class SamOutput {
public:
// Parses the whole output_text (lines separated by \n). Empty text is a valid
// "no matches" result. Throws ApiError on a malformed line; caps lines and
// records.
static std::vector<SamFrame> parse(const std::string& text);
static std::size_t record_count(const std::vector<SamFrame>& frames);
};
struct SamPoint {
int x = 0, y = 0;
bool positive = true;
};
struct SamObject {
long id = 0;
std::optional<SamBox> box; // x1..y2 and w,h
std::vector<SamPoint> points;
int w = 0, h = 0; // media size for the points
};
class SamPrompt {
public:
// One short noun phrase; commas, semicolons, plus signs, newlines and
// "and" list several concepts (docs: that returns nothing), so they are rejected.
static std::string concept_text(const std::string& phrase);
// Box and point prompts for one frame: "<Nf>id<|box;...|>,id<|point;...|>-<|point;...|>".
static std::string objects_text(long frame, const std::vector<SamObject>& objects);
// Several frames, in the order given.
static std::string frames_text(const std::vector<std::pair<long, std::vector<SamObject>>>& frames);
};
struct SamRequest {
std::string model = "sam-3.1";
std::string prompt; // from SamPrompt
std::string image_url; // exactly one of image_url / video_url / video_file_id
std::string video_url;
std::string video_file_id;
std::string propagation_direction; // "" (both) | forward | backward | both (video prompts)
bool include_confidence = false;
std::string mask_encoding; // "" (lossless) | lossless | one_bit
bool stream = true; // docs: recommended
ResponseRequest to_response_request() const;
};
} // namespace molo::meta