feat(clients): meta: Meta Model API module, Muse Code protocol client, demo and tests
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
#pragma once
|
||||
|
||||
// Segment Anything Model 3.1 (model sam-3.1) over the Responses API. There is
|
||||
// no segmentation endpoint: the prompt (a noun phrase, or box / point tokens)
|
||||
// goes in an input_text part beside one input_image or input_video part, and
|
||||
// the answer arrives as special-token text in one output_text lane, one line
|
||||
// per frame:
|
||||
// <Nf>id<|box;x1=..;y1=..;x2=..;y2=..;w=..;h=..[;c=..]|><|mask;x=0;y=0[;c=..];data=H,W,<enc>payload|>,id...
|
||||
// This module builds the request, parses that text, and decodes masks
|
||||
// (one_bit "!" and lossless "~") to a 0/1 raster. The mask codec is ported
|
||||
// from the official meta-sam parser (see SPEC Sources) and checked against its
|
||||
// conformance cases.
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "meta/response_request.h"
|
||||
|
||||
namespace molo::meta {
|
||||
|
||||
struct SamBox {
|
||||
int x1 = 0, y1 = 0, x2 = 0, y2 = 0; // x2 / y2 are inclusive, as on the wire
|
||||
int w = 0, h = 0; // source frame size
|
||||
double confidence = -1; // -1 = absent
|
||||
int right() const { return x2 + 1; } // exclusive edge
|
||||
int bottom() const { return y2 + 1; }
|
||||
};
|
||||
|
||||
struct SamMask {
|
||||
int height = 0, width = 0;
|
||||
char encoding = 0; // '~' lossless, '!' one_bit
|
||||
std::string payload; // including the encoding marker
|
||||
double confidence = -1;
|
||||
// Row-major height*width bytes of 0/1. Throws ApiError on a malformed payload.
|
||||
std::vector<std::uint8_t> decode() const;
|
||||
std::string encoding_name() const { return encoding == '!' ? "one_bit" : "lossless"; }
|
||||
};
|
||||
|
||||
struct SamRecord {
|
||||
long object_id = 0;
|
||||
SamBox box;
|
||||
SamMask mask;
|
||||
};
|
||||
|
||||
struct SamFrame {
|
||||
long frame = 0; // zero-based, not necessarily contiguous or ascending
|
||||
std::vector<SamRecord> records;
|
||||
};
|
||||
|
||||
class SamOutput {
|
||||
public:
|
||||
// Parses the whole output_text (lines separated by \n). Empty text is a valid
|
||||
// "no matches" result. Throws ApiError on a malformed line; caps lines and
|
||||
// records.
|
||||
static std::vector<SamFrame> parse(const std::string& text);
|
||||
static std::size_t record_count(const std::vector<SamFrame>& frames);
|
||||
};
|
||||
|
||||
struct SamPoint {
|
||||
int x = 0, y = 0;
|
||||
bool positive = true;
|
||||
};
|
||||
|
||||
struct SamObject {
|
||||
long id = 0;
|
||||
std::optional<SamBox> box; // x1..y2 and w,h
|
||||
std::vector<SamPoint> points;
|
||||
int w = 0, h = 0; // media size for the points
|
||||
};
|
||||
|
||||
class SamPrompt {
|
||||
public:
|
||||
// One short noun phrase; commas, semicolons, plus signs, newlines and
|
||||
// "and" list several concepts (docs: that returns nothing), so they are rejected.
|
||||
static std::string concept_text(const std::string& phrase);
|
||||
// Box and point prompts for one frame: "<Nf>id<|box;...|>,id<|point;...|>-<|point;...|>".
|
||||
static std::string objects_text(long frame, const std::vector<SamObject>& objects);
|
||||
// Several frames, in the order given.
|
||||
static std::string frames_text(const std::vector<std::pair<long, std::vector<SamObject>>>& frames);
|
||||
};
|
||||
|
||||
struct SamRequest {
|
||||
std::string model = "sam-3.1";
|
||||
std::string prompt; // from SamPrompt
|
||||
std::string image_url; // exactly one of image_url / video_url / video_file_id
|
||||
std::string video_url;
|
||||
std::string video_file_id;
|
||||
std::string propagation_direction; // "" (both) | forward | backward | both (video prompts)
|
||||
bool include_confidence = false;
|
||||
std::string mask_encoding; // "" (lossless) | lossless | one_bit
|
||||
bool stream = true; // docs: recommended
|
||||
|
||||
ResponseRequest to_response_request() const;
|
||||
};
|
||||
|
||||
} // namespace molo::meta
|
||||
Reference in New Issue
Block a user