99 lines
3.4 KiB
C++
99 lines
3.4 KiB
C++
#pragma once
|
|
|
|
// Segment Anything Model 3.1 (model sam-3.1) over the Responses API. There is
|
|
// no segmentation endpoint: the prompt (a noun phrase, or box / point tokens)
|
|
// goes in an input_text part beside one input_image or input_video part, and
|
|
// the answer arrives as special-token text in one output_text lane, one line
|
|
// per frame:
|
|
// <Nf>id<|box;x1=..;y1=..;x2=..;y2=..;w=..;h=..[;c=..]|><|mask;x=0;y=0[;c=..];data=H,W,<enc>payload|>,id...
|
|
// This module builds the request, parses that text, and decodes masks
|
|
// (one_bit "!" and lossless "~") to a 0/1 raster. The mask codec is ported
|
|
// from the official meta-sam parser (see SPEC Sources) and checked against its
|
|
// conformance cases.
|
|
|
|
#include <cstdint>
|
|
#include <optional>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "meta/response_request.h"
|
|
|
|
namespace molo::meta {
|
|
|
|
struct SamBox {
|
|
int x1 = 0, y1 = 0, x2 = 0, y2 = 0; // x2 / y2 are inclusive, as on the wire
|
|
int w = 0, h = 0; // source frame size
|
|
double confidence = -1; // -1 = absent
|
|
int right() const { return x2 + 1; } // exclusive edge
|
|
int bottom() const { return y2 + 1; }
|
|
};
|
|
|
|
struct SamMask {
|
|
int height = 0, width = 0;
|
|
char encoding = 0; // '~' lossless, '!' one_bit
|
|
std::string payload; // including the encoding marker
|
|
double confidence = -1;
|
|
// Row-major height*width bytes of 0/1. Throws ApiError on a malformed payload.
|
|
std::vector<std::uint8_t> decode() const;
|
|
std::string encoding_name() const { return encoding == '!' ? "one_bit" : "lossless"; }
|
|
};
|
|
|
|
struct SamRecord {
|
|
long object_id = 0;
|
|
SamBox box;
|
|
SamMask mask;
|
|
};
|
|
|
|
struct SamFrame {
|
|
long frame = 0; // zero-based, not necessarily contiguous or ascending
|
|
std::vector<SamRecord> records;
|
|
};
|
|
|
|
class SamOutput {
|
|
public:
|
|
// Parses the whole output_text (lines separated by \n). Empty text is a valid
|
|
// "no matches" result. Throws ApiError on a malformed line; caps lines and
|
|
// records.
|
|
static std::vector<SamFrame> parse(const std::string& text);
|
|
static std::size_t record_count(const std::vector<SamFrame>& frames);
|
|
};
|
|
|
|
struct SamPoint {
|
|
int x = 0, y = 0;
|
|
bool positive = true;
|
|
};
|
|
|
|
struct SamObject {
|
|
long id = 0;
|
|
std::optional<SamBox> box; // x1..y2 and w,h
|
|
std::vector<SamPoint> points;
|
|
int w = 0, h = 0; // media size for the points
|
|
};
|
|
|
|
class SamPrompt {
|
|
public:
|
|
// One short noun phrase; commas, semicolons, plus signs, newlines and
|
|
// "and" list several concepts (docs: that returns nothing), so they are rejected.
|
|
static std::string concept_text(const std::string& phrase);
|
|
// Box and point prompts for one frame: "<Nf>id<|box;...|>,id<|point;...|>-<|point;...|>".
|
|
static std::string objects_text(long frame, const std::vector<SamObject>& objects);
|
|
// Several frames, in the order given.
|
|
static std::string frames_text(const std::vector<std::pair<long, std::vector<SamObject>>>& frames);
|
|
};
|
|
|
|
struct SamRequest {
|
|
std::string model = "sam-3.1";
|
|
std::string prompt; // from SamPrompt
|
|
std::string image_url; // exactly one of image_url / video_url / video_file_id
|
|
std::string video_url;
|
|
std::string video_file_id;
|
|
std::string propagation_direction; // "" (both) | forward | backward | both (video prompts)
|
|
bool include_confidence = false;
|
|
std::string mask_encoding; // "" (lossless) | lossless | one_bit
|
|
bool stream = true; // docs: recommended
|
|
|
|
ResponseRequest to_response_request() const;
|
|
};
|
|
|
|
} // namespace molo::meta
|