Files
moloapp/src/google/models/media.h
T

79 lines
2.8 KiB
C++
Raw Normal View History

#pragma once
// Media generation on Vertex AI publisher models: images (Imagen, predict),
// music (Lyria, predict) and video (Veo, predictLongRunning plus
// fetchPredictOperation polling, bounded).
#include <optional>
#include <string>
#include <vector>
#include "google/operation.h"
#include "google/models/vertex_service.h"
namespace molo::google::models {
struct MediaItem {
std::string mime;
std::string bytes; // decoded; empty when the result lives at `uri`
std::string uri; // gs:// location when the request asked for storage
};
struct ImageRequest {
std::string prompt;
int count = 1; // 1..kMaxImages
std::string aspect_ratio; // "", "1:1", "16:9", ...
std::string negative_prompt;
};
struct MusicRequest {
std::string prompt;
std::string negative_prompt;
int count = 1; // 1..kMaxMusicSamples
std::optional<int> seed;
};
struct VideoRequest {
std::string prompt;
int count = 1; // 1..kMaxVideoSamples
int duration_seconds = 0; // 0: model default; else 1..kMaxVideoSeconds
std::string aspect_ratio; // "", "16:9", "9:16"
std::string storage_uri; // optional gs:// output prefix
};
class MediaGenerator : public VertexService {
public:
MediaGenerator(std::shared_ptr<GoogleClient> client, VertexTarget target);
std::vector<MediaItem> generate_image(const std::string& model,
const ImageRequest& request) const;
std::vector<MediaItem> generate_music(const std::string& model,
const MusicRequest& request) const;
// Starts a video job; returns the operation name.
std::string start_video(const std::string& model,
const VideoRequest& request) const;
// One poll (POST :fetchPredictOperation).
Operation video_status(const std::string& model,
const std::string& operation_name) const;
// Polls until done within kMaxVideoPolls. Throws DEADLINE_EXCEEDED at the
// cap and OPERATION_FAILED when the job failed. sleeper injectable.
std::vector<MediaItem> wait_video(const std::string& model,
const std::string& operation_name,
const Sleeper& sleeper = nullptr) const;
PlannedCall plan_image(const std::string& model, const ImageRequest& r) const;
PlannedCall plan_music(const std::string& model, const MusicRequest& r) const;
PlannedCall plan_start_video(const std::string& model, const VideoRequest& r) const;
PlannedCall plan_video_status(const std::string& model,
const std::string& operation_name) const;
static std::vector<MediaItem> items_from(const clients::body::Json& list);
private:
static void check_prompt(const std::string& prompt);
std::vector<MediaItem> run_predict(const PlannedCall& call) const;
};
} // namespace molo::google::models