Files
moloapp/src/vendors/mistral/batch.h
T

61 lines
2.0 KiB
C++

#pragma once
// Batch jobs: /v1/batch/jobs. A job reads uploaded .jsonl files (purpose
// "batch") or up to 10000 inline requests and runs one endpoint over them at a
// lower price. create is paid; cancel and delete are explicit methods.
#include <optional>
#include <string>
#include <vector>
#include "vendors/mistral/types.h"
namespace molo::mistral {
struct BatchRequestItem {
std::string custom_id; // optional, <= 256 chars
Json body = Json::object();
};
struct BatchParams {
// spec ApiEndpoint enum
std::string endpoint = "/v1/chat/completions";
std::string model; // model or agent_id, one of them
std::string agent_id;
std::vector<std::string> input_files; // file ids, <= 100
std::vector<BatchRequestItem> requests; // <= 10000 (spec maxItems)
Json metadata; // null or object
int timeout_hours = 24; // 1..168 (spec)
void validate() const;
Json to_json() const;
};
struct BatchListParams {
int page = 0;
int page_size = 100; // 1..100
std::string model, agent_id, status, order_by;
std::string created_after;
std::optional<bool> created_by_me;
void validate() const;
};
struct BatchJob {
std::string id, status, endpoint, model, output_file, error_file;
long created_at = 0, total_requests = 0, completed_requests = 0, succeeded_requests = 0, failed_requests = 0;
Json raw;
bool terminal() const; // SUCCESS FAILED TIMEOUT_EXCEEDED CANCELLED
static BatchJob parse(const Json& doc);
};
class Batch : public MistralService {
public:
explicit Batch(std::shared_ptr<llm::ApiClient> client) : MistralService(std::move(client)) {}
BatchJob create(const BatchParams& params); // costs money
ListPage list(const BatchListParams& params = {});
BatchJob get(const std::string& job_id, bool inline_outputs = false);
BatchJob cancel(const std::string& job_id); // explicit
Json remove(const std::string& job_id); // explicit, irreversible
};
} // namespace molo::mistral