61 lines
2.0 KiB
C++
61 lines
2.0 KiB
C++
#pragma once
|
|
|
|
// Batch jobs: /v1/batch/jobs. A job reads uploaded .jsonl files (purpose
|
|
// "batch") or up to 10000 inline requests and runs one endpoint over them at a
|
|
// lower price. create is paid; cancel and delete are explicit methods.
|
|
|
|
#include <optional>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "vendors/mistral/types.h"
|
|
|
|
namespace molo::mistral {
|
|
|
|
struct BatchRequestItem {
|
|
std::string custom_id; // optional, <= 256 chars
|
|
Json body = Json::object();
|
|
};
|
|
|
|
struct BatchParams {
|
|
// spec ApiEndpoint enum
|
|
std::string endpoint = "/v1/chat/completions";
|
|
std::string model; // model or agent_id, one of them
|
|
std::string agent_id;
|
|
std::vector<std::string> input_files; // file ids, <= 100
|
|
std::vector<BatchRequestItem> requests; // <= 10000 (spec maxItems)
|
|
Json metadata; // null or object
|
|
int timeout_hours = 24; // 1..168 (spec)
|
|
void validate() const;
|
|
Json to_json() const;
|
|
};
|
|
|
|
struct BatchListParams {
|
|
int page = 0;
|
|
int page_size = 100; // 1..100
|
|
std::string model, agent_id, status, order_by;
|
|
std::string created_after;
|
|
std::optional<bool> created_by_me;
|
|
void validate() const;
|
|
};
|
|
|
|
struct BatchJob {
|
|
std::string id, status, endpoint, model, output_file, error_file;
|
|
long created_at = 0, total_requests = 0, completed_requests = 0, succeeded_requests = 0, failed_requests = 0;
|
|
Json raw;
|
|
bool terminal() const; // SUCCESS FAILED TIMEOUT_EXCEEDED CANCELLED
|
|
static BatchJob parse(const Json& doc);
|
|
};
|
|
|
|
class Batch : public MistralService {
|
|
public:
|
|
explicit Batch(std::shared_ptr<llm::ApiClient> client) : MistralService(std::move(client)) {}
|
|
BatchJob create(const BatchParams& params); // costs money
|
|
ListPage list(const BatchListParams& params = {});
|
|
BatchJob get(const std::string& job_id, bool inline_outputs = false);
|
|
BatchJob cancel(const std::string& job_id); // explicit
|
|
Json remove(const std::string& job_id); // explicit, irreversible
|
|
};
|
|
|
|
} // namespace molo::mistral
|