42 lines
1.5 KiB
C++
42 lines
1.5 KiB
C++
#pragma once
|
|||
|
|
|
||
|
|
// Evals: definitions (CRUD), runs (create, retrieve, list, cancel, delete)
|
||
|
|
// and per-run output items.
|
||
|
|
|
||
|
|
#include <string>
|
||
|
|
|
||
|
|
#include "openai/platform/service.h"
|
||
|
|
|
||
|
|
namespace molo::openai::platform {
|
||
|
|
|
||
|
|
class EvalsApi : public PlatformService {
|
||
|
|
public:
|
||
|
|
using PlatformService::PlatformService;
|
||
|
|
|
||
|
|
// body: data_source_config, testing_criteria (required), name, metadata.
|
||
|
|
Json create(const Json& body);
|
||
|
|
Json retrieve(const std::string& eval_id);
|
||
|
|
Json update(const std::string& eval_id, const Json& body); // name, metadata
|
||
|
|
Json list(const Json& query = Json::object()); // after, limit, order, order_by
|
||
|
|
// Destructive: deletes the eval and its runs.
|
||
|
|
Json remove(const std::string& eval_id);
|
||
|
|
};
|
||
|
|
|
||
|
|
class EvalRunsApi : public PlatformService {
|
||
|
|
public:
|
||
|
|
using PlatformService::PlatformService;
|
||
|
|
|
||
|
|
// Paid (runs the data source and graders). body: data_source (required),
|
||
|
|
// name, metadata.
|
||
|
|
Json create(const std::string& eval_id, const Json& body);
|
||
|
|
Json retrieve(const std::string& eval_id, const std::string& run_id);
|
||
|
|
Json list(const std::string& eval_id, const Json& query = Json::object()); // status too
|
||
|
|
// Destructive.
|
||
|
|
Json cancel(const std::string& eval_id, const std::string& run_id);
|
||
|
|
Json remove(const std::string& eval_id, const std::string& run_id);
|
||
|
|
Json output_items(const std::string& eval_id, const std::string& run_id, const Json& query = Json::object());
|
||
|
|
Json output_item(const std::string& eval_id, const std::string& run_id, const std::string& item_id);
|
||
|
|
};
|
||
|
|
|
||
|
|
} // namespace molo::openai::platform
|