88 lines
2.5 KiB
C++
88 lines
2.5 KiB
C++
#include "operability/chaos/experiment.h"
|
|||
|
|
|
||
|
|
#include "operability/caps.h"
|
||
|
|
#include "supervisor/common/clock.hpp"
|
||
|
|
#include "supervisor/common/observer.hpp"
|
||
|
|
|
||
|
|
namespace molo::chaos {
|
||
|
|
|
||
|
|
Experiment::Experiment(::molo::common::Clock& clock, ::molo::common::Observer& observer,
|
||
|
|
std::string name, int blast_pct, std::int64_t duration_ms, double max_burn)
|
||
|
|
: clock_(clock),
|
||
|
|
observer_(observer),
|
||
|
|
name_(std::move(name)),
|
||
|
|
blast_pct_(blast_pct),
|
||
|
|
duration_ms_(duration_ms),
|
||
|
|
max_burn_(max_burn) {
|
||
|
|
if (blast_pct_ < 0) blast_pct_ = 0;
|
||
|
|
if (blast_pct_ > ::molo::operability::Caps::kChaosMaxBlastPct) {
|
||
|
|
blast_pct_ = ::molo::operability::Caps::kChaosMaxBlastPct;
|
||
|
|
}
|
||
|
|
if (duration_ms_ < 0) duration_ms_ = 0;
|
||
|
|
if (max_burn_ < 0.0) max_burn_ = 0.0;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Experiment::start() {
|
||
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
||
|
|
if (state_ != State::kIdle || name_.empty()) return false;
|
||
|
|
state_ = State::kRunning;
|
||
|
|
start_ms_ = clock_.steady_ms();
|
||
|
|
observer_.event("chaos.experiment.start", name_);
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
void Experiment::finish() {
|
||
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
||
|
|
if (state_ != State::kRunning) return;
|
||
|
|
state_ = State::kFinished;
|
||
|
|
observer_.event("chaos.experiment.finish", name_);
|
||
|
|
}
|
||
|
|
|
||
|
|
void Experiment::abort(const std::string& reason) {
|
||
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
||
|
|
if (state_ != State::kRunning) return;
|
||
|
|
state_ = State::kAborted;
|
||
|
|
observer_.event("chaos.experiment.abort", name_ + " " + reason);
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Experiment::should_apply(std::uint64_t seq) {
|
||
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
||
|
|
if (state_ != State::kRunning) return false;
|
||
|
|
if (duration_ms_ > 0 && clock_.steady_ms() - start_ms_ >= duration_ms_) {
|
||
|
|
state_ = State::kFinished;
|
||
|
|
observer_.event("chaos.experiment.finish", name_);
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
if (blast_pct_ <= 0) return false;
|
||
|
|
if (blast_pct_ >= 100) return true;
|
||
|
|
std::uint64_t x = seq + 0x9e3779b97f4a7c15ULL;
|
||
|
|
x ^= x >> 30;
|
||
|
|
x *= 0xbf58476d1ce4e5b9ULL;
|
||
|
|
x ^= x >> 27;
|
||
|
|
return (x % 100) < static_cast<std::uint64_t>(blast_pct_);
|
||
|
|
}
|
||
|
|
|
||
|
|
bool Experiment::abort_if_breached(const BurnProbe& probe) {
|
||
|
|
if (!probe) return false;
|
||
|
|
double burn = 0.0;
|
||
|
|
try {
|
||
|
|
burn = probe();
|
||
|
|
} catch (...) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
if (burn <= max_burn_) return false;
|
||
|
|
abort("burn ceiling breached");
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
Experiment::State Experiment::state() const {
|
||
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
||
|
|
return state_;
|
||
|
|
}
|
||
|
|
|
||
|
|
const std::string& Experiment::name() const { return name_; }
|
||
|
|
|
||
|
|
int Experiment::blast_pct() const { return blast_pct_; }
|
||
|
|
|
||
|
|
} // namespace molo::chaos
|