feat: implement MoloApp plans 01-15 with integration
Full C++23 framework tree: runtime, observability, security, data, app, realtime, supervisor, clients, CLI, example plus enterprise tenancy, audit, operability, governance, supplychain. Postgres via docker compose for tests.
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
// Standalone operability test runner. No framework, stdlib only: each
|
||||
// suite returns its failure count and main sums them.
|
||||
#include <cstdio>
|
||||
|
||||
int run_coord_tests();
|
||||
int run_lifecycle_tests();
|
||||
int run_breaker_bulkhead_tests();
|
||||
int run_shed_deadline_tests();
|
||||
int run_deploy_tests();
|
||||
int run_slo_tests();
|
||||
int run_chaos_tests();
|
||||
int run_debug_tests();
|
||||
|
||||
int main() {
|
||||
int fail = 0;
|
||||
fail += run_coord_tests();
|
||||
fail += run_lifecycle_tests();
|
||||
fail += run_breaker_bulkhead_tests();
|
||||
fail += run_shed_deadline_tests();
|
||||
fail += run_deploy_tests();
|
||||
fail += run_slo_tests();
|
||||
fail += run_chaos_tests();
|
||||
fail += run_debug_tests();
|
||||
if (fail == 0) {
|
||||
std::printf("operability: all suites passed\n");
|
||||
return 0;
|
||||
}
|
||||
std::printf("operability: %d check(s) failed\n", fail);
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Breaker and bulkhead tests: trip thresholds, half open probes, bulkhead
|
||||
// admission and promotion. Frozen clock throughout.
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/resilience/breaker.h"
|
||||
#include "operability/resilience/bulkhead.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
using State = molo::resilience::Breaker::State;
|
||||
|
||||
void test_breaker_consecutive_trip() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Breaker b(clock, obs, "svc");
|
||||
CHECK(b.state() == State::kClosed);
|
||||
CHECK(b.allow());
|
||||
// Pad the window with successes so the rate rule stays quiet and only
|
||||
// the 20 consecutive failures rule can trip.
|
||||
for (int i = 0; i < 100; ++i) b.on_success();
|
||||
for (int i = 0; i < 19; ++i) b.on_failure();
|
||||
CHECK(b.state() == State::kClosed);
|
||||
b.on_failure();
|
||||
CHECK(b.state() == State::kOpen);
|
||||
CHECK(!b.allow());
|
||||
}
|
||||
|
||||
void test_breaker_half_open_close() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Breaker b(clock, obs, "svc");
|
||||
for (int i = 0; i < 20; ++i) b.on_failure();
|
||||
CHECK(!b.allow());
|
||||
clock.advance(31000); // past the 30 s open hold
|
||||
CHECK(b.allow());
|
||||
CHECK(b.state() == State::kHalfOpen);
|
||||
b.on_success();
|
||||
// Only 3 probes may be inflight; use them one at a time.
|
||||
CHECK(b.allow());
|
||||
b.on_success();
|
||||
CHECK(b.allow());
|
||||
b.on_success();
|
||||
CHECK(b.state() == State::kClosed);
|
||||
CHECK(b.allow());
|
||||
}
|
||||
|
||||
void test_breaker_half_open_reopen() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Breaker b(clock, obs, "svc");
|
||||
for (int i = 0; i < 20; ++i) b.on_failure();
|
||||
clock.advance(31000);
|
||||
CHECK(b.allow());
|
||||
b.on_failure();
|
||||
CHECK(b.state() == State::kOpen);
|
||||
CHECK(!b.allow());
|
||||
}
|
||||
|
||||
void test_breaker_rate_trip() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Breaker b(clock, obs, "svc");
|
||||
for (int i = 0; i < 5; ++i) b.on_success();
|
||||
for (int i = 0; i < 4; ++i) b.on_failure();
|
||||
CHECK(b.state() == State::kClosed); // 4/9, below 50 percent
|
||||
b.on_failure(); // 5/10, hits 50 percent
|
||||
CHECK(b.state() == State::kOpen);
|
||||
}
|
||||
|
||||
void test_breaker_window_prunes_old_failures() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Breaker b(clock, obs, "svc");
|
||||
for (int i = 0; i < 5; ++i) b.on_failure();
|
||||
clock.advance(11000); // past the 10 s window
|
||||
b.on_success(); // resets consecutive count and prunes the old samples
|
||||
for (int i = 0; i < 4; ++i) b.on_failure();
|
||||
for (int i = 0; i < 5; ++i) b.on_success();
|
||||
// Window holds 10 samples with 4 failures: 40 percent, stays closed. If
|
||||
// the 5 expired failures were still counted it would read 9/15 and trip.
|
||||
CHECK(b.state() == State::kClosed);
|
||||
CHECK(b.allow());
|
||||
}
|
||||
|
||||
void test_breaker_success_resets_consecutive() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Breaker b(clock, obs, "svc");
|
||||
for (int i = 0; i < 200; ++i) b.on_success(); // keep the rate rule quiet
|
||||
for (int i = 0; i < 19; ++i) b.on_failure();
|
||||
b.on_success(); // breaks the streak: 19 + 1 + 19 must not trip 20
|
||||
for (int i = 0; i < 19; ++i) b.on_failure();
|
||||
CHECK(b.state() == State::kClosed);
|
||||
CHECK(b.allow());
|
||||
}
|
||||
|
||||
void test_bulkhead_admit_queue_reject() {
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Bulkhead bh(obs, 1, 2, 1);
|
||||
using A = molo::resilience::Bulkhead::Admission;
|
||||
CHECK(bh.try_acquire("p") == A::kAdmitted);
|
||||
CHECK(bh.try_acquire("p") == A::kAdmitted);
|
||||
CHECK(bh.inflight("p") == 2);
|
||||
CHECK(bh.try_acquire("p") == A::kQueued);
|
||||
CHECK(bh.queued("p") == 1);
|
||||
CHECK(bh.try_acquire("p") == A::kRejected);
|
||||
bh.release("p"); // promotes the queued request
|
||||
CHECK(bh.queued("p") == 0);
|
||||
CHECK(bh.inflight("p") == 2);
|
||||
bh.release("p");
|
||||
CHECK(bh.inflight("p") == 1);
|
||||
bh.release("p");
|
||||
CHECK(bh.inflight("p") == 0);
|
||||
}
|
||||
|
||||
void test_bulkhead_caps_partitions() {
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Bulkhead bh(obs, 100, 1000, 1000);
|
||||
CHECK(bh.partition_count() == 32);
|
||||
// Clamped to 64 concurrent: 64 admitted, then queued, never more inflight.
|
||||
int admitted = 0;
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
if (bh.try_acquire("solo") == molo::resilience::Bulkhead::Admission::kAdmitted) ++admitted;
|
||||
}
|
||||
CHECK(admitted == 64);
|
||||
CHECK(bh.inflight("solo") == 64);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_breaker_bulkhead_tests() {
|
||||
test_breaker_consecutive_trip();
|
||||
test_breaker_half_open_close();
|
||||
test_breaker_half_open_reopen();
|
||||
test_breaker_rate_trip();
|
||||
test_breaker_window_prunes_old_failures();
|
||||
test_breaker_success_resets_consecutive();
|
||||
test_bulkhead_admit_queue_reject();
|
||||
test_bulkhead_caps_partitions();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// Chaos tests: deterministic hooks, injector decisions and caps,
|
||||
// experiment blast radius, duration and burn abort.
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/chaos/experiment.h"
|
||||
#include "operability/chaos/hook.h"
|
||||
#include "operability/chaos/injector.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
using Action = molo::chaos::Hook::Action;
|
||||
using Decision = molo::chaos::Injector::Decision;
|
||||
|
||||
void test_hook_sampling() {
|
||||
molo::chaos::Hook always("a", Action::kFail, 100);
|
||||
molo::chaos::Hook never("n", Action::kFail, 0);
|
||||
molo::chaos::Hook none("x", Action::kNone, 100);
|
||||
for (std::uint64_t s = 0; s < 16; ++s) {
|
||||
CHECK(always.fires_for(s));
|
||||
CHECK(!never.fires_for(s));
|
||||
CHECK(!none.fires_for(s));
|
||||
}
|
||||
// Partial rate fires for some but not all of a large sweep.
|
||||
molo::chaos::Hook half("h", Action::kFail, 50);
|
||||
int fired = 0;
|
||||
for (std::uint64_t s = 0; s < 1000; ++s) {
|
||||
if (half.fires_for(s)) ++fired;
|
||||
}
|
||||
CHECK(fired > 300 && fired < 700);
|
||||
}
|
||||
|
||||
void test_injector_decisions() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::chaos::Injector inj(clock, obs);
|
||||
CHECK(inj.evaluate(1) == Decision::kProceed);
|
||||
CHECK(inj.add_hook(molo::chaos::Hook("slow", Action::kDelay, 100, 250)));
|
||||
CHECK(inj.evaluate(2) == Decision::kDelayed);
|
||||
CHECK(clock.steady_ms() == 250);
|
||||
CHECK(inj.add_hook(molo::chaos::Hook("boom", Action::kFail, 100)));
|
||||
CHECK(inj.evaluate(3) == Decision::kFailed);
|
||||
CHECK(inj.add_hook(molo::chaos::Hook("stop", Action::kAbort, 100)));
|
||||
CHECK(inj.evaluate(4) == Decision::kAborted);
|
||||
CHECK(inj.size() == 3);
|
||||
}
|
||||
|
||||
void test_injector_caps_and_dupes() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::chaos::Injector inj(clock, obs);
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
CHECK(inj.add_hook(molo::chaos::Hook("h" + std::to_string(i), Action::kFail, 0)));
|
||||
}
|
||||
CHECK(!inj.add_hook(molo::chaos::Hook("one-more", Action::kFail, 0)));
|
||||
CHECK(!inj.add_hook(molo::chaos::Hook("h0", Action::kFail, 0)));
|
||||
CHECK(!inj.add_hook(molo::chaos::Hook("", Action::kFail, 0)));
|
||||
CHECK(inj.remove_hook("h0"));
|
||||
CHECK(!inj.remove_hook("h0"));
|
||||
CHECK(inj.size() == 63);
|
||||
inj.clear();
|
||||
CHECK(inj.size() == 0);
|
||||
}
|
||||
|
||||
void test_experiment_blast_and_duration() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::chaos::Experiment ex(clock, obs, "latency", 100, 1000, 2.0);
|
||||
CHECK(!ex.should_apply(1)); // idle: never applies
|
||||
CHECK(ex.start());
|
||||
CHECK(!ex.start());
|
||||
CHECK(ex.should_apply(1));
|
||||
clock.advance(1001);
|
||||
CHECK(!ex.should_apply(2)); // duration elapsed, auto finished
|
||||
CHECK(ex.state() == molo::chaos::Experiment::State::kFinished);
|
||||
}
|
||||
|
||||
void test_experiment_zero_blast() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::chaos::Experiment ex(clock, obs, "off", 0, 0, 2.0);
|
||||
CHECK(ex.start());
|
||||
CHECK(!ex.should_apply(1));
|
||||
ex.finish();
|
||||
CHECK(ex.state() == molo::chaos::Experiment::State::kFinished);
|
||||
}
|
||||
|
||||
void test_experiment_burn_abort() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::chaos::Experiment ex(clock, obs, "risky", 10, 0, 1.0);
|
||||
CHECK(ex.start());
|
||||
CHECK(!ex.abort_if_breached([] { return 0.5; }));
|
||||
CHECK(ex.state() == molo::chaos::Experiment::State::kRunning);
|
||||
CHECK(ex.abort_if_breached([] { return 4.0; }));
|
||||
CHECK(ex.state() == molo::chaos::Experiment::State::kAborted);
|
||||
CHECK(!ex.should_apply(1));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_chaos_tests() {
|
||||
test_hook_sampling();
|
||||
test_injector_decisions();
|
||||
test_injector_caps_and_dupes();
|
||||
test_experiment_blast_and_duration();
|
||||
test_experiment_zero_blast();
|
||||
test_experiment_burn_abort();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// Coordination tests: lease, distributed lock, fence, leader election.
|
||||
// Frozen clock throughout; no real waiting.
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/coord/dist_lock.h"
|
||||
#include "operability/coord/election.h"
|
||||
#include "operability/coord/fence.h"
|
||||
#include "operability/coord/lease.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
void test_lease_basic() {
|
||||
molo::common::FrozenClock clock(1000);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::Lease a(kv, clock, obs, "k/a", "owner-a");
|
||||
molo::coord::Lease b(kv, clock, obs, "k/a", "owner-b");
|
||||
CHECK(a.acquire(5000));
|
||||
CHECK(a.holds());
|
||||
CHECK(!b.acquire(5000));
|
||||
CHECK(a.renew(5000));
|
||||
a.release();
|
||||
CHECK(!a.holds());
|
||||
CHECK(b.acquire(5000));
|
||||
CHECK(b.holds());
|
||||
}
|
||||
|
||||
void test_lease_expiry() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::Lease a(kv, clock, obs, "k/exp", "a");
|
||||
molo::coord::Lease b(kv, clock, obs, "k/exp", "b");
|
||||
CHECK(a.acquire(1000));
|
||||
clock.advance(1001);
|
||||
CHECK(b.acquire(1000));
|
||||
CHECK(!a.renew(1000));
|
||||
CHECK(!a.holds());
|
||||
}
|
||||
|
||||
void test_lease_rejects_bad_args() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::Lease bad(kv, clock, obs, "", "");
|
||||
CHECK(!bad.acquire(1000));
|
||||
molo::coord::Lease c(kv, clock, obs, "k/c", "c");
|
||||
CHECK(!c.acquire(0));
|
||||
CHECK(!c.renew(1000));
|
||||
CHECK(molo::coord::Lease::live_count() >= 2);
|
||||
}
|
||||
|
||||
void test_dist_lock() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::DistLock a(kv, clock, obs, "k/lock", "a");
|
||||
molo::coord::DistLock b(kv, clock, obs, "k/lock", "b");
|
||||
const std::uint64_t t1 = a.try_lock(60000, 0);
|
||||
CHECK(t1 != 0);
|
||||
CHECK(a.locked());
|
||||
CHECK(b.try_lock(60000, 0) == 0);
|
||||
a.unlock();
|
||||
CHECK(!a.locked());
|
||||
const std::uint64_t t2 = b.try_lock(60000, 100);
|
||||
CHECK(t2 != 0);
|
||||
CHECK(t2 > t1);
|
||||
b.unlock();
|
||||
}
|
||||
|
||||
void test_fence() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::coord::Fence f1(kv, "k/f");
|
||||
molo::coord::Fence f2(kv, "k/f");
|
||||
const std::uint64_t t1 = f1.next();
|
||||
const std::uint64_t t2 = f2.next();
|
||||
CHECK(t1 != 0 && t2 != 0 && t2 > t1);
|
||||
CHECK(f1.check(t2));
|
||||
CHECK(!f1.check(t1));
|
||||
}
|
||||
|
||||
void test_election() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::LeaderElection n1(kv, clock, obs, "molo/scheduler/leader", "n1");
|
||||
molo::coord::LeaderElection n2(kv, clock, obs, "molo/scheduler/leader", "n2");
|
||||
CHECK(n1.tick());
|
||||
CHECK(n1.is_leader());
|
||||
CHECK(!n2.tick());
|
||||
CHECK(!n2.is_leader());
|
||||
CHECK(n2.leader() == "n1");
|
||||
n1.resign();
|
||||
CHECK(!n1.is_leader());
|
||||
CHECK(n2.tick());
|
||||
CHECK(n2.is_leader());
|
||||
}
|
||||
|
||||
void test_election_failover_on_expiry() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::LeaderElection n1(kv, clock, obs, "e/fail", "n1", 10000, 3000);
|
||||
molo::coord::LeaderElection n2(kv, clock, obs, "e/fail", "n2", 10000, 3000);
|
||||
CHECK(n1.tick());
|
||||
clock.advance(11000); // n1 never renews
|
||||
CHECK(n2.tick());
|
||||
CHECK(n2.is_leader());
|
||||
CHECK(!n1.tick());
|
||||
CHECK(!n1.is_leader());
|
||||
}
|
||||
|
||||
void test_election_renew_keeps_leadership() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::coord::LeaderElection n1(kv, clock, obs, "e/keep", "n1", 10000, 3000);
|
||||
CHECK(n1.tick());
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
clock.advance(3000);
|
||||
CHECK(n1.tick());
|
||||
}
|
||||
CHECK(n1.is_leader());
|
||||
CHECK(n1.leader() == "n1");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_coord_tests() {
|
||||
test_lease_basic();
|
||||
test_lease_expiry();
|
||||
test_lease_rejects_bad_args();
|
||||
test_dist_lock();
|
||||
test_fence();
|
||||
test_election();
|
||||
test_election_failover_on_expiry();
|
||||
test_election_renew_keeps_leadership();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Debug tests: endpoint routing, auth gate, response caps and snapshots.
|
||||
// When MOLO_ENABLE_DEBUG is off the endpoint must 404 everything; the
|
||||
// standalone build runs this suite twice, once with the flag and once
|
||||
// without, to prove both sides.
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/debug/endpoint.h"
|
||||
#include "operability/debug/snapshot.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
void test_endpoint_disabled() {
|
||||
molo::debug::Endpoint ep([] { return std::string("{}"); });
|
||||
const auto r = ep.handle("/debug/ping", "");
|
||||
CHECK(r.status == 404);
|
||||
CHECK(ep.handle("/debug/snapshot", "").status == 404);
|
||||
}
|
||||
|
||||
void test_endpoint_routing() {
|
||||
molo::debug::Endpoint ep([] { return std::string("{\"ok\":true}"); });
|
||||
const auto ping = ep.handle("/debug/ping", "");
|
||||
CHECK(ping.status == 200);
|
||||
CHECK(ping.body == "pong");
|
||||
const auto snap = ep.handle("/debug/snapshot", "");
|
||||
CHECK(snap.status == 200);
|
||||
CHECK(snap.content_type == "application/json");
|
||||
CHECK(snap.body == "{\"ok\":true}");
|
||||
CHECK(ep.handle("/debug/nope", "").status == 404);
|
||||
CHECK(ep.handle("/metrics", "").status == 404);
|
||||
CHECK(ep.handle(std::string(600, 'a'), "").status == 414);
|
||||
}
|
||||
|
||||
void test_endpoint_auth() {
|
||||
molo::debug::Endpoint ep([] { return std::string("{}"); },
|
||||
[](const std::string& t) { return t == "secret"; });
|
||||
CHECK(ep.handle("/debug/ping", "wrong").status == 403);
|
||||
CHECK(ep.handle("/debug/ping", "secret").status == 200);
|
||||
}
|
||||
|
||||
void test_endpoint_truncates() {
|
||||
molo::debug::Endpoint ep([] { return std::string(2 * 1024 * 1024, 'x'); });
|
||||
const auto r = ep.handle("/debug/snapshot", "");
|
||||
CHECK(r.status == 200);
|
||||
CHECK(r.body.size() == 1024 * 1024);
|
||||
}
|
||||
|
||||
void test_snapshot_capture() {
|
||||
molo::common::FrozenClock clock(5000);
|
||||
molo::debug::Snapshot snap(clock, "1.2.3");
|
||||
CHECK(snap.add_section("breaker", "{\"open\":0}"));
|
||||
CHECK(!snap.add_section("breaker", "{}"));
|
||||
clock.advance(100);
|
||||
const std::string out = snap.capture();
|
||||
CHECK(out.find("\"version\":\"1.2.3\"") != std::string::npos);
|
||||
CHECK(out.find("\"wall_ms\":5100") != std::string::npos);
|
||||
CHECK(out.find("\"uptime_ms\":100") != std::string::npos);
|
||||
CHECK(out.find("\"breaker\":{\"open\":0}") != std::string::npos);
|
||||
CHECK(snap.section_count() == 1);
|
||||
CHECK(snap.remove_section("breaker"));
|
||||
CHECK(!snap.remove_section("breaker"));
|
||||
CHECK(snap.section_count() == 0);
|
||||
}
|
||||
|
||||
void test_snapshot_caps() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::debug::Snapshot snap(clock);
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
CHECK(snap.add_section("s" + std::to_string(i), "1"));
|
||||
}
|
||||
CHECK(!snap.add_section("overflow", "1"));
|
||||
CHECK(!snap.add_section("", "1"));
|
||||
molo::common::FrozenClock clock2(0);
|
||||
molo::debug::Snapshot big(clock2);
|
||||
CHECK(big.add_section("huge", "\"" + std::string(2 * 1024 * 1024, 'y') + "\""));
|
||||
CHECK(big.capture().size() == 1024 * 1024);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_debug_tests() {
|
||||
if (!molo::debug::Endpoint::enabled()) {
|
||||
test_endpoint_disabled();
|
||||
} else {
|
||||
test_endpoint_routing();
|
||||
test_endpoint_auth();
|
||||
test_endpoint_truncates();
|
||||
}
|
||||
test_snapshot_capture();
|
||||
test_snapshot_caps();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
// Deploy strategy tests: rolling batches with readiness gates and the
|
||||
// blue/green stage machine with rollback. Frozen clock throughout.
|
||||
#include <cstdio>
|
||||
|
||||
#include "operability/deploy/blue_green.h"
|
||||
#include "operability/deploy/rolling.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
using State = molo::deploy::Strategy::State;
|
||||
|
||||
void test_rolling_happy_path() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Rolling::Config cfg;
|
||||
cfg.total_instances = 4;
|
||||
cfg.batch_size = 2;
|
||||
cfg.pause_between_batches_ms = 100;
|
||||
int upgraded = 0;
|
||||
molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t) { return true; },
|
||||
[&](std::size_t) { ++upgraded; });
|
||||
CHECK(r.start());
|
||||
CHECK(!r.start()); // second start refuses
|
||||
CHECK(r.step());
|
||||
CHECK(r.step());
|
||||
CHECK(r.state() == State::kSucceeded);
|
||||
CHECK(r.upgraded() == 4);
|
||||
CHECK(r.failed() == 0);
|
||||
CHECK(r.pending() == 0);
|
||||
CHECK(upgraded == 4);
|
||||
CHECK(clock.steady_ms() >= 100); // batch pause observed
|
||||
}
|
||||
|
||||
void test_rolling_gate_failure() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Rolling::Config cfg;
|
||||
cfg.total_instances = 3;
|
||||
cfg.batch_size = 1;
|
||||
cfg.ready_timeout_ms = 200;
|
||||
molo::deploy::Rolling r(cfg, clock, obs,
|
||||
[](std::size_t i) { return i != 1; }); // index 1 never ready
|
||||
CHECK(r.start());
|
||||
CHECK(r.step()); // index 0 upgrades
|
||||
CHECK(!r.step()); // index 1 times out, max_unavailable is 0
|
||||
CHECK(r.state() == State::kFailed);
|
||||
CHECK(r.upgraded() == 1);
|
||||
CHECK(r.failed() == 1);
|
||||
}
|
||||
|
||||
void test_rolling_tolerates_limited_outage() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Rolling::Config cfg;
|
||||
cfg.total_instances = 3;
|
||||
cfg.batch_size = 1;
|
||||
cfg.ready_timeout_ms = 100;
|
||||
cfg.max_unavailable = 1;
|
||||
molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t i) { return i != 1; });
|
||||
CHECK(r.start());
|
||||
CHECK(r.step());
|
||||
CHECK(r.step()); // failure tolerated, run continues
|
||||
CHECK(!r.step()); // ends failed because one instance failed
|
||||
CHECK(r.upgraded() == 2);
|
||||
CHECK(r.failed() == 1);
|
||||
}
|
||||
|
||||
void test_rolling_rejects_empty() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Rolling::Config cfg;
|
||||
molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t) { return true; });
|
||||
CHECK(!r.start());
|
||||
}
|
||||
|
||||
void test_blue_green_happy_path() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::BlueGreen::Config cfg;
|
||||
cfg.green_instances = 2;
|
||||
cfg.rollback_window_ms = 1000;
|
||||
bool switched = false;
|
||||
bool retired_green = false;
|
||||
molo::deploy::BlueGreen bg(
|
||||
cfg, clock, obs, [](std::size_t) { return true; },
|
||||
[&] {
|
||||
switched = true;
|
||||
return true;
|
||||
},
|
||||
[&](bool green) { retired_green = green; });
|
||||
CHECK(bg.start());
|
||||
CHECK(bg.step()); // deploy green
|
||||
CHECK(bg.step()); // verify green
|
||||
CHECK(bg.step()); // switch
|
||||
CHECK(switched);
|
||||
CHECK(bg.serving_green());
|
||||
CHECK(bg.step()); // still inside the rollback window
|
||||
CHECK(bg.state() == State::kRunning);
|
||||
clock.advance(1001);
|
||||
CHECK(bg.step()); // window elapsed, move to retire
|
||||
CHECK(bg.step()); // retire blue
|
||||
CHECK(bg.state() == State::kSucceeded);
|
||||
CHECK(retired_green);
|
||||
}
|
||||
|
||||
void test_blue_green_rollback() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::BlueGreen::Config cfg;
|
||||
cfg.green_instances = 1;
|
||||
cfg.rollback_window_ms = 60000;
|
||||
molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return true; });
|
||||
CHECK(bg.start());
|
||||
CHECK(bg.step());
|
||||
CHECK(bg.step());
|
||||
CHECK(bg.step());
|
||||
CHECK(bg.serving_green());
|
||||
CHECK(bg.rollback());
|
||||
CHECK(!bg.serving_green());
|
||||
CHECK(bg.state() == State::kRolledBack);
|
||||
CHECK(!bg.rollback()); // second rollback refuses
|
||||
}
|
||||
|
||||
void test_blue_green_verify_timeout() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::BlueGreen::Config cfg;
|
||||
cfg.green_instances = 1;
|
||||
cfg.ready_timeout_ms = 100;
|
||||
molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return false; });
|
||||
CHECK(bg.start());
|
||||
CHECK(bg.step());
|
||||
CHECK(!bg.step()); // green never ready
|
||||
CHECK(bg.state() == State::kFailed);
|
||||
CHECK(!bg.serving_green());
|
||||
}
|
||||
|
||||
void test_blue_green_switch_refused() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::BlueGreen::Config cfg;
|
||||
cfg.green_instances = 1;
|
||||
molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return true; },
|
||||
[] { return false; });
|
||||
CHECK(bg.start());
|
||||
CHECK(bg.step());
|
||||
CHECK(bg.step());
|
||||
CHECK(!bg.step()); // switch hook refuses
|
||||
CHECK(bg.state() == State::kFailed);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_deploy_tests() {
|
||||
test_rolling_happy_path();
|
||||
test_rolling_gate_failure();
|
||||
test_rolling_tolerates_limited_outage();
|
||||
test_rolling_rejects_empty();
|
||||
test_blue_green_happy_path();
|
||||
test_blue_green_rollback();
|
||||
test_blue_green_verify_timeout();
|
||||
test_blue_green_switch_refused();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Lifecycle tests: boot completion and the four phase drain.
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/deploy/lifecycle.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
void test_mark_serving() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Lifecycle lc(clock, obs);
|
||||
bool called = false;
|
||||
lc.on_serving([&] { called = true; });
|
||||
lc.mark_serving();
|
||||
CHECK(called);
|
||||
CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kServing);
|
||||
}
|
||||
|
||||
void test_clean_drain() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Lifecycle lc(clock, obs);
|
||||
bool stopped = false;
|
||||
bool done = false;
|
||||
lc.on_stop_listeners([&] { stopped = true; });
|
||||
lc.on_connections_drained([] { return true; });
|
||||
lc.on_background_drained([] { return true; });
|
||||
lc.on_complete([&] { done = true; });
|
||||
CHECK(lc.run_drain(1000));
|
||||
CHECK(stopped && done);
|
||||
CHECK(lc.clean());
|
||||
CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kDone);
|
||||
}
|
||||
|
||||
void test_delayed_drain() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Lifecycle lc(clock, obs);
|
||||
// Probes flip true once the frozen clock passes their marks; the drain
|
||||
// loop advances the clock via sleep_ms, so no real waiting happens.
|
||||
lc.on_connections_drained([&] { return clock.steady_ms() >= 200; });
|
||||
lc.on_background_drained([&] { return clock.steady_ms() >= 400; });
|
||||
CHECK(lc.run_drain(5000, 50));
|
||||
CHECK(lc.clean());
|
||||
CHECK(clock.steady_ms() >= 400);
|
||||
}
|
||||
|
||||
void test_drain_timeout_drops_background() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Lifecycle lc(clock, obs);
|
||||
bool dropped = false;
|
||||
lc.on_connections_drained([] { return true; });
|
||||
lc.on_background_drained([] { return false; });
|
||||
lc.on_drop_background([&](const std::string&) { dropped = true; });
|
||||
CHECK(!lc.run_drain(200, 25));
|
||||
CHECK(!lc.clean());
|
||||
CHECK(dropped);
|
||||
CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kDone);
|
||||
}
|
||||
|
||||
void test_drain_timeout_connections() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::deploy::Lifecycle lc(clock, obs);
|
||||
lc.on_connections_drained([] { return false; });
|
||||
lc.on_background_drained([] { return true; });
|
||||
CHECK(!lc.run_drain(100, 25));
|
||||
CHECK(!lc.clean());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_lifecycle_tests() {
|
||||
test_mark_serving();
|
||||
test_clean_drain();
|
||||
test_delayed_drain();
|
||||
test_drain_timeout_drops_background();
|
||||
test_drain_timeout_connections();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Shedder and deadline tests: overload shedding with Retry-After plus
|
||||
// X-Deadline-Ms parsing and remaining budget math.
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/resilience/deadline.h"
|
||||
#include "operability/resilience/shedder.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
void test_shedderadmits_when_cool() {
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Shedder s(obs, [] { return false; });
|
||||
CHECK(!s.should_shed());
|
||||
}
|
||||
|
||||
void test_shedder_sheds_with_retry_after() {
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Shedder s(obs, [] { return true; }, 7);
|
||||
CHECK(s.should_shed());
|
||||
const auto r = s.reject();
|
||||
CHECK(r.status == 503);
|
||||
CHECK(r.retry_after == "7");
|
||||
CHECK(s.retry_after_s() == 7);
|
||||
}
|
||||
|
||||
void test_shedder_clamps_retry_after() {
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Shedder hi(obs, [] { return true; }, 99999);
|
||||
CHECK(hi.retry_after_s() == 120);
|
||||
molo::resilience::Shedder lo(obs, [] { return true; }, -5);
|
||||
CHECK(lo.retry_after_s() == 0);
|
||||
}
|
||||
|
||||
void test_deadline_parse() {
|
||||
const auto ok = molo::resilience::Deadline::parse_header("12345");
|
||||
CHECK(ok.has_value() && *ok == 12345);
|
||||
CHECK(!molo::resilience::Deadline::parse_header("").has_value());
|
||||
CHECK(!molo::resilience::Deadline::parse_header("abc").has_value());
|
||||
CHECK(!molo::resilience::Deadline::parse_header("12x").has_value());
|
||||
CHECK(!molo::resilience::Deadline::parse_header("99999999999999999999999").has_value());
|
||||
CHECK(molo::resilience::Deadline::format_header(42) == "42");
|
||||
CHECK(molo::resilience::Deadline::format_header(-1) == "0");
|
||||
}
|
||||
|
||||
void test_deadline_remaining() {
|
||||
molo::common::FrozenClock clock(100000);
|
||||
molo::resilience::Deadline d(clock);
|
||||
CHECK(d.remaining_ms(100500) == 450); // 500 left minus the 50 ms margin
|
||||
CHECK(d.remaining_ms(100050) == 0); // exactly the margin: unusable
|
||||
CHECK(d.remaining_ms(100040) == 0); // inside the margin
|
||||
CHECK(d.remaining_ms(90000) == 0); // already past
|
||||
CHECK(!d.expired(100500));
|
||||
CHECK(d.expired(100040));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_shed_deadline_tests() {
|
||||
test_shedderadmits_when_cool();
|
||||
test_shedder_sheds_with_retry_after();
|
||||
test_shedder_clamps_retry_after();
|
||||
test_deadline_parse();
|
||||
test_deadline_remaining();
|
||||
return g_fail;
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// SLO tests: budgets, burn rates, bad event counters, prometheus render.
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
|
||||
#include "operability/resilience/slo.h"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int g_fail = 0;
|
||||
|
||||
#define CHECK(cond) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
++g_fail; \
|
||||
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
bool near(double a, double b) {
|
||||
const double d = a > b ? a - b : b - a;
|
||||
return d < 0.0001;
|
||||
}
|
||||
|
||||
void test_budget_and_burn() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Slo::Config cfg;
|
||||
cfg.target_ratio = 0.9;
|
||||
cfg.window_ms = 3600000;
|
||||
molo::resilience::Slo slo(clock, obs, cfg);
|
||||
for (int i = 0; i < 19; ++i) slo.record("/api/items", true);
|
||||
slo.record("/api/items", false, "boom");
|
||||
// 20 samples, 1 error, 10 percent budget: half the budget remains.
|
||||
CHECK(near(slo.budget_remaining("/api/items"), 0.5));
|
||||
// Observed 5 percent over budgeted 10 percent: burn 0.5.
|
||||
CHECK(near(slo.burn_rate("/api/items", 3600000), 0.5));
|
||||
CHECK(slo.bad_events("/api/items", "boom") == 1);
|
||||
CHECK(slo.bad_events("/api/items", "other") == 0);
|
||||
}
|
||||
|
||||
void test_unknown_route_is_healthy() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Slo slo(clock, obs);
|
||||
CHECK(near(slo.budget_remaining("/nope"), 1.0));
|
||||
CHECK(near(slo.burn_rate("/nope", 60000), 0.0));
|
||||
}
|
||||
|
||||
void test_budget_exhausted() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Slo::Config cfg;
|
||||
cfg.target_ratio = 0.99;
|
||||
molo::resilience::Slo slo(clock, obs, cfg);
|
||||
for (int i = 0; i < 10; ++i) slo.record("/bad", false, "down");
|
||||
CHECK(near(slo.budget_remaining("/bad"), 0.0));
|
||||
CHECK(slo.burn_rate("/bad", 3600000) > 1.0);
|
||||
}
|
||||
|
||||
void test_window_pruning_restores_budget() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Slo::Config cfg;
|
||||
cfg.target_ratio = 0.9;
|
||||
cfg.window_ms = 1000;
|
||||
molo::resilience::Slo slo(clock, obs, cfg);
|
||||
slo.record("/flap", false, "x");
|
||||
CHECK(near(slo.budget_remaining("/flap"), 0.0));
|
||||
clock.advance(2000);
|
||||
CHECK(near(slo.budget_remaining("/flap"), 1.0));
|
||||
CHECK(near(slo.burn_rate("/flap", 1000), 0.0));
|
||||
}
|
||||
|
||||
void test_prometheus_render() {
|
||||
molo::common::FrozenClock clock(0);
|
||||
molo::common::RecordingObserver obs;
|
||||
molo::resilience::Slo slo(clock, obs);
|
||||
slo.record("/api/items", true);
|
||||
slo.record("/api/items", false, "boom");
|
||||
const std::string out = slo.render_prometheus();
|
||||
CHECK(out.find("molo_slo_budget_remaining_ratio") != std::string::npos);
|
||||
CHECK(out.find("molo_slo_burn_rate") != std::string::npos);
|
||||
CHECK(out.find("molo_slo_bad_events_total") != std::string::npos);
|
||||
CHECK(out.find("route=\"/api/items\"") != std::string::npos);
|
||||
CHECK(out.find("reason=\"boom\"") != std::string::npos);
|
||||
CHECK(out.find("window=\"5m\"") != std::string::npos);
|
||||
CHECK(out.find("window=\"1h\"") != std::string::npos);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int run_slo_tests() {
|
||||
test_budget_and_burn();
|
||||
test_unknown_route_is_healthy();
|
||||
test_budget_exhausted();
|
||||
test_window_pruning_restores_budget();
|
||||
test_prometheus_render();
|
||||
return g_fail;
|
||||
}
|
||||
Reference in New Issue
Block a user