feat: implement MoloApp plans 01-15 with integration

Full C++23 framework tree: runtime, observability, security,
data, app, realtime, supervisor, clients, CLI, example plus
enterprise tenancy, audit, operability, governance,
supplychain. Postgres via docker compose for tests.
This commit is contained in:
2026-10-23 18:20:04 +02:00
parent 54068f1953
commit 3c4334cec9
1068 changed files with 100899 additions and 2 deletions
+30
View File
@@ -0,0 +1,30 @@
// Standalone operability test runner. No framework, stdlib only: each
// suite returns its failure count and main sums them.
#include <cstdio>
int run_coord_tests();
int run_lifecycle_tests();
int run_breaker_bulkhead_tests();
int run_shed_deadline_tests();
int run_deploy_tests();
int run_slo_tests();
int run_chaos_tests();
int run_debug_tests();
int main() {
int fail = 0;
fail += run_coord_tests();
fail += run_lifecycle_tests();
fail += run_breaker_bulkhead_tests();
fail += run_shed_deadline_tests();
fail += run_deploy_tests();
fail += run_slo_tests();
fail += run_chaos_tests();
fail += run_debug_tests();
if (fail == 0) {
std::printf("operability: all suites passed\n");
return 0;
}
std::printf("operability: %d check(s) failed\n", fail);
return 1;
}
@@ -0,0 +1,154 @@
// Breaker and bulkhead tests: trip thresholds, half open probes, bulkhead
// admission and promotion. Frozen clock throughout.
#include <cstdio>
#include <string>
#include "operability/resilience/breaker.h"
#include "operability/resilience/bulkhead.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
using State = molo::resilience::Breaker::State;
void test_breaker_consecutive_trip() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Breaker b(clock, obs, "svc");
CHECK(b.state() == State::kClosed);
CHECK(b.allow());
// Pad the window with successes so the rate rule stays quiet and only
// the 20 consecutive failures rule can trip.
for (int i = 0; i < 100; ++i) b.on_success();
for (int i = 0; i < 19; ++i) b.on_failure();
CHECK(b.state() == State::kClosed);
b.on_failure();
CHECK(b.state() == State::kOpen);
CHECK(!b.allow());
}
void test_breaker_half_open_close() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Breaker b(clock, obs, "svc");
for (int i = 0; i < 20; ++i) b.on_failure();
CHECK(!b.allow());
clock.advance(31000); // past the 30 s open hold
CHECK(b.allow());
CHECK(b.state() == State::kHalfOpen);
b.on_success();
// Only 3 probes may be inflight; use them one at a time.
CHECK(b.allow());
b.on_success();
CHECK(b.allow());
b.on_success();
CHECK(b.state() == State::kClosed);
CHECK(b.allow());
}
void test_breaker_half_open_reopen() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Breaker b(clock, obs, "svc");
for (int i = 0; i < 20; ++i) b.on_failure();
clock.advance(31000);
CHECK(b.allow());
b.on_failure();
CHECK(b.state() == State::kOpen);
CHECK(!b.allow());
}
void test_breaker_rate_trip() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Breaker b(clock, obs, "svc");
for (int i = 0; i < 5; ++i) b.on_success();
for (int i = 0; i < 4; ++i) b.on_failure();
CHECK(b.state() == State::kClosed); // 4/9, below 50 percent
b.on_failure(); // 5/10, hits 50 percent
CHECK(b.state() == State::kOpen);
}
void test_breaker_window_prunes_old_failures() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Breaker b(clock, obs, "svc");
for (int i = 0; i < 5; ++i) b.on_failure();
clock.advance(11000); // past the 10 s window
b.on_success(); // resets consecutive count and prunes the old samples
for (int i = 0; i < 4; ++i) b.on_failure();
for (int i = 0; i < 5; ++i) b.on_success();
// Window holds 10 samples with 4 failures: 40 percent, stays closed. If
// the 5 expired failures were still counted it would read 9/15 and trip.
CHECK(b.state() == State::kClosed);
CHECK(b.allow());
}
void test_breaker_success_resets_consecutive() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Breaker b(clock, obs, "svc");
for (int i = 0; i < 200; ++i) b.on_success(); // keep the rate rule quiet
for (int i = 0; i < 19; ++i) b.on_failure();
b.on_success(); // breaks the streak: 19 + 1 + 19 must not trip 20
for (int i = 0; i < 19; ++i) b.on_failure();
CHECK(b.state() == State::kClosed);
CHECK(b.allow());
}
void test_bulkhead_admit_queue_reject() {
molo::common::RecordingObserver obs;
molo::resilience::Bulkhead bh(obs, 1, 2, 1);
using A = molo::resilience::Bulkhead::Admission;
CHECK(bh.try_acquire("p") == A::kAdmitted);
CHECK(bh.try_acquire("p") == A::kAdmitted);
CHECK(bh.inflight("p") == 2);
CHECK(bh.try_acquire("p") == A::kQueued);
CHECK(bh.queued("p") == 1);
CHECK(bh.try_acquire("p") == A::kRejected);
bh.release("p"); // promotes the queued request
CHECK(bh.queued("p") == 0);
CHECK(bh.inflight("p") == 2);
bh.release("p");
CHECK(bh.inflight("p") == 1);
bh.release("p");
CHECK(bh.inflight("p") == 0);
}
void test_bulkhead_caps_partitions() {
molo::common::RecordingObserver obs;
molo::resilience::Bulkhead bh(obs, 100, 1000, 1000);
CHECK(bh.partition_count() == 32);
// Clamped to 64 concurrent: 64 admitted, then queued, never more inflight.
int admitted = 0;
for (int i = 0; i < 64; ++i) {
if (bh.try_acquire("solo") == molo::resilience::Bulkhead::Admission::kAdmitted) ++admitted;
}
CHECK(admitted == 64);
CHECK(bh.inflight("solo") == 64);
}
} // namespace
int run_breaker_bulkhead_tests() {
test_breaker_consecutive_trip();
test_breaker_half_open_close();
test_breaker_half_open_reopen();
test_breaker_rate_trip();
test_breaker_window_prunes_old_failures();
test_breaker_success_resets_consecutive();
test_bulkhead_admit_queue_reject();
test_bulkhead_caps_partitions();
return g_fail;
}
+122
View File
@@ -0,0 +1,122 @@
// Chaos tests: deterministic hooks, injector decisions and caps,
// experiment blast radius, duration and burn abort.
#include <cstdio>
#include <string>
#include "operability/chaos/experiment.h"
#include "operability/chaos/hook.h"
#include "operability/chaos/injector.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
using Action = molo::chaos::Hook::Action;
using Decision = molo::chaos::Injector::Decision;
void test_hook_sampling() {
molo::chaos::Hook always("a", Action::kFail, 100);
molo::chaos::Hook never("n", Action::kFail, 0);
molo::chaos::Hook none("x", Action::kNone, 100);
for (std::uint64_t s = 0; s < 16; ++s) {
CHECK(always.fires_for(s));
CHECK(!never.fires_for(s));
CHECK(!none.fires_for(s));
}
// Partial rate fires for some but not all of a large sweep.
molo::chaos::Hook half("h", Action::kFail, 50);
int fired = 0;
for (std::uint64_t s = 0; s < 1000; ++s) {
if (half.fires_for(s)) ++fired;
}
CHECK(fired > 300 && fired < 700);
}
void test_injector_decisions() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::chaos::Injector inj(clock, obs);
CHECK(inj.evaluate(1) == Decision::kProceed);
CHECK(inj.add_hook(molo::chaos::Hook("slow", Action::kDelay, 100, 250)));
CHECK(inj.evaluate(2) == Decision::kDelayed);
CHECK(clock.steady_ms() == 250);
CHECK(inj.add_hook(molo::chaos::Hook("boom", Action::kFail, 100)));
CHECK(inj.evaluate(3) == Decision::kFailed);
CHECK(inj.add_hook(molo::chaos::Hook("stop", Action::kAbort, 100)));
CHECK(inj.evaluate(4) == Decision::kAborted);
CHECK(inj.size() == 3);
}
void test_injector_caps_and_dupes() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::chaos::Injector inj(clock, obs);
for (int i = 0; i < 64; ++i) {
CHECK(inj.add_hook(molo::chaos::Hook("h" + std::to_string(i), Action::kFail, 0)));
}
CHECK(!inj.add_hook(molo::chaos::Hook("one-more", Action::kFail, 0)));
CHECK(!inj.add_hook(molo::chaos::Hook("h0", Action::kFail, 0)));
CHECK(!inj.add_hook(molo::chaos::Hook("", Action::kFail, 0)));
CHECK(inj.remove_hook("h0"));
CHECK(!inj.remove_hook("h0"));
CHECK(inj.size() == 63);
inj.clear();
CHECK(inj.size() == 0);
}
void test_experiment_blast_and_duration() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::chaos::Experiment ex(clock, obs, "latency", 100, 1000, 2.0);
CHECK(!ex.should_apply(1)); // idle: never applies
CHECK(ex.start());
CHECK(!ex.start());
CHECK(ex.should_apply(1));
clock.advance(1001);
CHECK(!ex.should_apply(2)); // duration elapsed, auto finished
CHECK(ex.state() == molo::chaos::Experiment::State::kFinished);
}
void test_experiment_zero_blast() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::chaos::Experiment ex(clock, obs, "off", 0, 0, 2.0);
CHECK(ex.start());
CHECK(!ex.should_apply(1));
ex.finish();
CHECK(ex.state() == molo::chaos::Experiment::State::kFinished);
}
void test_experiment_burn_abort() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::chaos::Experiment ex(clock, obs, "risky", 10, 0, 1.0);
CHECK(ex.start());
CHECK(!ex.abort_if_breached([] { return 0.5; }));
CHECK(ex.state() == molo::chaos::Experiment::State::kRunning);
CHECK(ex.abort_if_breached([] { return 4.0; }));
CHECK(ex.state() == molo::chaos::Experiment::State::kAborted);
CHECK(!ex.should_apply(1));
}
} // namespace
int run_chaos_tests() {
test_hook_sampling();
test_injector_decisions();
test_injector_caps_and_dupes();
test_experiment_blast_and_duration();
test_experiment_zero_blast();
test_experiment_burn_abort();
return g_fail;
}
+155
View File
@@ -0,0 +1,155 @@
// Coordination tests: lease, distributed lock, fence, leader election.
// Frozen clock throughout; no real waiting.
#include <cstdint>
#include <cstdio>
#include <string>
#include "operability/coord/dist_lock.h"
#include "operability/coord/election.h"
#include "operability/coord/fence.h"
#include "operability/coord/lease.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/kv_store.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
void test_lease_basic() {
molo::common::FrozenClock clock(1000);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::Lease a(kv, clock, obs, "k/a", "owner-a");
molo::coord::Lease b(kv, clock, obs, "k/a", "owner-b");
CHECK(a.acquire(5000));
CHECK(a.holds());
CHECK(!b.acquire(5000));
CHECK(a.renew(5000));
a.release();
CHECK(!a.holds());
CHECK(b.acquire(5000));
CHECK(b.holds());
}
void test_lease_expiry() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::Lease a(kv, clock, obs, "k/exp", "a");
molo::coord::Lease b(kv, clock, obs, "k/exp", "b");
CHECK(a.acquire(1000));
clock.advance(1001);
CHECK(b.acquire(1000));
CHECK(!a.renew(1000));
CHECK(!a.holds());
}
void test_lease_rejects_bad_args() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::Lease bad(kv, clock, obs, "", "");
CHECK(!bad.acquire(1000));
molo::coord::Lease c(kv, clock, obs, "k/c", "c");
CHECK(!c.acquire(0));
CHECK(!c.renew(1000));
CHECK(molo::coord::Lease::live_count() >= 2);
}
void test_dist_lock() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::DistLock a(kv, clock, obs, "k/lock", "a");
molo::coord::DistLock b(kv, clock, obs, "k/lock", "b");
const std::uint64_t t1 = a.try_lock(60000, 0);
CHECK(t1 != 0);
CHECK(a.locked());
CHECK(b.try_lock(60000, 0) == 0);
a.unlock();
CHECK(!a.locked());
const std::uint64_t t2 = b.try_lock(60000, 100);
CHECK(t2 != 0);
CHECK(t2 > t1);
b.unlock();
}
void test_fence() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::coord::Fence f1(kv, "k/f");
molo::coord::Fence f2(kv, "k/f");
const std::uint64_t t1 = f1.next();
const std::uint64_t t2 = f2.next();
CHECK(t1 != 0 && t2 != 0 && t2 > t1);
CHECK(f1.check(t2));
CHECK(!f1.check(t1));
}
void test_election() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::LeaderElection n1(kv, clock, obs, "molo/scheduler/leader", "n1");
molo::coord::LeaderElection n2(kv, clock, obs, "molo/scheduler/leader", "n2");
CHECK(n1.tick());
CHECK(n1.is_leader());
CHECK(!n2.tick());
CHECK(!n2.is_leader());
CHECK(n2.leader() == "n1");
n1.resign();
CHECK(!n1.is_leader());
CHECK(n2.tick());
CHECK(n2.is_leader());
}
void test_election_failover_on_expiry() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::LeaderElection n1(kv, clock, obs, "e/fail", "n1", 10000, 3000);
molo::coord::LeaderElection n2(kv, clock, obs, "e/fail", "n2", 10000, 3000);
CHECK(n1.tick());
clock.advance(11000); // n1 never renews
CHECK(n2.tick());
CHECK(n2.is_leader());
CHECK(!n1.tick());
CHECK(!n1.is_leader());
}
void test_election_renew_keeps_leadership() {
molo::common::FrozenClock clock(0);
molo::common::InMemoryKvStore kv(clock);
molo::common::RecordingObserver obs;
molo::coord::LeaderElection n1(kv, clock, obs, "e/keep", "n1", 10000, 3000);
CHECK(n1.tick());
for (int i = 0; i < 10; ++i) {
clock.advance(3000);
CHECK(n1.tick());
}
CHECK(n1.is_leader());
CHECK(n1.leader() == "n1");
}
} // namespace
int run_coord_tests() {
test_lease_basic();
test_lease_expiry();
test_lease_rejects_bad_args();
test_dist_lock();
test_fence();
test_election();
test_election_failover_on_expiry();
test_election_renew_keeps_leadership();
return g_fail;
}
+103
View File
@@ -0,0 +1,103 @@
// Debug tests: endpoint routing, auth gate, response caps and snapshots.
// When MOLO_ENABLE_DEBUG is off the endpoint must 404 everything; the
// standalone build runs this suite twice, once with the flag and once
// without, to prove both sides.
#include <cstdio>
#include <string>
#include "operability/debug/endpoint.h"
#include "operability/debug/snapshot.h"
#include "supervisor/common/clock.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
void test_endpoint_disabled() {
molo::debug::Endpoint ep([] { return std::string("{}"); });
const auto r = ep.handle("/debug/ping", "");
CHECK(r.status == 404);
CHECK(ep.handle("/debug/snapshot", "").status == 404);
}
void test_endpoint_routing() {
molo::debug::Endpoint ep([] { return std::string("{\"ok\":true}"); });
const auto ping = ep.handle("/debug/ping", "");
CHECK(ping.status == 200);
CHECK(ping.body == "pong");
const auto snap = ep.handle("/debug/snapshot", "");
CHECK(snap.status == 200);
CHECK(snap.content_type == "application/json");
CHECK(snap.body == "{\"ok\":true}");
CHECK(ep.handle("/debug/nope", "").status == 404);
CHECK(ep.handle("/metrics", "").status == 404);
CHECK(ep.handle(std::string(600, 'a'), "").status == 414);
}
void test_endpoint_auth() {
molo::debug::Endpoint ep([] { return std::string("{}"); },
[](const std::string& t) { return t == "secret"; });
CHECK(ep.handle("/debug/ping", "wrong").status == 403);
CHECK(ep.handle("/debug/ping", "secret").status == 200);
}
void test_endpoint_truncates() {
molo::debug::Endpoint ep([] { return std::string(2 * 1024 * 1024, 'x'); });
const auto r = ep.handle("/debug/snapshot", "");
CHECK(r.status == 200);
CHECK(r.body.size() == 1024 * 1024);
}
void test_snapshot_capture() {
molo::common::FrozenClock clock(5000);
molo::debug::Snapshot snap(clock, "1.2.3");
CHECK(snap.add_section("breaker", "{\"open\":0}"));
CHECK(!snap.add_section("breaker", "{}"));
clock.advance(100);
const std::string out = snap.capture();
CHECK(out.find("\"version\":\"1.2.3\"") != std::string::npos);
CHECK(out.find("\"wall_ms\":5100") != std::string::npos);
CHECK(out.find("\"uptime_ms\":100") != std::string::npos);
CHECK(out.find("\"breaker\":{\"open\":0}") != std::string::npos);
CHECK(snap.section_count() == 1);
CHECK(snap.remove_section("breaker"));
CHECK(!snap.remove_section("breaker"));
CHECK(snap.section_count() == 0);
}
void test_snapshot_caps() {
molo::common::FrozenClock clock(0);
molo::debug::Snapshot snap(clock);
for (int i = 0; i < 64; ++i) {
CHECK(snap.add_section("s" + std::to_string(i), "1"));
}
CHECK(!snap.add_section("overflow", "1"));
CHECK(!snap.add_section("", "1"));
molo::common::FrozenClock clock2(0);
molo::debug::Snapshot big(clock2);
CHECK(big.add_section("huge", "\"" + std::string(2 * 1024 * 1024, 'y') + "\""));
CHECK(big.capture().size() == 1024 * 1024);
}
} // namespace
int run_debug_tests() {
if (!molo::debug::Endpoint::enabled()) {
test_endpoint_disabled();
} else {
test_endpoint_routing();
test_endpoint_auth();
test_endpoint_truncates();
}
test_snapshot_capture();
test_snapshot_caps();
return g_fail;
}
+176
View File
@@ -0,0 +1,176 @@
// Deploy strategy tests: rolling batches with readiness gates and the
// blue/green stage machine with rollback. Frozen clock throughout.
#include <cstdio>
#include "operability/deploy/blue_green.h"
#include "operability/deploy/rolling.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
using State = molo::deploy::Strategy::State;
void test_rolling_happy_path() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Rolling::Config cfg;
cfg.total_instances = 4;
cfg.batch_size = 2;
cfg.pause_between_batches_ms = 100;
int upgraded = 0;
molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t) { return true; },
[&](std::size_t) { ++upgraded; });
CHECK(r.start());
CHECK(!r.start()); // second start refuses
CHECK(r.step());
CHECK(r.step());
CHECK(r.state() == State::kSucceeded);
CHECK(r.upgraded() == 4);
CHECK(r.failed() == 0);
CHECK(r.pending() == 0);
CHECK(upgraded == 4);
CHECK(clock.steady_ms() >= 100); // batch pause observed
}
void test_rolling_gate_failure() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Rolling::Config cfg;
cfg.total_instances = 3;
cfg.batch_size = 1;
cfg.ready_timeout_ms = 200;
molo::deploy::Rolling r(cfg, clock, obs,
[](std::size_t i) { return i != 1; }); // index 1 never ready
CHECK(r.start());
CHECK(r.step()); // index 0 upgrades
CHECK(!r.step()); // index 1 times out, max_unavailable is 0
CHECK(r.state() == State::kFailed);
CHECK(r.upgraded() == 1);
CHECK(r.failed() == 1);
}
void test_rolling_tolerates_limited_outage() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Rolling::Config cfg;
cfg.total_instances = 3;
cfg.batch_size = 1;
cfg.ready_timeout_ms = 100;
cfg.max_unavailable = 1;
molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t i) { return i != 1; });
CHECK(r.start());
CHECK(r.step());
CHECK(r.step()); // failure tolerated, run continues
CHECK(!r.step()); // ends failed because one instance failed
CHECK(r.upgraded() == 2);
CHECK(r.failed() == 1);
}
void test_rolling_rejects_empty() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Rolling::Config cfg;
molo::deploy::Rolling r(cfg, clock, obs, [](std::size_t) { return true; });
CHECK(!r.start());
}
void test_blue_green_happy_path() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::BlueGreen::Config cfg;
cfg.green_instances = 2;
cfg.rollback_window_ms = 1000;
bool switched = false;
bool retired_green = false;
molo::deploy::BlueGreen bg(
cfg, clock, obs, [](std::size_t) { return true; },
[&] {
switched = true;
return true;
},
[&](bool green) { retired_green = green; });
CHECK(bg.start());
CHECK(bg.step()); // deploy green
CHECK(bg.step()); // verify green
CHECK(bg.step()); // switch
CHECK(switched);
CHECK(bg.serving_green());
CHECK(bg.step()); // still inside the rollback window
CHECK(bg.state() == State::kRunning);
clock.advance(1001);
CHECK(bg.step()); // window elapsed, move to retire
CHECK(bg.step()); // retire blue
CHECK(bg.state() == State::kSucceeded);
CHECK(retired_green);
}
void test_blue_green_rollback() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::BlueGreen::Config cfg;
cfg.green_instances = 1;
cfg.rollback_window_ms = 60000;
molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return true; });
CHECK(bg.start());
CHECK(bg.step());
CHECK(bg.step());
CHECK(bg.step());
CHECK(bg.serving_green());
CHECK(bg.rollback());
CHECK(!bg.serving_green());
CHECK(bg.state() == State::kRolledBack);
CHECK(!bg.rollback()); // second rollback refuses
}
void test_blue_green_verify_timeout() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::BlueGreen::Config cfg;
cfg.green_instances = 1;
cfg.ready_timeout_ms = 100;
molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return false; });
CHECK(bg.start());
CHECK(bg.step());
CHECK(!bg.step()); // green never ready
CHECK(bg.state() == State::kFailed);
CHECK(!bg.serving_green());
}
void test_blue_green_switch_refused() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::BlueGreen::Config cfg;
cfg.green_instances = 1;
molo::deploy::BlueGreen bg(cfg, clock, obs, [](std::size_t) { return true; },
[] { return false; });
CHECK(bg.start());
CHECK(bg.step());
CHECK(bg.step());
CHECK(!bg.step()); // switch hook refuses
CHECK(bg.state() == State::kFailed);
}
} // namespace
int run_deploy_tests() {
test_rolling_happy_path();
test_rolling_gate_failure();
test_rolling_tolerates_limited_outage();
test_rolling_rejects_empty();
test_blue_green_happy_path();
test_blue_green_rollback();
test_blue_green_verify_timeout();
test_blue_green_switch_refused();
return g_fail;
}
+94
View File
@@ -0,0 +1,94 @@
// Lifecycle tests: boot completion and the four phase drain.
#include <cstdio>
#include <string>
#include "operability/deploy/lifecycle.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
void test_mark_serving() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Lifecycle lc(clock, obs);
bool called = false;
lc.on_serving([&] { called = true; });
lc.mark_serving();
CHECK(called);
CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kServing);
}
void test_clean_drain() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Lifecycle lc(clock, obs);
bool stopped = false;
bool done = false;
lc.on_stop_listeners([&] { stopped = true; });
lc.on_connections_drained([] { return true; });
lc.on_background_drained([] { return true; });
lc.on_complete([&] { done = true; });
CHECK(lc.run_drain(1000));
CHECK(stopped && done);
CHECK(lc.clean());
CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kDone);
}
void test_delayed_drain() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Lifecycle lc(clock, obs);
// Probes flip true once the frozen clock passes their marks; the drain
// loop advances the clock via sleep_ms, so no real waiting happens.
lc.on_connections_drained([&] { return clock.steady_ms() >= 200; });
lc.on_background_drained([&] { return clock.steady_ms() >= 400; });
CHECK(lc.run_drain(5000, 50));
CHECK(lc.clean());
CHECK(clock.steady_ms() >= 400);
}
void test_drain_timeout_drops_background() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Lifecycle lc(clock, obs);
bool dropped = false;
lc.on_connections_drained([] { return true; });
lc.on_background_drained([] { return false; });
lc.on_drop_background([&](const std::string&) { dropped = true; });
CHECK(!lc.run_drain(200, 25));
CHECK(!lc.clean());
CHECK(dropped);
CHECK(lc.phase() == molo::deploy::Lifecycle::Phase::kDone);
}
void test_drain_timeout_connections() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::deploy::Lifecycle lc(clock, obs);
lc.on_connections_drained([] { return false; });
lc.on_background_drained([] { return true; });
CHECK(!lc.run_drain(100, 25));
CHECK(!lc.clean());
}
} // namespace
int run_lifecycle_tests() {
test_mark_serving();
test_clean_drain();
test_delayed_drain();
test_drain_timeout_drops_background();
test_drain_timeout_connections();
return g_fail;
}
@@ -0,0 +1,78 @@
// Shedder and deadline tests: overload shedding with Retry-After plus
// X-Deadline-Ms parsing and remaining budget math.
#include <cstdio>
#include <string>
#include "operability/resilience/deadline.h"
#include "operability/resilience/shedder.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
void test_shedderadmits_when_cool() {
molo::common::RecordingObserver obs;
molo::resilience::Shedder s(obs, [] { return false; });
CHECK(!s.should_shed());
}
void test_shedder_sheds_with_retry_after() {
molo::common::RecordingObserver obs;
molo::resilience::Shedder s(obs, [] { return true; }, 7);
CHECK(s.should_shed());
const auto r = s.reject();
CHECK(r.status == 503);
CHECK(r.retry_after == "7");
CHECK(s.retry_after_s() == 7);
}
void test_shedder_clamps_retry_after() {
molo::common::RecordingObserver obs;
molo::resilience::Shedder hi(obs, [] { return true; }, 99999);
CHECK(hi.retry_after_s() == 120);
molo::resilience::Shedder lo(obs, [] { return true; }, -5);
CHECK(lo.retry_after_s() == 0);
}
void test_deadline_parse() {
const auto ok = molo::resilience::Deadline::parse_header("12345");
CHECK(ok.has_value() && *ok == 12345);
CHECK(!molo::resilience::Deadline::parse_header("").has_value());
CHECK(!molo::resilience::Deadline::parse_header("abc").has_value());
CHECK(!molo::resilience::Deadline::parse_header("12x").has_value());
CHECK(!molo::resilience::Deadline::parse_header("99999999999999999999999").has_value());
CHECK(molo::resilience::Deadline::format_header(42) == "42");
CHECK(molo::resilience::Deadline::format_header(-1) == "0");
}
void test_deadline_remaining() {
molo::common::FrozenClock clock(100000);
molo::resilience::Deadline d(clock);
CHECK(d.remaining_ms(100500) == 450); // 500 left minus the 50 ms margin
CHECK(d.remaining_ms(100050) == 0); // exactly the margin: unusable
CHECK(d.remaining_ms(100040) == 0); // inside the margin
CHECK(d.remaining_ms(90000) == 0); // already past
CHECK(!d.expired(100500));
CHECK(d.expired(100040));
}
} // namespace
int run_shed_deadline_tests() {
test_shedderadmits_when_cool();
test_shedder_sheds_with_retry_after();
test_shedder_clamps_retry_after();
test_deadline_parse();
test_deadline_remaining();
return g_fail;
}
+101
View File
@@ -0,0 +1,101 @@
// SLO tests: budgets, burn rates, bad event counters, prometheus render.
#include <cstdio>
#include <string>
#include "operability/resilience/slo.h"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int g_fail = 0;
#define CHECK(cond) \
do { \
if (!(cond)) { \
++g_fail; \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
} \
} while (0)
bool near(double a, double b) {
const double d = a > b ? a - b : b - a;
return d < 0.0001;
}
void test_budget_and_burn() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Slo::Config cfg;
cfg.target_ratio = 0.9;
cfg.window_ms = 3600000;
molo::resilience::Slo slo(clock, obs, cfg);
for (int i = 0; i < 19; ++i) slo.record("/api/items", true);
slo.record("/api/items", false, "boom");
// 20 samples, 1 error, 10 percent budget: half the budget remains.
CHECK(near(slo.budget_remaining("/api/items"), 0.5));
// Observed 5 percent over budgeted 10 percent: burn 0.5.
CHECK(near(slo.burn_rate("/api/items", 3600000), 0.5));
CHECK(slo.bad_events("/api/items", "boom") == 1);
CHECK(slo.bad_events("/api/items", "other") == 0);
}
void test_unknown_route_is_healthy() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Slo slo(clock, obs);
CHECK(near(slo.budget_remaining("/nope"), 1.0));
CHECK(near(slo.burn_rate("/nope", 60000), 0.0));
}
void test_budget_exhausted() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Slo::Config cfg;
cfg.target_ratio = 0.99;
molo::resilience::Slo slo(clock, obs, cfg);
for (int i = 0; i < 10; ++i) slo.record("/bad", false, "down");
CHECK(near(slo.budget_remaining("/bad"), 0.0));
CHECK(slo.burn_rate("/bad", 3600000) > 1.0);
}
void test_window_pruning_restores_budget() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Slo::Config cfg;
cfg.target_ratio = 0.9;
cfg.window_ms = 1000;
molo::resilience::Slo slo(clock, obs, cfg);
slo.record("/flap", false, "x");
CHECK(near(slo.budget_remaining("/flap"), 0.0));
clock.advance(2000);
CHECK(near(slo.budget_remaining("/flap"), 1.0));
CHECK(near(slo.burn_rate("/flap", 1000), 0.0));
}
void test_prometheus_render() {
molo::common::FrozenClock clock(0);
molo::common::RecordingObserver obs;
molo::resilience::Slo slo(clock, obs);
slo.record("/api/items", true);
slo.record("/api/items", false, "boom");
const std::string out = slo.render_prometheus();
CHECK(out.find("molo_slo_budget_remaining_ratio") != std::string::npos);
CHECK(out.find("molo_slo_burn_rate") != std::string::npos);
CHECK(out.find("molo_slo_bad_events_total") != std::string::npos);
CHECK(out.find("route=\"/api/items\"") != std::string::npos);
CHECK(out.find("reason=\"boom\"") != std::string::npos);
CHECK(out.find("window=\"5m\"") != std::string::npos);
CHECK(out.find("window=\"1h\"") != std::string::npos);
}
} // namespace
int run_slo_tests() {
test_budget_and_burn();
test_unknown_route_is_healthy();
test_budget_exhausted();
test_window_pruning_restores_budget();
test_prometheus_render();
return g_fail;
}