// Coordination tests: lease, distributed lock, fence, leader election. // Frozen clock throughout; no real waiting. #include #include #include #include "operability/coord/dist_lock.h" #include "operability/coord/election.h" #include "operability/coord/fence.h" #include "operability/coord/lease.h" #include "supervisor/common/clock.hpp" #include "supervisor/common/kv_store.hpp" #include "supervisor/common/observer.hpp" namespace { int g_fail = 0; #define CHECK(cond) \ do { \ if (!(cond)) { \ ++g_fail; \ std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ } \ } while (0) void test_lease_basic() { molo::common::FrozenClock clock(1000); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::Lease a(kv, clock, obs, "k/a", "owner-a"); molo::coord::Lease b(kv, clock, obs, "k/a", "owner-b"); CHECK(a.acquire(5000)); CHECK(a.holds()); CHECK(!b.acquire(5000)); CHECK(a.renew(5000)); a.release(); CHECK(!a.holds()); CHECK(b.acquire(5000)); CHECK(b.holds()); } void test_lease_expiry() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::Lease a(kv, clock, obs, "k/exp", "a"); molo::coord::Lease b(kv, clock, obs, "k/exp", "b"); CHECK(a.acquire(1000)); clock.advance(1001); CHECK(b.acquire(1000)); CHECK(!a.renew(1000)); CHECK(!a.holds()); } void test_lease_rejects_bad_args() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::Lease bad(kv, clock, obs, "", ""); CHECK(!bad.acquire(1000)); molo::coord::Lease c(kv, clock, obs, "k/c", "c"); CHECK(!c.acquire(0)); CHECK(!c.renew(1000)); CHECK(molo::coord::Lease::live_count() >= 2); } void test_dist_lock() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::DistLock a(kv, clock, obs, "k/lock", "a"); molo::coord::DistLock b(kv, clock, obs, "k/lock", "b"); const std::uint64_t t1 = a.try_lock(60000, 0); CHECK(t1 != 0); CHECK(a.locked()); CHECK(b.try_lock(60000, 0) == 0); a.unlock(); CHECK(!a.locked()); const std::uint64_t t2 = b.try_lock(60000, 100); CHECK(t2 != 0); CHECK(t2 > t1); b.unlock(); } void test_fence() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::coord::Fence f1(kv, "k/f"); molo::coord::Fence f2(kv, "k/f"); const std::uint64_t t1 = f1.next(); const std::uint64_t t2 = f2.next(); CHECK(t1 != 0 && t2 != 0 && t2 > t1); CHECK(f1.check(t2)); CHECK(!f1.check(t1)); } void test_election() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::LeaderElection n1(kv, clock, obs, "molo/scheduler/leader", "n1"); molo::coord::LeaderElection n2(kv, clock, obs, "molo/scheduler/leader", "n2"); CHECK(n1.tick()); CHECK(n1.is_leader()); CHECK(!n2.tick()); CHECK(!n2.is_leader()); CHECK(n2.leader() == "n1"); n1.resign(); CHECK(!n1.is_leader()); CHECK(n2.tick()); CHECK(n2.is_leader()); } void test_election_failover_on_expiry() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::LeaderElection n1(kv, clock, obs, "e/fail", "n1", 10000, 3000); molo::coord::LeaderElection n2(kv, clock, obs, "e/fail", "n2", 10000, 3000); CHECK(n1.tick()); clock.advance(11000); // n1 never renews CHECK(n2.tick()); CHECK(n2.is_leader()); CHECK(!n1.tick()); CHECK(!n1.is_leader()); } void test_election_renew_keeps_leadership() { molo::common::FrozenClock clock(0); molo::common::InMemoryKvStore kv(clock); molo::common::RecordingObserver obs; molo::coord::LeaderElection n1(kv, clock, obs, "e/keep", "n1", 10000, 3000); CHECK(n1.tick()); for (int i = 0; i < 10; ++i) { clock.advance(3000); CHECK(n1.tick()); } CHECK(n1.is_leader()); CHECK(n1.leader() == "n1"); } } // namespace int run_coord_tests() { test_lease_basic(); test_lease_expiry(); test_lease_rejects_bad_args(); test_dist_lock(); test_fence(); test_election(); test_election_failover_on_expiry(); test_election_renew_keeps_leadership(); return g_fail; }