156 lines
4.7 KiB
C++
156 lines
4.7 KiB
C++
// Coordination tests: lease, distributed lock, fence, leader election.
|
|||
|
|
// Frozen clock throughout; no real waiting.
|
||
|
|
#include <cstdint>
|
||
|
|
#include <cstdio>
|
||
|
|
#include <string>
|
||
|
|
|
||
|
|
#include "operability/coord/dist_lock.h"
|
||
|
|
#include "operability/coord/election.h"
|
||
|
|
#include "operability/coord/fence.h"
|
||
|
|
#include "operability/coord/lease.h"
|
||
|
|
#include "supervisor/common/clock.hpp"
|
||
|
|
#include "supervisor/common/kv_store.hpp"
|
||
|
|
#include "supervisor/common/observer.hpp"
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
|
||
|
|
int g_fail = 0;
|
||
|
|
|
||
|
|
#define CHECK(cond) \
|
||
|
|
do { \
|
||
|
|
if (!(cond)) { \
|
||
|
|
++g_fail; \
|
||
|
|
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
|
||
|
|
} \
|
||
|
|
} while (0)
|
||
|
|
|
||
|
|
void test_lease_basic() {
|
||
|
|
molo::common::FrozenClock clock(1000);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::Lease a(kv, clock, obs, "k/a", "owner-a");
|
||
|
|
molo::coord::Lease b(kv, clock, obs, "k/a", "owner-b");
|
||
|
|
CHECK(a.acquire(5000));
|
||
|
|
CHECK(a.holds());
|
||
|
|
CHECK(!b.acquire(5000));
|
||
|
|
CHECK(a.renew(5000));
|
||
|
|
a.release();
|
||
|
|
CHECK(!a.holds());
|
||
|
|
CHECK(b.acquire(5000));
|
||
|
|
CHECK(b.holds());
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_lease_expiry() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::Lease a(kv, clock, obs, "k/exp", "a");
|
||
|
|
molo::coord::Lease b(kv, clock, obs, "k/exp", "b");
|
||
|
|
CHECK(a.acquire(1000));
|
||
|
|
clock.advance(1001);
|
||
|
|
CHECK(b.acquire(1000));
|
||
|
|
CHECK(!a.renew(1000));
|
||
|
|
CHECK(!a.holds());
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_lease_rejects_bad_args() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::Lease bad(kv, clock, obs, "", "");
|
||
|
|
CHECK(!bad.acquire(1000));
|
||
|
|
molo::coord::Lease c(kv, clock, obs, "k/c", "c");
|
||
|
|
CHECK(!c.acquire(0));
|
||
|
|
CHECK(!c.renew(1000));
|
||
|
|
CHECK(molo::coord::Lease::live_count() >= 2);
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_dist_lock() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::DistLock a(kv, clock, obs, "k/lock", "a");
|
||
|
|
molo::coord::DistLock b(kv, clock, obs, "k/lock", "b");
|
||
|
|
const std::uint64_t t1 = a.try_lock(60000, 0);
|
||
|
|
CHECK(t1 != 0);
|
||
|
|
CHECK(a.locked());
|
||
|
|
CHECK(b.try_lock(60000, 0) == 0);
|
||
|
|
a.unlock();
|
||
|
|
CHECK(!a.locked());
|
||
|
|
const std::uint64_t t2 = b.try_lock(60000, 100);
|
||
|
|
CHECK(t2 != 0);
|
||
|
|
CHECK(t2 > t1);
|
||
|
|
b.unlock();
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_fence() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::coord::Fence f1(kv, "k/f");
|
||
|
|
molo::coord::Fence f2(kv, "k/f");
|
||
|
|
const std::uint64_t t1 = f1.next();
|
||
|
|
const std::uint64_t t2 = f2.next();
|
||
|
|
CHECK(t1 != 0 && t2 != 0 && t2 > t1);
|
||
|
|
CHECK(f1.check(t2));
|
||
|
|
CHECK(!f1.check(t1));
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_election() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::LeaderElection n1(kv, clock, obs, "molo/scheduler/leader", "n1");
|
||
|
|
molo::coord::LeaderElection n2(kv, clock, obs, "molo/scheduler/leader", "n2");
|
||
|
|
CHECK(n1.tick());
|
||
|
|
CHECK(n1.is_leader());
|
||
|
|
CHECK(!n2.tick());
|
||
|
|
CHECK(!n2.is_leader());
|
||
|
|
CHECK(n2.leader() == "n1");
|
||
|
|
n1.resign();
|
||
|
|
CHECK(!n1.is_leader());
|
||
|
|
CHECK(n2.tick());
|
||
|
|
CHECK(n2.is_leader());
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_election_failover_on_expiry() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::LeaderElection n1(kv, clock, obs, "e/fail", "n1", 10000, 3000);
|
||
|
|
molo::coord::LeaderElection n2(kv, clock, obs, "e/fail", "n2", 10000, 3000);
|
||
|
|
CHECK(n1.tick());
|
||
|
|
clock.advance(11000); // n1 never renews
|
||
|
|
CHECK(n2.tick());
|
||
|
|
CHECK(n2.is_leader());
|
||
|
|
CHECK(!n1.tick());
|
||
|
|
CHECK(!n1.is_leader());
|
||
|
|
}
|
||
|
|
|
||
|
|
void test_election_renew_keeps_leadership() {
|
||
|
|
molo::common::FrozenClock clock(0);
|
||
|
|
molo::common::InMemoryKvStore kv(clock);
|
||
|
|
molo::common::RecordingObserver obs;
|
||
|
|
molo::coord::LeaderElection n1(kv, clock, obs, "e/keep", "n1", 10000, 3000);
|
||
|
|
CHECK(n1.tick());
|
||
|
|
for (int i = 0; i < 10; ++i) {
|
||
|
|
clock.advance(3000);
|
||
|
|
CHECK(n1.tick());
|
||
|
|
}
|
||
|
|
CHECK(n1.is_leader());
|
||
|
|
CHECK(n1.leader() == "n1");
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
int run_coord_tests() {
|
||
|
|
test_lease_basic();
|
||
|
|
test_lease_expiry();
|
||
|
|
test_lease_rejects_bad_args();
|
||
|
|
test_dist_lock();
|
||
|
|
test_fence();
|
||
|
|
test_election();
|
||
|
|
test_election_failover_on_expiry();
|
||
|
|
test_election_renew_keeps_leadership();
|
||
|
|
return g_fail;
|
||
|
|
}
|