feat: implement MoloApp plans 01-15 with integration
Full C++23 framework tree: runtime, observability, security, data, app, realtime, supervisor, clients, CLI, example plus enterprise tenancy, audit, operability, governance, supplychain. Postgres via docker compose for tests.
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
// V7-5: durable streams (persist, reconnect, redeliver) and ephemeral topics.
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <set>
|
||||
|
||||
#include "supervisor/bus/stream_bus.hpp"
|
||||
#include "supervisor/bus/topic_bus.hpp"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
const std::filesystem::path kTmp = "build/supervisor/tmp/test-bus";
|
||||
|
||||
void test_durable_stream_reconnect_no_loss() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = molo::common::null_observer();
|
||||
|
||||
{
|
||||
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
|
||||
bus.create_stream("events");
|
||||
bus.create_consumer("events", "worker", 60000);
|
||||
for (int i = 1; i <= 10000; ++i) {
|
||||
bus.publish("events", "msg-" + std::to_string(i));
|
||||
}
|
||||
// Consume half, then "disconnect" by destroying the bus mid-stream.
|
||||
int acked = 0;
|
||||
while (acked < 5000) {
|
||||
auto batch = bus.poll("events", "worker", 500);
|
||||
CHECK(!batch.empty(), "poll returns messages");
|
||||
for (const auto& msg : batch) {
|
||||
CHECK(msg.data == "msg-" + std::to_string(msg.seq), "payload matches seq");
|
||||
CHECK(bus.ack("events", "worker", msg.seq), "ack works");
|
||||
++acked;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Reconnect: a fresh bus over the same directory replays the log and
|
||||
// the persisted cursor resumes exactly where the acked prefix ended.
|
||||
{
|
||||
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
|
||||
bus.create_stream("events");
|
||||
bus.create_consumer("events", "worker", 60000);
|
||||
std::set<std::uint64_t> seen;
|
||||
std::uint64_t expect = 5001;
|
||||
while (seen.size() < 5000) {
|
||||
auto batch = bus.poll("events", "worker", 500);
|
||||
CHECK(!batch.empty(), "reconnected poll returns messages");
|
||||
for (const auto& msg : batch) {
|
||||
CHECK(msg.seq == expect, "no gaps after reconnect");
|
||||
++expect;
|
||||
seen.insert(msg.seq);
|
||||
CHECK(bus.ack("events", "worker", msg.seq), "reconnected ack works");
|
||||
}
|
||||
}
|
||||
CHECK(seen.size() == 5000 && *seen.begin() == 5001 && *seen.rbegin() == 10000,
|
||||
"zero loss, zero gaps across reconnect");
|
||||
CHECK(bus.lag("events", "worker") == 0, "caught up, lag zero");
|
||||
}
|
||||
std::cout << "PASS test_durable_stream_reconnect_no_loss" << std::endl;
|
||||
}
|
||||
|
||||
void test_redelivery_until_acked() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = std::make_shared<molo::common::RecordingObserver>();
|
||||
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
|
||||
bus.create_stream("jobs");
|
||||
bus.create_consumer("jobs", "w", 1000); // 1s ack deadline
|
||||
bus.publish("jobs", "a");
|
||||
bus.publish("jobs", "b");
|
||||
|
||||
auto first = bus.poll("jobs", "w", 10);
|
||||
CHECK(first.size() == 2, "two messages delivered");
|
||||
CHECK(first[0].redeliveries == 0, "first delivery is fresh");
|
||||
// No acks: advance past the deadline, the same messages redeliver.
|
||||
clock.advance(1500);
|
||||
auto second = bus.poll("jobs", "w", 10);
|
||||
CHECK(second.size() == 2, "both redeliver after deadline");
|
||||
CHECK(second[0].seq == first[0].seq && second[0].redeliveries == 1, "redelivery counted");
|
||||
CHECK(obs->counter_value("molo_bus_stream_redelivered_total") == 2, "redelivery metric");
|
||||
CHECK(bus.ack("jobs", "w", first[0].seq), "ack first");
|
||||
CHECK(!bus.ack("jobs", "w", first[0].seq), "double ack fails");
|
||||
clock.advance(1500);
|
||||
auto third = bus.poll("jobs", "w", 10);
|
||||
CHECK(third.size() == 1 && third[0].seq == first[1].seq, "only unacked redelivers");
|
||||
CHECK(bus.renew("jobs", "w", first[1].seq), "renew extends the deadline");
|
||||
clock.advance(500);
|
||||
CHECK(bus.poll("jobs", "w", 10).empty(), "renewed message waits");
|
||||
clock.advance(600);
|
||||
CHECK(bus.poll("jobs", "w", 10).size() == 1, "redelivers after renewed deadline");
|
||||
std::cout << "PASS test_redelivery_until_acked" << std::endl;
|
||||
}
|
||||
|
||||
void test_retention_and_caps() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
|
||||
|
||||
molo::bus::StreamSpec tiny;
|
||||
tiny.retention_bytes = 1024;
|
||||
tiny.retention_hours = 0; // size-only retention
|
||||
bus.create_stream("tiny", tiny);
|
||||
bus.create_consumer("tiny", "c");
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
bus.publish("tiny", std::string(100, 'x'));
|
||||
}
|
||||
CHECK(bus.last_seq("tiny") == 100, "seqs assigned across retention");
|
||||
auto retained = bus.poll("tiny", "c", 200);
|
||||
CHECK(retained.size() < 100 && !retained.empty(), "retention evicted the oldest");
|
||||
if (!retained.empty()) {
|
||||
CHECK(retained.front().seq > 1, "oldest seqs gone");
|
||||
CHECK(retained.back().seq == 100, "tail intact");
|
||||
}
|
||||
|
||||
molo::bus::StreamSpec timed;
|
||||
timed.retention_hours = 1;
|
||||
timed.retention_bytes = 1024 * 1024 * 1024;
|
||||
bus.create_stream("timed", timed);
|
||||
bus.publish("timed", "old");
|
||||
clock.advance(2 * 3600 * 1000);
|
||||
CHECK(bus.enforce_retention("timed") == 1, "time retention evicts one");
|
||||
bus.publish("timed", "new");
|
||||
|
||||
bool threw = false;
|
||||
try {
|
||||
bus.publish("tiny", std::string(1048576 + 1, 'y'));
|
||||
} catch (const molo::bus::BusError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw, "oversize stream payload rejected");
|
||||
threw = false;
|
||||
try {
|
||||
bus.create_stream("../escape");
|
||||
} catch (const molo::bus::BusError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw, "path escape in stream name rejected");
|
||||
std::cout << "PASS test_retention_and_caps" << std::endl;
|
||||
}
|
||||
|
||||
void test_ephemeral_topics() {
|
||||
molo::common::SystemClock clock;
|
||||
molo::bus::TopicBus bus(clock);
|
||||
|
||||
CHECK(bus.publish("a:b", "lost") == 0, "no-subscriber publish is a counted no-op");
|
||||
const auto sub1 = bus.subscribe("a:b");
|
||||
const auto sub2 = bus.subscribe("a:*");
|
||||
const auto late = bus.subscribe("a:b");
|
||||
CHECK(bus.publish("a:b", "hi") == 3, "exact plus wildcard plus late deliver");
|
||||
auto got = bus.poll(sub1, 10);
|
||||
CHECK(got.size() == 1 && got[0].payload == "hi", "exact subscriber receives");
|
||||
CHECK(bus.poll(sub2, 10).size() == 1, "wildcard subscriber receives");
|
||||
CHECK(bus.poll(late, 10).size() == 1, "subscriber present at publish receives");
|
||||
|
||||
// Offline means offline: subscribe after the publish, get nothing.
|
||||
CHECK(bus.publish("a:b", "before") == 3, "three live subscribers");
|
||||
const auto offline = bus.subscribe("a:b");
|
||||
bus.poll(sub1, 10);
|
||||
bus.poll(sub2, 10);
|
||||
bus.poll(late, 10);
|
||||
CHECK(bus.poll(offline, 10).empty(), "offline subscriber gets nothing past");
|
||||
CHECK(bus.publish("a:b:c", "deep") == 0, "wildcard matches one segment only");
|
||||
|
||||
// Backpressure: oldest-drop with a counter, publisher never blocks.
|
||||
for (int i = 0; i < 1034; ++i) {
|
||||
bus.publish("a:b", "flood-" + std::to_string(i));
|
||||
}
|
||||
CHECK(bus.dropped_total() > 0, "overflow counted");
|
||||
CHECK(bus.pending(sub1) == 1024, "subscriber queue capped at 1024");
|
||||
auto tail = bus.poll(sub1, 2000);
|
||||
CHECK(tail.front().payload == "flood-10", "oldest dropped first");
|
||||
CHECK(bus.unsubscribe(sub1) && !bus.unsubscribe(sub1), "unsubscribe once");
|
||||
std::cout << "PASS test_ephemeral_topics" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_durable_stream_reconnect_no_loss();
|
||||
test_redelivery_until_acked();
|
||||
test_retention_and_caps();
|
||||
test_ephemeral_topics();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL BUS TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
// V7-6: durable queue survives a hard kill; unacked work replays.
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
#include "supervisor/tasks/scheduler.hpp"
|
||||
#include "supervisor/tasks/task_queue.hpp"
|
||||
#include "supervisor/tasks/task_registry.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
const std::filesystem::path kTmp = "build/supervisor/tmp/test-durability";
|
||||
|
||||
struct IdempotentCounter : molo::tasks::TaskHandler {
|
||||
std::mutex* mutex;
|
||||
std::set<std::string>* completed;
|
||||
std::set<std::string>* seen_twice;
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
|
||||
std::lock_guard<std::mutex> lock(*mutex);
|
||||
if (!completed->insert(ctx.envelope.id).second) {
|
||||
seen_twice->insert(ctx.envelope.id); // redelivery tolerated
|
||||
}
|
||||
return molo::tasks::TaskOutcome::Success;
|
||||
}
|
||||
};
|
||||
|
||||
void test_hard_kill_replays_unacked() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
|
||||
std::set<std::string> ids;
|
||||
{
|
||||
molo::tasks::QueueOptions options;
|
||||
options.durable_dir = (kTmp / "q").string();
|
||||
options.lease_ms = 60000;
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
for (int i = 0; i < 50; ++i) {
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "work";
|
||||
ids.insert(queue.enqueue(std::move(env)));
|
||||
}
|
||||
CHECK(queue.stats().queued == 50, "50 queued before the kill");
|
||||
// Lease 10 without ack: in-flight at crash time.
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
CHECK(queue.lease(0).has_value(), "leases before the kill");
|
||||
}
|
||||
// Hard kill: the queue object just dies here. No shutdown, no acks.
|
||||
}
|
||||
{
|
||||
molo::tasks::QueueOptions options;
|
||||
options.durable_dir = (kTmp / "q").string();
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
CHECK(queue.stats().queued == 50, "all 50 replay as queued after reboot");
|
||||
// Every replayed id matches a pre-crash id.
|
||||
auto live = queue.list_queued(100);
|
||||
CHECK(live.size() == 50, "50 live envelopes");
|
||||
for (const auto& env : live) {
|
||||
CHECK(ids.count(env.id) == 1, "replayed id is pre-crash");
|
||||
}
|
||||
molo::tasks::TaskRegistry registry;
|
||||
std::mutex mutex;
|
||||
std::set<std::string> completed, seen_twice;
|
||||
registry.register_handler("work", [&] {
|
||||
auto h = std::make_unique<IdempotentCounter>();
|
||||
h->mutex = &mutex;
|
||||
h->completed = &completed;
|
||||
h->seen_twice = &seen_twice;
|
||||
return h;
|
||||
});
|
||||
int ran = 0;
|
||||
while (queue.run_one(registry, 0)) {
|
||||
++ran;
|
||||
}
|
||||
CHECK(ran == 50 && completed.size() == 50, "all replayed work completes");
|
||||
CHECK(completed == ids, "completed set equals the pre-crash set");
|
||||
CHECK(queue.stats().queued == 0, "queue drains fully");
|
||||
}
|
||||
std::cout << "PASS test_hard_kill_replays_unacked" << std::endl;
|
||||
}
|
||||
|
||||
void test_retry_state_survives_reboot() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
std::string id;
|
||||
{
|
||||
molo::tasks::QueueOptions options;
|
||||
options.durable_dir = (kTmp / "q").string();
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "flaky";
|
||||
id = queue.enqueue(std::move(env));
|
||||
auto leased = queue.lease(0);
|
||||
CHECK(leased.has_value(), "leases the task");
|
||||
CHECK(queue.fail(leased->lease_id, true, "boom"), "fails retryable");
|
||||
auto stored = queue.get(id);
|
||||
CHECK(stored.has_value() && stored->attempts == 1, "attempts persisted");
|
||||
CHECK(stored.has_value() && stored->run_at_ms.has_value(), "backoff run_at persisted");
|
||||
}
|
||||
{
|
||||
molo::tasks::QueueOptions options;
|
||||
options.durable_dir = (kTmp / "q").string();
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
auto stored = queue.get(id);
|
||||
CHECK(stored.has_value(), "retried task survives reboot");
|
||||
if (stored.has_value()) {
|
||||
CHECK(stored->attempts == 1, "attempt count survives");
|
||||
CHECK(stored->run_at_ms.has_value(), "backoff deadline survives");
|
||||
}
|
||||
}
|
||||
std::cout << "PASS test_retry_state_survives_reboot" << std::endl;
|
||||
}
|
||||
|
||||
void test_compact_and_torn_tail() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
{
|
||||
molo::tasks::QueueOptions options;
|
||||
options.durable_dir = (kTmp / "q").string();
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
molo::tasks::TaskRegistry registry;
|
||||
registry.register_handler("w", [] {
|
||||
struct H : molo::tasks::TaskHandler {
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
|
||||
return molo::tasks::TaskOutcome::Success;
|
||||
}
|
||||
};
|
||||
return std::make_unique<H>();
|
||||
});
|
||||
for (int i = 0; i < 20; ++i) {
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "w";
|
||||
queue.enqueue(std::move(env));
|
||||
}
|
||||
while (queue.run_one(registry, 0)) {
|
||||
}
|
||||
const auto log = kTmp / "q" / "task_queue.log";
|
||||
const std::uint64_t before = fs::file_size(log);
|
||||
CHECK(before > 0, "op log written");
|
||||
queue.compact();
|
||||
CHECK(fs::file_size(log) < before, "compact shrinks the log");
|
||||
}
|
||||
{
|
||||
// Torn tail: crash garbage at the end of the log is truncated on open.
|
||||
molo::tasks::QueueOptions options;
|
||||
options.durable_dir = (kTmp / "q").string();
|
||||
{
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "w";
|
||||
queue.enqueue(std::move(env));
|
||||
}
|
||||
}
|
||||
const auto log = kTmp / "q" / "task_queue.log";
|
||||
{
|
||||
std::ofstream out(log, std::ios::app | std::ios::binary);
|
||||
out << "junk";
|
||||
}
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
CHECK(queue.stats().queued == 5, "torn tail truncated, 5 tasks replay");
|
||||
}
|
||||
std::cout << "PASS test_compact_and_torn_tail" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void test_schedule_entries_survive_reboot() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
molo::tasks::QueueOptions options;
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
{
|
||||
molo::tasks::SchedulerOptions sched_options;
|
||||
sched_options.instance_id = "sched-durable";
|
||||
sched_options.durable_dir = (kTmp / "sched").string();
|
||||
molo::tasks::Scheduler sched(queue, kv, clock, obs, sched_options);
|
||||
molo::tasks::ScheduleEntry interval;
|
||||
interval.key = "every-minute";
|
||||
interval.task_name = "work";
|
||||
interval.kind = molo::tasks::ScheduleKind::Interval;
|
||||
interval.interval_secs = 60;
|
||||
sched.put(interval);
|
||||
molo::tasks::ScheduleEntry cron;
|
||||
cron.key = "nightly";
|
||||
cron.task_name = "work";
|
||||
cron.kind = molo::tasks::ScheduleKind::Cron;
|
||||
cron.cron = "0 2 * * *";
|
||||
sched.put(cron);
|
||||
molo::tasks::ScheduleEntry temp;
|
||||
temp.key = "temp";
|
||||
temp.task_name = "work";
|
||||
temp.kind = molo::tasks::ScheduleKind::Interval;
|
||||
sched.put(temp);
|
||||
CHECK(sched.remove("temp"), "temp entry removed before reboot");
|
||||
}
|
||||
{
|
||||
molo::tasks::SchedulerOptions sched_options;
|
||||
sched_options.instance_id = "sched-durable";
|
||||
sched_options.durable_dir = (kTmp / "sched").string();
|
||||
molo::tasks::Scheduler sched(queue, kv, clock, obs, sched_options);
|
||||
auto entries = sched.entries();
|
||||
CHECK(entries.size() == 2, "two entries replay after reboot");
|
||||
bool has_interval = false, has_cron = false;
|
||||
for (const auto& entry : entries) {
|
||||
if (entry.key == "every-minute" && entry.kind == molo::tasks::ScheduleKind::Interval) {
|
||||
has_interval = true;
|
||||
}
|
||||
if (entry.key == "nightly" && entry.cron == "0 2 * * *") {
|
||||
has_cron = true;
|
||||
}
|
||||
CHECK(entry.key != "temp", "removed entry stays removed");
|
||||
}
|
||||
CHECK(has_interval && has_cron, "entry kinds and crons survive");
|
||||
}
|
||||
std::cout << "PASS test_schedule_entries_survive_reboot" << std::endl;
|
||||
}
|
||||
|
||||
int main() {
|
||||
test_hard_kill_replays_unacked();
|
||||
test_retry_state_survives_reboot();
|
||||
test_compact_and_torn_tail();
|
||||
test_schedule_entries_survive_reboot();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL DURABILITY TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// V7-8: two nodes behind one shared bus; broadcast plus catch-up.
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
|
||||
#include "supervisor/bus/stream_bus.hpp"
|
||||
#include "supervisor/bus/topic_bus.hpp"
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
const std::filesystem::path kTmp = "build/supervisor/tmp/test-multi";
|
||||
|
||||
// One stateless app node (S7-9): local sockets hang off the ephemeral bus,
|
||||
// catch-up replays off the durable bus, sessions live in shared KV.
|
||||
struct Node {
|
||||
std::string id;
|
||||
molo::bus::TopicBus::SubscriberId room_sub = 0;
|
||||
int received_live = 0;
|
||||
std::vector<std::string> caught_up;
|
||||
};
|
||||
|
||||
void test_broadcast_reaches_both_nodes() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
|
||||
// Shared backends both nodes attach to.
|
||||
molo::bus::TopicBus live(clock);
|
||||
molo::bus::StreamBus bus(kTmp / "bus", clock, obs);
|
||||
bus.create_stream("roomlog");
|
||||
molo::common::InMemoryKvStore kv(clock);
|
||||
|
||||
Node a;
|
||||
a.id = "node-a";
|
||||
Node b;
|
||||
b.id = "node-b";
|
||||
a.room_sub = live.subscribe("room:lobby");
|
||||
b.room_sub = live.subscribe("room:lobby");
|
||||
bus.create_consumer("roomlog", "node-a");
|
||||
bus.create_consumer("roomlog", "node-b");
|
||||
|
||||
// Broadcast emit: publish once to the bus; every node re-emits to its
|
||||
// local sockets (here: each node polls its own subscription).
|
||||
auto broadcast = [&](const std::string& payload) {
|
||||
const std::size_t delivered = live.publish("room:lobby", payload);
|
||||
bus.publish("roomlog", payload); // durable copy for reconnect catch-up
|
||||
return delivered;
|
||||
};
|
||||
CHECK(broadcast("hello") == 2, "one publish, two nodes");
|
||||
|
||||
for (Node* node : {&a, &b}) {
|
||||
auto got = live.poll(node->room_sub, 10);
|
||||
CHECK(got.size() == 1 && got[0].payload == "hello", "live delivery on " + node->id);
|
||||
node->received_live += static_cast<int>(got.size());
|
||||
// Nodes also drain the durable log to keep their cursors current.
|
||||
for (const auto& msg : bus.poll("roomlog", node->id, 10)) {
|
||||
bus.ack("roomlog", node->id, msg.seq);
|
||||
}
|
||||
}
|
||||
|
||||
// Shared session store: written via node A, visible via node B.
|
||||
kv.set("session:u1", "{\"user\":\"u1\"}", 3600);
|
||||
std::string session;
|
||||
CHECK(kv.get("session:u1", session) && session == "{\"user\":\"u1\"}", "sessions shared");
|
||||
|
||||
// Kill node B: its live subscription dies with it.
|
||||
CHECK(live.unsubscribe(b.room_sub), "node B dies");
|
||||
CHECK(broadcast("msg-2") == 1, "dead node gets nothing live");
|
||||
auto got_a = live.poll(a.room_sub, 10);
|
||||
CHECK(got_a.size() == 1 && got_a[0].payload == "msg-2", "node A receives msg-2 live");
|
||||
for (const auto& msg : bus.poll("roomlog", "node-a", 10)) {
|
||||
bus.ack("roomlog", "node-a", msg.seq);
|
||||
}
|
||||
|
||||
// Node B restarts: live messages sent while dead are gone (ephemeral
|
||||
// tier), but the durable log replays them in order.
|
||||
b.room_sub = live.subscribe("room:lobby");
|
||||
CHECK(live.poll(b.room_sub, 10).empty(), "no ephemeral replay while dead");
|
||||
for (const auto& msg : bus.poll("roomlog", "node-b", 10)) {
|
||||
b.caught_up.push_back(msg.data);
|
||||
bus.ack("roomlog", "node-b", msg.seq);
|
||||
}
|
||||
CHECK(b.caught_up.size() == 1 && b.caught_up[0] == "msg-2", "durable catch-up replays msg-2");
|
||||
CHECK(bus.lag("roomlog", "node-b") == 0, "node B caught up");
|
||||
std::cout << "PASS test_broadcast_reaches_both_nodes" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_broadcast_reaches_both_nodes();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL MULTI-INSTANCE TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
// V7-7: two-instance replication, optimistic plus LWW plus hook conflicts.
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
#include "supervisor/replication/conflict_resolver.hpp"
|
||||
#include "supervisor/replication/hybrid_clock.hpp"
|
||||
#include "supervisor/replication/replicator.hpp"
|
||||
#include "supervisor/replication/sync_session.hpp"
|
||||
#include "supervisor/replication/version_vector.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
const std::filesystem::path kTmp = "build/supervisor/tmp/test-replication";
|
||||
|
||||
// In-process SyncPeer over another Replicator (production binds this to
|
||||
// the bus transport; the sync logic is identical).
|
||||
class ReplicatorSyncPeer : public molo::replication::SyncPeer {
|
||||
public:
|
||||
explicit ReplicatorSyncPeer(molo::replication::Replicator& remote) : remote_(remote) {}
|
||||
|
||||
std::vector<molo::replication::OplogEntry> fetch_since(
|
||||
const molo::replication::VersionVector& have, std::size_t max_entries,
|
||||
std::size_t max_bytes) override {
|
||||
return remote_.entries_since(have.get(remote_.instance()) + 1, max_entries, max_bytes);
|
||||
}
|
||||
|
||||
void submit(const std::vector<molo::replication::OplogEntry>& entries) override {
|
||||
for (const auto& entry : entries) {
|
||||
try {
|
||||
remote_.apply_remote(entry);
|
||||
} catch (const molo::replication::ConflictError&) {
|
||||
// Per-entry independence: one loser's ConflictError never blocks
|
||||
// the rest of the batch. The resolver already logged and counted it.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void peer_versions(molo::replication::VersionVector& out) const override {
|
||||
remote_.versions(out);
|
||||
}
|
||||
|
||||
std::uint64_t last_seq() const override { return remote_.last_seq(); }
|
||||
std::string instance() const override { return remote_.instance(); }
|
||||
|
||||
private:
|
||||
molo::replication::Replicator& remote_;
|
||||
};
|
||||
|
||||
void sync_both(molo::replication::Replicator& a, molo::replication::Replicator& b,
|
||||
std::shared_ptr<molo::common::Observer> obs) {
|
||||
ReplicatorSyncPeer peer_b(b);
|
||||
molo::replication::SyncSession(a, peer_b, obs).round_trip();
|
||||
ReplicatorSyncPeer peer_a(a);
|
||||
molo::replication::SyncSession(b, peer_a, obs).round_trip();
|
||||
}
|
||||
|
||||
void test_optimistic_conflict_and_recovery() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = std::make_shared<molo::common::RecordingObserver>();
|
||||
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
|
||||
molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs);
|
||||
a.declare_model("orders", molo::replication::ConflictPolicy::Optimistic);
|
||||
b.declare_model("orders", molo::replication::ConflictPolicy::Optimistic);
|
||||
|
||||
a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 1, "{\"v\":1}");
|
||||
sync_both(a, b, obs);
|
||||
CHECK(b.current("orders", "k1").has_value(), "base write replicates");
|
||||
CHECK(obs->counter_value("molo_replication_conflicts_total") == 0, "no conflict on fast-forward");
|
||||
|
||||
// Concurrent write-write at the same version with different data.
|
||||
clock.advance(10);
|
||||
a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"a\"}");
|
||||
b.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"b\"}");
|
||||
|
||||
// Direct resolver check: exactly one winner plus ConflictError semantics.
|
||||
auto la = a.current("orders", "k1");
|
||||
auto lb = b.current("orders", "k1");
|
||||
auto outcome = a.resolver().resolve(*la, *lb);
|
||||
CHECK(outcome.conflict && outcome.conflict_error, "write-write conflict flagged");
|
||||
CHECK(outcome.winner_version == 2, "winner version carried");
|
||||
|
||||
bool threw = false;
|
||||
try {
|
||||
a.apply_remote(*lb);
|
||||
} catch (const molo::replication::ConflictError& e) {
|
||||
threw = true;
|
||||
CHECK(e.winner_version() == 2, "ConflictError carries winner version");
|
||||
CHECK(e.model() == "orders" && e.key() == "k1", "ConflictError names model and key");
|
||||
}
|
||||
CHECK(threw, "loser gets ConflictError");
|
||||
|
||||
// Sync keeps each incumbent; the conflict is counted, nothing is lost.
|
||||
sync_both(a, b, obs);
|
||||
CHECK(obs->counter_value("molo_replication_conflicts_total") > 0, "sync counts conflicts");
|
||||
CHECK(a.resolver().conflict_log_size() > 0, "conflict logged");
|
||||
|
||||
// Documented recovery: the loser re-reads and retries with a higher version.
|
||||
a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 3, "{\"by\":\"a3\"}");
|
||||
sync_both(a, b, obs);
|
||||
auto ca = a.current("orders", "k1");
|
||||
auto cb = b.current("orders", "k1");
|
||||
CHECK(ca.has_value() && cb.has_value(), "both converge");
|
||||
if (ca.has_value() && cb.has_value()) {
|
||||
CHECK(ca->version == 3 && cb->version == 3, "higher version wins everywhere");
|
||||
CHECK(ca->data == cb->data, "converged data agrees");
|
||||
}
|
||||
std::cout << "PASS test_optimistic_conflict_and_recovery" << std::endl;
|
||||
}
|
||||
|
||||
void test_lww_cache_conflict() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
|
||||
molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs);
|
||||
a.declare_model("cache", molo::replication::ConflictPolicy::LastWriterWins);
|
||||
b.declare_model("cache", molo::replication::ConflictPolicy::LastWriterWins);
|
||||
|
||||
// Same frozen millisecond: hybrid counters tie at first write, so the
|
||||
// instance id tie-break decides deterministically (node-b wins).
|
||||
a.local_write("cache", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
|
||||
b.local_write("cache", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"b\"}");
|
||||
sync_both(a, b, obs);
|
||||
auto ca = a.current("cache", "k");
|
||||
auto cb = b.current("cache", "k");
|
||||
CHECK(ca.has_value() && cb.has_value(), "both resolve LWW");
|
||||
if (ca.has_value() && cb.has_value()) {
|
||||
CHECK(ca->data == "{\"by\":\"b\"}" && cb->data == "{\"by\":\"b\"}",
|
||||
"higher hybrid timestamp wins on both");
|
||||
CHECK(ca->instance == "node-b", "winner instance recorded");
|
||||
}
|
||||
CHECK(a.resolver().conflict_log_size() == 1, "loser kept in conflict log");
|
||||
auto log = a.resolver().conflict_log(10);
|
||||
CHECK(!log.empty() && log[0].loser.data == "{\"by\":\"a\"}", "log holds the loser");
|
||||
|
||||
// A genuinely later write wins regardless of instance.
|
||||
clock.advance(5);
|
||||
a.local_write("cache", "k", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"a2\"}");
|
||||
sync_both(a, b, obs);
|
||||
CHECK(b.current("cache", "k")->data == "{\"by\":\"a2\"}", "later wall clock wins");
|
||||
std::cout << "PASS test_lww_cache_conflict" << std::endl;
|
||||
}
|
||||
|
||||
void test_hook_policy_and_fallback() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
|
||||
|
||||
// Hook that always prefers the remote side.
|
||||
a.declare_model("special", molo::replication::ConflictPolicy::Hook,
|
||||
[](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) {
|
||||
return 1;
|
||||
});
|
||||
a.local_write("special", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
|
||||
molo::replication::OplogEntry remote;
|
||||
remote.instance = "node-b";
|
||||
remote.ts = molo::replication::HybridTs{1760000000000LL, 0, "node-b"};
|
||||
remote.model = "special";
|
||||
remote.key = "k";
|
||||
remote.op = molo::replication::OplogOp::Upsert;
|
||||
remote.version = 1;
|
||||
remote.data = "{\"by\":\"b\"}";
|
||||
auto outcome = a.apply_remote(remote);
|
||||
CHECK(outcome.remote_won && outcome.winner.data == "{\"by\":\"b\"}", "hook decision honored");
|
||||
CHECK(a.resolver().conflict_log(10)[0].policy == "hook", "hook conflict logged");
|
||||
|
||||
// Throwing hook falls back to LWW immediately.
|
||||
a.declare_model("broken", molo::replication::ConflictPolicy::Hook,
|
||||
[](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) {
|
||||
throw std::runtime_error("hook bug");
|
||||
return 0;
|
||||
});
|
||||
a.local_write("broken", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
|
||||
molo::replication::OplogEntry remote2 = remote;
|
||||
remote2.model = "broken";
|
||||
remote2.ts = molo::replication::HybridTs{1760000000000LL, 99, "node-b"};
|
||||
auto outcome2 = a.apply_remote(remote2);
|
||||
CHECK(outcome2.detail.find("hook fallback") != std::string::npos, "throw falls back to LWW");
|
||||
CHECK(outcome2.remote_won, "LWW fallback picks the higher ts");
|
||||
|
||||
// Slow hook past the 5s budget falls back to LWW (S7-8).
|
||||
a.declare_model("slow", molo::replication::ConflictPolicy::Hook,
|
||||
[](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5200));
|
||||
return 0;
|
||||
});
|
||||
a.local_write("slow", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
|
||||
molo::replication::OplogEntry remote3 = remote;
|
||||
remote3.model = "slow";
|
||||
auto outcome3 = a.apply_remote(remote3);
|
||||
CHECK(outcome3.detail.find("hook fallback") != std::string::npos, "5s timeout falls back");
|
||||
std::cout << "PASS test_hook_policy_and_fallback" << std::endl;
|
||||
}
|
||||
|
||||
void test_sync_batches_and_lag() {
|
||||
namespace fs = std::filesystem;
|
||||
fs::remove_all(kTmp);
|
||||
molo::common::FrozenClock clock(1760000000000LL, 1000);
|
||||
auto obs = std::make_shared<molo::common::RecordingObserver>();
|
||||
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
|
||||
molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs);
|
||||
for (int i = 0; i < 600; ++i) {
|
||||
a.local_write("bulk", "k" + std::to_string(i), molo::replication::OplogOp::Upsert, 1, "{}");
|
||||
}
|
||||
ReplicatorSyncPeer peer_a(a);
|
||||
molo::replication::SyncSession session(b, peer_a, obs);
|
||||
auto s1 = session.round_trip();
|
||||
CHECK(s1.pulled == 500, "first round honors the 500-entry batch cap");
|
||||
CHECK(s1.lag == 100, "lag counts the remaining 100");
|
||||
auto s2 = session.round_trip();
|
||||
CHECK(s2.pulled == 100, "second round completes the catch-up");
|
||||
CHECK(s2.lag == 0, "lag zero when caught up");
|
||||
CHECK(b.current("bulk", "k599").has_value(), "tail entry arrived");
|
||||
CHECK(obs->gauges().count("molo_replication_sync_lag") == 1, "lag gauge exported");
|
||||
|
||||
// Deletes replicate as tombstones.
|
||||
a.local_write("bulk", "k0", molo::replication::OplogOp::Delete, 2, "{}");
|
||||
session.round_trip();
|
||||
auto tomb = b.current("bulk", "k0");
|
||||
CHECK(tomb.has_value() && tomb->op == molo::replication::OplogOp::Delete, "delete replicates");
|
||||
std::cout << "PASS test_sync_batches_and_lag" << std::endl;
|
||||
}
|
||||
|
||||
void test_vectors_clocks_and_codecs() {
|
||||
molo::replication::VersionVector v1, v2;
|
||||
v1.update("a", 5);
|
||||
v1.update("b", 3);
|
||||
v2.update("a", 5);
|
||||
v2.update("b", 4);
|
||||
CHECK(v1.compare(v2) == molo::replication::VectorOrder::Before, "vector before");
|
||||
CHECK(v2.compare(v1) == molo::replication::VectorOrder::After, "vector after");
|
||||
v1.update("c", 1);
|
||||
CHECK(v1.compare(v2) == molo::replication::VectorOrder::Concurrent, "vector concurrent");
|
||||
v1.merge(v2);
|
||||
CHECK(v1.get("b") == 4 && v1.get("c") == 1, "merge takes maxima");
|
||||
molo::replication::VersionVector back;
|
||||
CHECK(molo::replication::VersionVector::decode(v1.encode(), back), "vector codec roundtrip");
|
||||
CHECK(back.compare(v1) == molo::replication::VectorOrder::Equal, "vector codec preserves");
|
||||
|
||||
molo::common::FrozenClock clock(1000, 1000);
|
||||
molo::replication::HybridClock hybrid("n1", clock);
|
||||
auto t1 = hybrid.now();
|
||||
auto t2 = hybrid.now();
|
||||
CHECK(t2.compare(t1) > 0, "same-ms counter orders");
|
||||
clock.advance(10);
|
||||
auto t3 = hybrid.now();
|
||||
CHECK(t3.wall_ms == 1010 && t3.counter == 0, "wall advance resets the counter");
|
||||
molo::replication::HybridTs parsed;
|
||||
CHECK(molo::replication::HybridTs::decode(t3.encode(), parsed) && parsed == t3, "ts codec");
|
||||
|
||||
molo::replication::OplogEntry entry;
|
||||
entry.seq = 7;
|
||||
entry.instance = "n1";
|
||||
entry.ts = t3;
|
||||
entry.model = "m";
|
||||
entry.key = "k";
|
||||
entry.op = molo::replication::OplogOp::Upsert;
|
||||
entry.version = 2;
|
||||
entry.data = "{\"x\":1}";
|
||||
molo::replication::OplogEntry entry_back;
|
||||
CHECK(molo::replication::OplogEntry::decode(entry.encode(), entry_back), "oplog codec");
|
||||
CHECK(entry_back.seq == 7 && entry_back.version == 2 && entry_back.data == "{\"x\":1}",
|
||||
"oplog fields survive");
|
||||
CHECK(!molo::replication::OplogEntry::decode("nope", entry_back), "oplog rejects garbage");
|
||||
|
||||
molo::replication::ConflictResolver resolver(clock, molo::common::null_observer());
|
||||
CHECK(resolver.policy_for("undeclared") == molo::replication::ConflictPolicy::Optimistic,
|
||||
"default policy is optimistic");
|
||||
std::cout << "PASS test_vectors_clocks_and_codecs" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_optimistic_conflict_and_recovery();
|
||||
test_lww_cache_conflict();
|
||||
test_hook_policy_and_fallback();
|
||||
test_sync_batches_and_lag();
|
||||
test_vectors_clocks_and_codecs();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL REPLICATION TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
// V7-4: scheduler intervals, cron, leader failover without duplicates.
|
||||
#include <atomic>
|
||||
#include <iostream>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/kv_store.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
#include "supervisor/tasks/cron.hpp"
|
||||
#include "supervisor/tasks/scheduler.hpp"
|
||||
#include "supervisor/tasks/task_queue.hpp"
|
||||
#include "supervisor/tasks/task_registry.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
constexpr std::int64_t kT0 = 1760000000000LL; // fixed frozen origin
|
||||
|
||||
struct Harness {
|
||||
molo::common::FrozenClock clock{kT0, 1000};
|
||||
molo::common::InMemoryKvStore kv{clock};
|
||||
molo::tasks::TaskQueue queue{molo::tasks::QueueOptions{}, clock,
|
||||
molo::common::null_observer()};
|
||||
molo::tasks::TaskRegistry registry;
|
||||
std::atomic<long> runs{0};
|
||||
|
||||
Harness() {
|
||||
registry.register_handler("tick", [&] {
|
||||
struct H : molo::tasks::TaskHandler {
|
||||
std::atomic<long>* runs;
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
|
||||
++(*runs);
|
||||
return molo::tasks::TaskOutcome::Success;
|
||||
}
|
||||
};
|
||||
auto h = std::make_unique<H>();
|
||||
h->runs = &runs;
|
||||
return h;
|
||||
});
|
||||
}
|
||||
|
||||
int drain() {
|
||||
int n = 0;
|
||||
while (queue.run_one(registry, 0)) {
|
||||
++n;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// Advances the frozen clock one second per scheduler tick, like the
|
||||
// production 1s tick loop. Time jumps longer than the leader TTL would
|
||||
// (correctly) drop leadership, so tests step through time instead.
|
||||
void step(molo::tasks::Scheduler& sched, int seconds) {
|
||||
for (int i = 0; i < seconds; ++i) {
|
||||
clock.advance(1000);
|
||||
sched.tick_once();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void test_interval_fires_per_tick() {
|
||||
Harness h;
|
||||
molo::tasks::SchedulerOptions options;
|
||||
options.instance_id = "sched-a";
|
||||
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options);
|
||||
|
||||
molo::tasks::ScheduleEntry entry;
|
||||
entry.key = "every-minute";
|
||||
entry.task_name = "tick";
|
||||
entry.kind = molo::tasks::ScheduleKind::Interval;
|
||||
entry.interval_secs = 60;
|
||||
sched.put(entry);
|
||||
|
||||
sched.tick_once();
|
||||
CHECK(sched.is_leader(), "single scheduler wins the lease");
|
||||
CHECK(h.queue.stats().queued == 0, "no fire before the interval");
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
h.step(sched, 60);
|
||||
CHECK(h.drain() == 1, "one run per 60s tick");
|
||||
}
|
||||
CHECK(h.runs.load() == 5, "frozen clock advances 5 ticks, 5 runs");
|
||||
sched.tick_once();
|
||||
CHECK(h.drain() == 0, "no fire without time passing");
|
||||
std::cout << "PASS test_interval_fires_per_tick" << std::endl;
|
||||
}
|
||||
|
||||
void test_leader_failover_no_duplicates() {
|
||||
Harness h;
|
||||
molo::tasks::SchedulerOptions a_opts;
|
||||
a_opts.instance_id = "sched-a";
|
||||
a_opts.ttl_secs = 2;
|
||||
a_opts.renew_secs = 1;
|
||||
molo::tasks::Scheduler a(h.queue, h.kv, h.clock, molo::common::null_observer(), a_opts);
|
||||
molo::tasks::SchedulerOptions b_opts;
|
||||
b_opts.instance_id = "sched-b";
|
||||
b_opts.ttl_secs = 2;
|
||||
b_opts.renew_secs = 1;
|
||||
molo::tasks::Scheduler b(h.queue, h.kv, h.clock, molo::common::null_observer(), b_opts);
|
||||
|
||||
molo::tasks::ScheduleEntry entry;
|
||||
entry.key = "job";
|
||||
entry.task_name = "tick";
|
||||
entry.kind = molo::tasks::ScheduleKind::Interval;
|
||||
entry.interval_secs = 60;
|
||||
a.put(entry);
|
||||
b.put(entry);
|
||||
|
||||
a.tick_once();
|
||||
b.tick_once();
|
||||
CHECK(a.is_leader() && !b.is_leader(), "first scheduler wins the lease");
|
||||
|
||||
for (int i = 0; i < 60; ++i) {
|
||||
h.clock.advance(1000);
|
||||
a.tick_once(); // leader fires at the window edge
|
||||
b.tick_once(); // follower stays passive
|
||||
}
|
||||
CHECK(h.drain() == 1, "exactly one fire with two schedulers");
|
||||
|
||||
// Kill the leader: it stops ticking, its lease expires, the follower
|
||||
// takes over within the TTL (V7-4 bound is 15s; TTL here is 2s).
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
h.clock.advance(1000);
|
||||
b.tick_once();
|
||||
}
|
||||
CHECK(b.is_leader(), "follower takes over after lease expiry");
|
||||
const long runs_before = h.runs.load();
|
||||
b.tick_once(); // takeover tick must not refire the shared last-fire window
|
||||
CHECK(h.drain() == 0, "no duplicate run after failover");
|
||||
CHECK(h.runs.load() == runs_before, "run count unchanged by failover");
|
||||
|
||||
h.step(b, 60);
|
||||
CHECK(h.drain() == 1, "new leader fires the next window");
|
||||
std::cout << "PASS test_leader_failover_no_duplicates" << std::endl;
|
||||
}
|
||||
|
||||
void test_cron_entries() {
|
||||
// Unit checks on the expression engine first.
|
||||
molo::tasks::CronExpr expr;
|
||||
std::string error;
|
||||
CHECK(molo::tasks::CronExpr::parse("* * * * *", expr, error), "every-minute parses");
|
||||
CHECK(molo::tasks::CronExpr::parse("*/15 9-17 * * 1-5", expr, error), "business hours parse");
|
||||
CHECK(!molo::tasks::CronExpr::parse("* * * *", expr, error), "4 fields rejected");
|
||||
CHECK(!molo::tasks::CronExpr::parse("61 * * * *", expr, error), "minute 61 rejected");
|
||||
CHECK(molo::tasks::CronExpr::parse("0 0 * * 7", expr, error), "dow 7 means sunday");
|
||||
|
||||
// 2026-10-23 is a Friday. 12:00 UTC that day:
|
||||
const std::int64_t friday_noon = 1792756800000LL; // 2026-10-23 12:00 UTC, a Friday
|
||||
molo::tasks::CronExpr noon;
|
||||
CHECK(molo::tasks::CronExpr::parse("0 12 * * *", noon, error), "noon parses");
|
||||
CHECK(noon.matches(friday_noon + 30000), "noon matches inside the minute");
|
||||
CHECK(!noon.matches(friday_noon + 60000), "noon misses the next minute");
|
||||
CHECK(noon.next_after(friday_noon) == friday_noon + 86400000LL, "next noon is tomorrow");
|
||||
|
||||
// End to end through the scheduler with a frozen clock.
|
||||
Harness h;
|
||||
h.clock.set(friday_noon - 60000, 1000); // 11:59
|
||||
molo::tasks::SchedulerOptions options;
|
||||
options.instance_id = "sched-cron";
|
||||
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options);
|
||||
molo::tasks::ScheduleEntry entry;
|
||||
entry.key = "noon-job";
|
||||
entry.task_name = "tick";
|
||||
entry.kind = molo::tasks::ScheduleKind::Cron;
|
||||
entry.cron = "0 12 * * *";
|
||||
sched.put(entry);
|
||||
sched.tick_once();
|
||||
CHECK(h.drain() == 0, "no fire at 11:59");
|
||||
h.step(sched, 60); // step into 12:00
|
||||
CHECK(h.drain() == 1, "fires at 12:00");
|
||||
sched.tick_once();
|
||||
CHECK(h.drain() == 0, "fires once per minute");
|
||||
std::cout << "PASS test_cron_entries" << std::endl;
|
||||
}
|
||||
|
||||
void test_once_and_missed_policy() {
|
||||
Harness h;
|
||||
auto obs = std::make_shared<molo::common::RecordingObserver>();
|
||||
molo::tasks::SchedulerOptions options;
|
||||
options.instance_id = "sched-once";
|
||||
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, obs, options);
|
||||
|
||||
molo::tasks::ScheduleEntry skip;
|
||||
skip.key = "once-skip";
|
||||
skip.task_name = "tick";
|
||||
skip.kind = molo::tasks::ScheduleKind::Once;
|
||||
skip.at_ms = kT0;
|
||||
skip.missed = molo::tasks::MissedPolicy::Skip;
|
||||
molo::tasks::ScheduleEntry run;
|
||||
run.key = "once-run";
|
||||
run.task_name = "tick";
|
||||
run.kind = molo::tasks::ScheduleKind::Once;
|
||||
run.at_ms = kT0;
|
||||
run.missed = molo::tasks::MissedPolicy::RunOnce;
|
||||
sched.put(skip);
|
||||
sched.put(run);
|
||||
|
||||
h.clock.advance(2 * 3600 * 1000); // miss both by 2h
|
||||
sched.tick_once();
|
||||
CHECK(h.drain() == 1, "run-once fires, skip drops");
|
||||
CHECK(obs->event_count("tasks.schedule.missed") == 1, "miss recorded");
|
||||
sched.tick_once();
|
||||
CHECK(h.drain() == 0, "once entries never refire");
|
||||
std::cout << "PASS test_once_and_missed_policy" << std::endl;
|
||||
}
|
||||
|
||||
void test_interval_floor_and_bad_cron() {
|
||||
Harness h;
|
||||
molo::tasks::SchedulerOptions options;
|
||||
options.instance_id = "sched-floor";
|
||||
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options);
|
||||
molo::tasks::ScheduleEntry fast;
|
||||
fast.key = "fast";
|
||||
fast.task_name = "tick";
|
||||
fast.kind = molo::tasks::ScheduleKind::Interval;
|
||||
fast.interval_secs = 5;
|
||||
sched.put(fast);
|
||||
CHECK(sched.entries()[0].interval_secs == 60, "interval floored at 60s");
|
||||
|
||||
molo::tasks::ScheduleEntry bad;
|
||||
bad.key = "bad";
|
||||
bad.task_name = "tick";
|
||||
bad.kind = molo::tasks::ScheduleKind::Cron;
|
||||
bad.cron = "not a cron";
|
||||
bool threw = false;
|
||||
try {
|
||||
sched.put(bad);
|
||||
} catch (const molo::tasks::TaskError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw, "bad cron rejected at put");
|
||||
CHECK(sched.remove("fast"), "remove works");
|
||||
CHECK(!sched.remove("fast"), "double remove fails");
|
||||
std::cout << "PASS test_interval_floor_and_bad_cron" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_interval_fires_per_tick();
|
||||
test_leader_failover_no_duplicates();
|
||||
test_cron_entries();
|
||||
test_once_and_missed_policy();
|
||||
test_interval_floor_and_bad_cron();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL SCHEDULER TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
// V7-1: supervisor restarts, intensity caps, strategies, health, status.
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
#include "supervisor/supervisor/supervisor.hpp"
|
||||
#include "supervisor/supervisor/supervisor_error.hpp"
|
||||
#include "supervisor/supervisor/worker_group.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
bool wait_for(const std::function<bool()>& pred, long timeout_ms) {
|
||||
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms);
|
||||
while (std::chrono::steady_clock::now() < deadline) {
|
||||
if (pred()) {
|
||||
return true;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
||||
}
|
||||
return pred();
|
||||
}
|
||||
|
||||
int run_until_stopped(const std::atomic<bool>& stop) {
|
||||
while (!stop.load()) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void test_three_workers_restart_and_status() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = std::make_shared<molo::common::RecordingObserver>();
|
||||
molo::supervisor::Supervisor sup(clock, obs);
|
||||
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "web";
|
||||
spec.size = 3;
|
||||
spec.monitor_ms = 10;
|
||||
spec.backoff_min_ms = 10;
|
||||
spec.backoff_max_ms = 50;
|
||||
const std::size_t idx = sup.add_group(spec, run_until_stopped);
|
||||
CHECK(idx == 0, "first group index is 0");
|
||||
sup.start_all();
|
||||
|
||||
CHECK(wait_for([&] {
|
||||
const std::string status = sup.status_json();
|
||||
return status.find("\"state\":\"running\"") != std::string::npos;
|
||||
}, 2000), "workers reach running");
|
||||
|
||||
const std::string status = sup.status_json();
|
||||
CHECK(status.find("\"name\":\"web\"") != std::string::npos, "status names the group");
|
||||
CHECK(status.find("\"phase\":\"running\"") != std::string::npos, "status shows running phase");
|
||||
CHECK(status.size() <= 1048576, "status under the 1 MiB cap");
|
||||
CHECK(obs->counter_value("molo_supervisor_worker_starts_total") >= 3, "3 starts counted");
|
||||
|
||||
sup.shutdown(2000);
|
||||
CHECK(sup.phase() == molo::supervisor::ShutdownPhase::Stopped, "shutdown reaches stopped");
|
||||
std::cout << "PASS test_three_workers_restart_and_status" << std::endl;
|
||||
}
|
||||
|
||||
void test_crash_restart_then_fatal() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = std::make_shared<molo::common::RecordingObserver>();
|
||||
molo::supervisor::Supervisor sup(clock, obs);
|
||||
|
||||
std::atomic<int> runs{0};
|
||||
auto crasher = [&](const std::atomic<bool>&) {
|
||||
++runs;
|
||||
return 1; // always crashes
|
||||
};
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "crasher";
|
||||
spec.size = 1;
|
||||
spec.monitor_ms = 5;
|
||||
spec.backoff_min_ms = 10;
|
||||
spec.backoff_max_ms = 20;
|
||||
spec.restart.policy = molo::supervisor::RestartPolicy::OnFailure;
|
||||
spec.restart.max_restarts = 2;
|
||||
spec.restart.window_secs = 60;
|
||||
sup.add_group(spec, crasher);
|
||||
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
sup.start_all();
|
||||
// Restart latency: the first restart must happen within 2s (V7-1).
|
||||
CHECK(wait_for([&] { return runs.load() >= 2; }, 2000), "crasher restarted under 2s");
|
||||
const auto restarted_ms =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0)
|
||||
.count();
|
||||
CHECK(restarted_ms < 2000, "restart latency under 2s");
|
||||
// Intensity cap: initial + 2 restarts, then fatal.
|
||||
CHECK(wait_for([&] { return sup.status_json().find("\"state\":\"fatal\"") != std::string::npos; },
|
||||
5000),
|
||||
"crasher goes fatal after the cap");
|
||||
CHECK(runs.load() == 3, "exactly initial plus max_restarts runs");
|
||||
CHECK(obs->counter_value("molo_supervisor_worker_fatal_total") == 1, "fatal counted");
|
||||
sup.shutdown(1000);
|
||||
std::cout << "PASS test_crash_restart_then_fatal" << std::endl;
|
||||
}
|
||||
|
||||
void test_explicit_restart_clears_fatal() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::supervisor::WorkerGroup group(
|
||||
[] {
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "flaky";
|
||||
spec.monitor_ms = 5;
|
||||
spec.backoff_min_ms = 5;
|
||||
spec.backoff_max_ms = 10;
|
||||
spec.restart.max_restarts = 1;
|
||||
return spec;
|
||||
}(),
|
||||
[](const std::atomic<bool>&) { return 1; }, clock, obs);
|
||||
group.start();
|
||||
CHECK(wait_for([&] { return group.fatal(); }, 5000), "group goes fatal");
|
||||
group.restart_child(0); // explicit restart clears Fatal
|
||||
CHECK(wait_for([&] { return group.fatal(); }, 5000), "fatal again after crash loop");
|
||||
group.stop();
|
||||
std::cout << "PASS test_explicit_restart_clears_fatal" << std::endl;
|
||||
}
|
||||
|
||||
void test_strategies_recover() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
const molo::supervisor::GroupStrategy strategies[] = {
|
||||
molo::supervisor::GroupStrategy::OneForOne,
|
||||
molo::supervisor::GroupStrategy::OneForAll,
|
||||
molo::supervisor::GroupStrategy::RestForOne,
|
||||
};
|
||||
for (auto strategy : strategies) {
|
||||
std::atomic<int> failures_left{1};
|
||||
auto sometimes_fails = [&](const std::atomic<bool>& stop) {
|
||||
if (failures_left.load() > 0 && failures_left.fetch_sub(1) == 1) {
|
||||
return 1; // exactly one crash across the whole group
|
||||
}
|
||||
return run_until_stopped(stop);
|
||||
};
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "strat";
|
||||
spec.strategy = strategy;
|
||||
spec.size = 2;
|
||||
spec.monitor_ms = 5;
|
||||
spec.backoff_min_ms = 5;
|
||||
spec.backoff_max_ms = 10;
|
||||
molo::supervisor::WorkerGroup group(spec, sometimes_fails, clock, obs);
|
||||
group.start();
|
||||
CHECK(wait_for([&] {
|
||||
auto states = group.states();
|
||||
return states.size() == 2 && states[0] == molo::supervisor::WorkerState::Running &&
|
||||
states[1] == molo::supervisor::WorkerState::Running;
|
||||
}, 5000), "strategy recovers to all running: " + molo::supervisor::to_string(strategy));
|
||||
CHECK(!group.fatal(), "no fatal after single failure");
|
||||
group.stop();
|
||||
}
|
||||
std::cout << "PASS test_strategies_recover" << std::endl;
|
||||
}
|
||||
|
||||
void test_health_probe_triggers_restart() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "sick";
|
||||
spec.monitor_ms = 5;
|
||||
spec.backoff_min_ms = 5;
|
||||
spec.backoff_max_ms = 10;
|
||||
spec.restart.max_restarts = 1;
|
||||
spec.health.interval_ms = 20;
|
||||
spec.health.max_failures = 2;
|
||||
spec.probe = [] { return false; }; // always sick
|
||||
std::atomic<int> runs{0};
|
||||
auto worker = [&](const std::atomic<bool>& stop) {
|
||||
++runs;
|
||||
return run_until_stopped(stop);
|
||||
};
|
||||
molo::supervisor::WorkerGroup group(spec, worker, clock, obs);
|
||||
group.start();
|
||||
CHECK(wait_for([&] { return group.fatal(); }, 8000), "sick worker restarts then fatal");
|
||||
CHECK(runs.load() >= 2, "unhealthy worker was restarted");
|
||||
group.stop();
|
||||
|
||||
// Healthy probe: worker stays up.
|
||||
molo::supervisor::GroupSpec good = spec;
|
||||
good.name = "healthy";
|
||||
good.probe = [] { return true; };
|
||||
molo::supervisor::WorkerGroup group2(good, run_until_stopped, clock, obs);
|
||||
group2.start();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(150));
|
||||
auto states = group2.states();
|
||||
CHECK(states[0] == molo::supervisor::WorkerState::Running, "healthy worker stays running");
|
||||
CHECK(!group2.fatal(), "healthy group not fatal");
|
||||
group2.stop();
|
||||
std::cout << "PASS test_health_probe_triggers_restart" << std::endl;
|
||||
}
|
||||
|
||||
void test_scale_and_restart_group() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::supervisor::Supervisor sup(clock, obs);
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "elastic";
|
||||
spec.monitor_ms = 5;
|
||||
spec.backoff_min_ms = 5;
|
||||
const std::size_t idx = sup.add_group(spec, run_until_stopped);
|
||||
sup.start_all();
|
||||
sup.scale_group(idx, 4);
|
||||
CHECK(wait_for([&] {
|
||||
const std::string status = sup.status_json();
|
||||
std::size_t count = 0, pos = 0;
|
||||
while ((pos = status.find("\"state\":\"running\"", pos)) != std::string::npos) {
|
||||
++count;
|
||||
++pos;
|
||||
}
|
||||
return count == 4;
|
||||
}, 3000), "scales to 4 running");
|
||||
sup.scale_group(idx, 1);
|
||||
CHECK(wait_for([&] { return sup.status_json().find("elastic#1") == std::string::npos; }, 3000),
|
||||
"scales down to 1");
|
||||
bool threw = false;
|
||||
try {
|
||||
sup.scale_group(idx, 65);
|
||||
} catch (const molo::supervisor::SupervisorError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw, "scale past 64 throws");
|
||||
sup.restart_group(idx);
|
||||
CHECK(wait_for([&] {
|
||||
return sup.status_json().find("\"state\":\"running\"") != std::string::npos;
|
||||
}, 3000), "group restarts");
|
||||
sup.shutdown(2000);
|
||||
std::cout << "PASS test_scale_and_restart_group" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_three_workers_restart_and_status();
|
||||
test_crash_restart_then_fatal();
|
||||
test_explicit_restart_clears_fatal();
|
||||
test_strategies_recover();
|
||||
test_health_probe_triggers_restart();
|
||||
test_scale_and_restart_group();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL SUPERVISOR TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
// V7-2, V7-3: task throughput, retries, backoff, dead letters, timeouts.
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
#include "supervisor/tasks/retry_policy.hpp"
|
||||
#include "supervisor/tasks/task_envelope.hpp"
|
||||
#include "supervisor/tasks/task_queue.hpp"
|
||||
#include "supervisor/tasks/task_registry.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
class CountingHandler : public molo::tasks::TaskHandler {
|
||||
public:
|
||||
explicit CountingHandler(std::atomic<long>& count) : count_(count) {}
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
|
||||
++count_;
|
||||
return molo::tasks::TaskOutcome::Success;
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic<long>& count_;
|
||||
};
|
||||
|
||||
void test_throughput_off_loop() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::tasks::QueueOptions options;
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
molo::tasks::TaskRegistry registry;
|
||||
std::atomic<long> done{0};
|
||||
registry.register_handler("count", [&] { return std::make_unique<CountingHandler>(done); });
|
||||
|
||||
// Baseline scheduling latency of this thread.
|
||||
auto probe = [] {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
||||
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
|
||||
t0)
|
||||
.count();
|
||||
};
|
||||
long baseline = 0;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
baseline += probe();
|
||||
}
|
||||
baseline /= 10;
|
||||
|
||||
std::atomic<bool> stop{false};
|
||||
std::vector<std::thread> workers;
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
workers.emplace_back([&] { queue.run_worker([&] { return stop.load(); }, registry); });
|
||||
}
|
||||
const int total = 500;
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
for (int i = 0; i < total; ++i) {
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "count";
|
||||
queue.enqueue(std::move(env));
|
||||
}
|
||||
const auto enqueue_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - t0)
|
||||
.count();
|
||||
long loaded = 0;
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
loaded += probe();
|
||||
}
|
||||
loaded /= 10;
|
||||
while (done.load() < total) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
||||
}
|
||||
stop = true;
|
||||
for (auto& w : workers) {
|
||||
w.join();
|
||||
}
|
||||
CHECK(done.load() == total, "all 500 tasks succeed");
|
||||
CHECK(queue.stats().succeeded_total == total, "stats count all successes");
|
||||
CHECK(enqueue_ms < 2000, "enqueue never blocks on execution");
|
||||
CHECK(loaded < std::max<long>(50000, baseline * 10), "producer thread stays responsive");
|
||||
std::cout << "PASS test_throughput_off_loop" << std::endl;
|
||||
}
|
||||
|
||||
void test_retry_backoff_schedule() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::tasks::QueueOptions options;
|
||||
options.lease_ms = 60000;
|
||||
options.retry = molo::tasks::RetrySpec{100, 1000, 0, 5}; // base 100ms, no jitter
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
molo::tasks::TaskRegistry registry;
|
||||
std::vector<std::int64_t> runs;
|
||||
std::mutex runs_mutex;
|
||||
registry.register_handler("flaky", [&] {
|
||||
struct H : molo::tasks::TaskHandler {
|
||||
std::vector<std::int64_t>* runs;
|
||||
std::mutex* mutex;
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
|
||||
std::lock_guard<std::mutex> lock(*mutex);
|
||||
runs->push_back(ctx.clock.steady_ms());
|
||||
return runs->size() < 3 ? molo::tasks::TaskOutcome::RetryableFailure
|
||||
: molo::tasks::TaskOutcome::Success;
|
||||
}
|
||||
};
|
||||
auto h = std::make_unique<H>();
|
||||
h->runs = &runs;
|
||||
h->mutex = &runs_mutex;
|
||||
return h;
|
||||
});
|
||||
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "flaky";
|
||||
env.max_retries = 5;
|
||||
const std::string id = queue.enqueue(std::move(env));
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
CHECK(queue.run_one(registry, 5000), "run_one executes the retry");
|
||||
}
|
||||
CHECK(runs.size() == 3, "handler ran 3 times");
|
||||
if (runs.size() == 3) {
|
||||
const long d1 = static_cast<long>(runs[1] - runs[0]);
|
||||
const long d2 = static_cast<long>(runs[2] - runs[1]);
|
||||
// delay(0)=100ms, delay(1)=200ms, within 20 percent plus 40ms slack.
|
||||
CHECK(d1 >= 80 && d1 <= 100 * 1.2 + 40, "first backoff near 100ms");
|
||||
CHECK(d2 >= 160 && d2 <= 200 * 1.2 + 40, "second backoff near 200ms");
|
||||
}
|
||||
CHECK(queue.stats().succeeded_total == 1, "eventual success counted");
|
||||
CHECK(queue.stats().retried_total == 2, "two retries counted");
|
||||
CHECK(!queue.get(id).has_value(), "acked task leaves the queue");
|
||||
std::cout << "PASS test_retry_backoff_schedule" << std::endl;
|
||||
}
|
||||
|
||||
void test_permanent_failure_dead_letters() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
|
||||
molo::tasks::TaskRegistry registry;
|
||||
registry.register_handler("doomed", [] {
|
||||
struct H : molo::tasks::TaskHandler {
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
|
||||
return molo::tasks::TaskOutcome::PermanentFailure;
|
||||
}
|
||||
};
|
||||
return std::make_unique<H>();
|
||||
});
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "doomed";
|
||||
const std::string id = queue.enqueue(std::move(env));
|
||||
CHECK(queue.run_one(registry, 1000), "doomed task executes");
|
||||
CHECK(queue.dead_letters().size() == 1, "permanent failure dead-letters");
|
||||
auto entries = queue.dead_letters().list(10);
|
||||
CHECK(entries[0].envelope.id == id, "dead entry keeps the task id");
|
||||
CHECK(entries[0].reason == "handler-permanent", "dead entry keeps the reason");
|
||||
|
||||
// Unknown handler names dead-letter too, never crash the worker.
|
||||
molo::tasks::TaskEnvelope ghost;
|
||||
ghost.name = "no.such.handler";
|
||||
queue.enqueue(std::move(ghost));
|
||||
CHECK(queue.run_one(registry, 1000), "ghost task executes");
|
||||
CHECK(queue.dead_letters().size() == 2, "unknown handler dead-letters");
|
||||
std::cout << "PASS test_permanent_failure_dead_letters" << std::endl;
|
||||
}
|
||||
|
||||
void test_idempotency_keys() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
|
||||
molo::tasks::TaskRegistry registry;
|
||||
std::atomic<long> done{0};
|
||||
registry.register_handler("once", [&] { return std::make_unique<CountingHandler>(done); });
|
||||
|
||||
molo::tasks::TaskEnvelope a;
|
||||
a.name = "once";
|
||||
a.idempotency_key = "order-123";
|
||||
const std::string first = queue.enqueue(a);
|
||||
molo::tasks::TaskEnvelope b;
|
||||
b.name = "once";
|
||||
b.idempotency_key = "order-123";
|
||||
const std::string second = queue.enqueue(std::move(b));
|
||||
CHECK(first == second, "duplicate key returns the existing id");
|
||||
CHECK(queue.stats().queued == 1, "duplicate does not re-enqueue");
|
||||
CHECK(queue.run_one(registry, 1000), "task runs");
|
||||
molo::tasks::TaskEnvelope c;
|
||||
c.name = "once";
|
||||
c.idempotency_key = "order-123";
|
||||
CHECK(queue.enqueue(std::move(c)) == first, "key reserved after completion");
|
||||
CHECK(done.load() == 1, "handler ran exactly once");
|
||||
std::cout << "PASS test_idempotency_keys" << std::endl;
|
||||
}
|
||||
|
||||
void test_caps_and_lease_expiry() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::tasks::QueueOptions options;
|
||||
options.max_depth = 2;
|
||||
options.lease_ms = 60;
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
|
||||
molo::tasks::TaskEnvelope big;
|
||||
big.name = "big";
|
||||
big.payload = std::string(1048576 + 1, 'x');
|
||||
bool threw = false;
|
||||
try {
|
||||
queue.enqueue(std::move(big));
|
||||
} catch (const molo::tasks::TaskError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw, "oversize payload rejected");
|
||||
|
||||
molo::tasks::TaskEnvelope t1;
|
||||
t1.name = "t";
|
||||
queue.enqueue(std::move(t1));
|
||||
molo::tasks::TaskEnvelope t2;
|
||||
t2.name = "t";
|
||||
queue.enqueue(std::move(t2));
|
||||
threw = false;
|
||||
try {
|
||||
molo::tasks::TaskEnvelope t3;
|
||||
t3.name = "t";
|
||||
queue.enqueue(std::move(t3));
|
||||
} catch (const molo::tasks::TaskError&) {
|
||||
threw = true;
|
||||
}
|
||||
CHECK(threw, "depth cap enforced");
|
||||
CHECK(!queue.heartbeat("nope"), "unknown heartbeat fails");
|
||||
|
||||
// Lease expiry on a fresh queue with a single task: the expired lease
|
||||
// redelivers the same envelope with attempts incremented.
|
||||
molo::tasks::QueueOptions exp_options;
|
||||
exp_options.lease_ms = 60;
|
||||
molo::tasks::TaskQueue exp_queue(exp_options, clock, obs);
|
||||
molo::tasks::TaskEnvelope solo;
|
||||
solo.name = "solo";
|
||||
exp_queue.enqueue(std::move(solo));
|
||||
auto leased = exp_queue.lease(0);
|
||||
CHECK(leased.has_value(), "leases one task");
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(120));
|
||||
auto redelivered = exp_queue.lease(0);
|
||||
CHECK(redelivered.has_value(), "expired lease redelivers");
|
||||
if (redelivered.has_value()) {
|
||||
CHECK(redelivered->envelope.id == leased->envelope.id, "same task redelivered");
|
||||
CHECK(redelivered->envelope.attempts == 1, "expiry increments attempts");
|
||||
CHECK(!exp_queue.ack(leased->lease_id), "stale lease cannot ack");
|
||||
CHECK(exp_queue.ack(redelivered->lease_id), "fresh lease acks");
|
||||
}
|
||||
std::cout << "PASS test_caps_and_lease_expiry" << std::endl;
|
||||
}
|
||||
|
||||
void test_hard_timeout_recycles() {
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = molo::common::null_observer();
|
||||
molo::tasks::QueueOptions options;
|
||||
options.orphan_grace_ms = 2000;
|
||||
molo::tasks::TaskQueue queue(options, clock, obs);
|
||||
molo::tasks::TaskRegistry registry;
|
||||
std::atomic<bool> saw_cancel{false};
|
||||
registry.register_handler("slow", [&] {
|
||||
struct H : molo::tasks::TaskHandler {
|
||||
std::atomic<bool>* saw_cancel;
|
||||
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(300));
|
||||
if (ctx.cancel_requested.load()) {
|
||||
saw_cancel->store(true);
|
||||
}
|
||||
return molo::tasks::TaskOutcome::Success;
|
||||
}
|
||||
};
|
||||
auto h = std::make_unique<H>();
|
||||
h->saw_cancel = &saw_cancel;
|
||||
return h;
|
||||
});
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.name = "slow";
|
||||
env.timeout_soft_ms = 50;
|
||||
env.timeout_hard_ms = 100;
|
||||
env.max_retries = 0; // one strike, then dead
|
||||
queue.enqueue(std::move(env));
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
CHECK(queue.run_one(registry, 1000), "slow task executes");
|
||||
const auto took = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - t0)
|
||||
.count();
|
||||
CHECK(took < 2000, "hard timeout bounds the run");
|
||||
CHECK(queue.recycle_requested(), "worker recycle requested");
|
||||
CHECK(saw_cancel.load(), "soft timeout sets the cancel flag");
|
||||
CHECK(queue.dead_letters().size() == 1, "exhausted task dead-letters");
|
||||
auto entries = queue.dead_letters().list(10);
|
||||
CHECK(entries[0].reason == "hard-timeout", "dead reason names the hard timeout");
|
||||
std::cout << "PASS test_hard_timeout_recycles" << std::endl;
|
||||
}
|
||||
|
||||
void test_envelope_json_roundtrip() {
|
||||
molo::tasks::TaskEnvelope env;
|
||||
env.id = "abc";
|
||||
env.name = "billing.charge";
|
||||
env.payload = "{\"amount\":42}";
|
||||
env.attempts = 2;
|
||||
env.max_retries = 5;
|
||||
env.run_at_ms = 1700000000000LL;
|
||||
env.timeout_soft_ms = 30000;
|
||||
env.timeout_hard_ms = 60000;
|
||||
env.idempotency_key = "k";
|
||||
env.enqueued_at_ms = 1700000001000LL;
|
||||
const std::string json = env.encode();
|
||||
molo::tasks::TaskEnvelope back;
|
||||
CHECK(molo::tasks::TaskEnvelope::decode(json, back), "envelope decodes");
|
||||
CHECK(back.id == "abc" && back.name == "billing.charge", "identity survives");
|
||||
CHECK(back.payload == "{\"amount\":42}", "payload survives");
|
||||
CHECK(back.attempts == 2 && back.max_retries == 5, "counters survive");
|
||||
CHECK(back.run_at_ms.has_value() && *back.run_at_ms == 1700000000000LL, "run_at survives");
|
||||
CHECK(back.idempotency_key == "k", "key survives");
|
||||
CHECK(!molo::tasks::TaskEnvelope::decode("{oops", back), "garbage rejected");
|
||||
CHECK(!molo::tasks::TaskEnvelope::decode("{\"id\":1}", back), "typeless rejected");
|
||||
CHECK(!molo::tasks::TaskEnvelope::generate_id().empty(), "ids generated");
|
||||
CHECK(molo::tasks::TaskEnvelope::generate_id() != molo::tasks::TaskEnvelope::generate_id(),
|
||||
"ids unique");
|
||||
std::cout << "PASS test_envelope_json_roundtrip" << std::endl;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_throughput_off_loop();
|
||||
test_retry_backoff_schedule();
|
||||
test_permanent_failure_dead_letters();
|
||||
test_idempotency_keys();
|
||||
test_caps_and_lease_expiry();
|
||||
test_hard_timeout_recycles();
|
||||
test_envelope_json_roundtrip();
|
||||
if (failures == 0) {
|
||||
std::cout << "ALL TASK TESTS PASSED" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
std::cout << failures << " FAILURES" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
Reference in New Issue
Block a user