feat: implement MoloApp plans 01-15 with integration

Full C++23 framework tree: runtime, observability, security,
data, app, realtime, supervisor, clients, CLI, example plus
enterprise tenancy, audit, operability, governance,
supplychain. Postgres via docker compose for tests.
This commit is contained in:
2026-10-23 18:20:04 +02:00
parent 54068f1953
commit 3c4334cec9
1068 changed files with 100899 additions and 2 deletions
+205
View File
@@ -0,0 +1,205 @@
// V7-5: durable streams (persist, reconnect, redeliver) and ephemeral topics.
#include <filesystem>
#include <iostream>
#include <set>
#include "supervisor/bus/stream_bus.hpp"
#include "supervisor/bus/topic_bus.hpp"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
const std::filesystem::path kTmp = "build/supervisor/tmp/test-bus";
void test_durable_stream_reconnect_no_loss() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = molo::common::null_observer();
{
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
bus.create_stream("events");
bus.create_consumer("events", "worker", 60000);
for (int i = 1; i <= 10000; ++i) {
bus.publish("events", "msg-" + std::to_string(i));
}
// Consume half, then "disconnect" by destroying the bus mid-stream.
int acked = 0;
while (acked < 5000) {
auto batch = bus.poll("events", "worker", 500);
CHECK(!batch.empty(), "poll returns messages");
for (const auto& msg : batch) {
CHECK(msg.data == "msg-" + std::to_string(msg.seq), "payload matches seq");
CHECK(bus.ack("events", "worker", msg.seq), "ack works");
++acked;
}
}
}
// Reconnect: a fresh bus over the same directory replays the log and
// the persisted cursor resumes exactly where the acked prefix ended.
{
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
bus.create_stream("events");
bus.create_consumer("events", "worker", 60000);
std::set<std::uint64_t> seen;
std::uint64_t expect = 5001;
while (seen.size() < 5000) {
auto batch = bus.poll("events", "worker", 500);
CHECK(!batch.empty(), "reconnected poll returns messages");
for (const auto& msg : batch) {
CHECK(msg.seq == expect, "no gaps after reconnect");
++expect;
seen.insert(msg.seq);
CHECK(bus.ack("events", "worker", msg.seq), "reconnected ack works");
}
}
CHECK(seen.size() == 5000 && *seen.begin() == 5001 && *seen.rbegin() == 10000,
"zero loss, zero gaps across reconnect");
CHECK(bus.lag("events", "worker") == 0, "caught up, lag zero");
}
std::cout << "PASS test_durable_stream_reconnect_no_loss" << std::endl;
}
void test_redelivery_until_acked() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = std::make_shared<molo::common::RecordingObserver>();
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
bus.create_stream("jobs");
bus.create_consumer("jobs", "w", 1000); // 1s ack deadline
bus.publish("jobs", "a");
bus.publish("jobs", "b");
auto first = bus.poll("jobs", "w", 10);
CHECK(first.size() == 2, "two messages delivered");
CHECK(first[0].redeliveries == 0, "first delivery is fresh");
// No acks: advance past the deadline, the same messages redeliver.
clock.advance(1500);
auto second = bus.poll("jobs", "w", 10);
CHECK(second.size() == 2, "both redeliver after deadline");
CHECK(second[0].seq == first[0].seq && second[0].redeliveries == 1, "redelivery counted");
CHECK(obs->counter_value("molo_bus_stream_redelivered_total") == 2, "redelivery metric");
CHECK(bus.ack("jobs", "w", first[0].seq), "ack first");
CHECK(!bus.ack("jobs", "w", first[0].seq), "double ack fails");
clock.advance(1500);
auto third = bus.poll("jobs", "w", 10);
CHECK(third.size() == 1 && third[0].seq == first[1].seq, "only unacked redelivers");
CHECK(bus.renew("jobs", "w", first[1].seq), "renew extends the deadline");
clock.advance(500);
CHECK(bus.poll("jobs", "w", 10).empty(), "renewed message waits");
clock.advance(600);
CHECK(bus.poll("jobs", "w", 10).size() == 1, "redelivers after renewed deadline");
std::cout << "PASS test_redelivery_until_acked" << std::endl;
}
void test_retention_and_caps() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = molo::common::null_observer();
molo::bus::StreamBus bus(kTmp / "streams", clock, obs);
molo::bus::StreamSpec tiny;
tiny.retention_bytes = 1024;
tiny.retention_hours = 0; // size-only retention
bus.create_stream("tiny", tiny);
bus.create_consumer("tiny", "c");
for (int i = 0; i < 100; ++i) {
bus.publish("tiny", std::string(100, 'x'));
}
CHECK(bus.last_seq("tiny") == 100, "seqs assigned across retention");
auto retained = bus.poll("tiny", "c", 200);
CHECK(retained.size() < 100 && !retained.empty(), "retention evicted the oldest");
if (!retained.empty()) {
CHECK(retained.front().seq > 1, "oldest seqs gone");
CHECK(retained.back().seq == 100, "tail intact");
}
molo::bus::StreamSpec timed;
timed.retention_hours = 1;
timed.retention_bytes = 1024 * 1024 * 1024;
bus.create_stream("timed", timed);
bus.publish("timed", "old");
clock.advance(2 * 3600 * 1000);
CHECK(bus.enforce_retention("timed") == 1, "time retention evicts one");
bus.publish("timed", "new");
bool threw = false;
try {
bus.publish("tiny", std::string(1048576 + 1, 'y'));
} catch (const molo::bus::BusError&) {
threw = true;
}
CHECK(threw, "oversize stream payload rejected");
threw = false;
try {
bus.create_stream("../escape");
} catch (const molo::bus::BusError&) {
threw = true;
}
CHECK(threw, "path escape in stream name rejected");
std::cout << "PASS test_retention_and_caps" << std::endl;
}
void test_ephemeral_topics() {
molo::common::SystemClock clock;
molo::bus::TopicBus bus(clock);
CHECK(bus.publish("a:b", "lost") == 0, "no-subscriber publish is a counted no-op");
const auto sub1 = bus.subscribe("a:b");
const auto sub2 = bus.subscribe("a:*");
const auto late = bus.subscribe("a:b");
CHECK(bus.publish("a:b", "hi") == 3, "exact plus wildcard plus late deliver");
auto got = bus.poll(sub1, 10);
CHECK(got.size() == 1 && got[0].payload == "hi", "exact subscriber receives");
CHECK(bus.poll(sub2, 10).size() == 1, "wildcard subscriber receives");
CHECK(bus.poll(late, 10).size() == 1, "subscriber present at publish receives");
// Offline means offline: subscribe after the publish, get nothing.
CHECK(bus.publish("a:b", "before") == 3, "three live subscribers");
const auto offline = bus.subscribe("a:b");
bus.poll(sub1, 10);
bus.poll(sub2, 10);
bus.poll(late, 10);
CHECK(bus.poll(offline, 10).empty(), "offline subscriber gets nothing past");
CHECK(bus.publish("a:b:c", "deep") == 0, "wildcard matches one segment only");
// Backpressure: oldest-drop with a counter, publisher never blocks.
for (int i = 0; i < 1034; ++i) {
bus.publish("a:b", "flood-" + std::to_string(i));
}
CHECK(bus.dropped_total() > 0, "overflow counted");
CHECK(bus.pending(sub1) == 1024, "subscriber queue capped at 1024");
auto tail = bus.poll(sub1, 2000);
CHECK(tail.front().payload == "flood-10", "oldest dropped first");
CHECK(bus.unsubscribe(sub1) && !bus.unsubscribe(sub1), "unsubscribe once");
std::cout << "PASS test_ephemeral_topics" << std::endl;
}
} // namespace
int main() {
test_durable_stream_reconnect_no_loss();
test_redelivery_until_acked();
test_retention_and_caps();
test_ephemeral_topics();
if (failures == 0) {
std::cout << "ALL BUS TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}
+253
View File
@@ -0,0 +1,253 @@
// V7-6: durable queue survives a hard kill; unacked work replays.
#include <filesystem>
#include <fstream>
#include <iostream>
#include <mutex>
#include <set>
#include "supervisor/common/clock.hpp"
#include "supervisor/common/kv_store.hpp"
#include "supervisor/common/observer.hpp"
#include "supervisor/tasks/scheduler.hpp"
#include "supervisor/tasks/task_queue.hpp"
#include "supervisor/tasks/task_registry.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
const std::filesystem::path kTmp = "build/supervisor/tmp/test-durability";
struct IdempotentCounter : molo::tasks::TaskHandler {
std::mutex* mutex;
std::set<std::string>* completed;
std::set<std::string>* seen_twice;
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
std::lock_guard<std::mutex> lock(*mutex);
if (!completed->insert(ctx.envelope.id).second) {
seen_twice->insert(ctx.envelope.id); // redelivery tolerated
}
return molo::tasks::TaskOutcome::Success;
}
};
void test_hard_kill_replays_unacked() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
std::set<std::string> ids;
{
molo::tasks::QueueOptions options;
options.durable_dir = (kTmp / "q").string();
options.lease_ms = 60000;
molo::tasks::TaskQueue queue(options, clock, obs);
for (int i = 0; i < 50; ++i) {
molo::tasks::TaskEnvelope env;
env.name = "work";
ids.insert(queue.enqueue(std::move(env)));
}
CHECK(queue.stats().queued == 50, "50 queued before the kill");
// Lease 10 without ack: in-flight at crash time.
for (int i = 0; i < 10; ++i) {
CHECK(queue.lease(0).has_value(), "leases before the kill");
}
// Hard kill: the queue object just dies here. No shutdown, no acks.
}
{
molo::tasks::QueueOptions options;
options.durable_dir = (kTmp / "q").string();
molo::tasks::TaskQueue queue(options, clock, obs);
CHECK(queue.stats().queued == 50, "all 50 replay as queued after reboot");
// Every replayed id matches a pre-crash id.
auto live = queue.list_queued(100);
CHECK(live.size() == 50, "50 live envelopes");
for (const auto& env : live) {
CHECK(ids.count(env.id) == 1, "replayed id is pre-crash");
}
molo::tasks::TaskRegistry registry;
std::mutex mutex;
std::set<std::string> completed, seen_twice;
registry.register_handler("work", [&] {
auto h = std::make_unique<IdempotentCounter>();
h->mutex = &mutex;
h->completed = &completed;
h->seen_twice = &seen_twice;
return h;
});
int ran = 0;
while (queue.run_one(registry, 0)) {
++ran;
}
CHECK(ran == 50 && completed.size() == 50, "all replayed work completes");
CHECK(completed == ids, "completed set equals the pre-crash set");
CHECK(queue.stats().queued == 0, "queue drains fully");
}
std::cout << "PASS test_hard_kill_replays_unacked" << std::endl;
}
void test_retry_state_survives_reboot() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
std::string id;
{
molo::tasks::QueueOptions options;
options.durable_dir = (kTmp / "q").string();
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskEnvelope env;
env.name = "flaky";
id = queue.enqueue(std::move(env));
auto leased = queue.lease(0);
CHECK(leased.has_value(), "leases the task");
CHECK(queue.fail(leased->lease_id, true, "boom"), "fails retryable");
auto stored = queue.get(id);
CHECK(stored.has_value() && stored->attempts == 1, "attempts persisted");
CHECK(stored.has_value() && stored->run_at_ms.has_value(), "backoff run_at persisted");
}
{
molo::tasks::QueueOptions options;
options.durable_dir = (kTmp / "q").string();
molo::tasks::TaskQueue queue(options, clock, obs);
auto stored = queue.get(id);
CHECK(stored.has_value(), "retried task survives reboot");
if (stored.has_value()) {
CHECK(stored->attempts == 1, "attempt count survives");
CHECK(stored->run_at_ms.has_value(), "backoff deadline survives");
}
}
std::cout << "PASS test_retry_state_survives_reboot" << std::endl;
}
void test_compact_and_torn_tail() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
{
molo::tasks::QueueOptions options;
options.durable_dir = (kTmp / "q").string();
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
registry.register_handler("w", [] {
struct H : molo::tasks::TaskHandler {
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
return molo::tasks::TaskOutcome::Success;
}
};
return std::make_unique<H>();
});
for (int i = 0; i < 20; ++i) {
molo::tasks::TaskEnvelope env;
env.name = "w";
queue.enqueue(std::move(env));
}
while (queue.run_one(registry, 0)) {
}
const auto log = kTmp / "q" / "task_queue.log";
const std::uint64_t before = fs::file_size(log);
CHECK(before > 0, "op log written");
queue.compact();
CHECK(fs::file_size(log) < before, "compact shrinks the log");
}
{
// Torn tail: crash garbage at the end of the log is truncated on open.
molo::tasks::QueueOptions options;
options.durable_dir = (kTmp / "q").string();
{
molo::tasks::TaskQueue queue(options, clock, obs);
for (int i = 0; i < 5; ++i) {
molo::tasks::TaskEnvelope env;
env.name = "w";
queue.enqueue(std::move(env));
}
}
const auto log = kTmp / "q" / "task_queue.log";
{
std::ofstream out(log, std::ios::app | std::ios::binary);
out << "junk";
}
molo::tasks::TaskQueue queue(options, clock, obs);
CHECK(queue.stats().queued == 5, "torn tail truncated, 5 tasks replay");
}
std::cout << "PASS test_compact_and_torn_tail" << std::endl;
}
} // namespace
void test_schedule_entries_survive_reboot() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::common::InMemoryKvStore kv(clock);
molo::tasks::QueueOptions options;
molo::tasks::TaskQueue queue(options, clock, obs);
{
molo::tasks::SchedulerOptions sched_options;
sched_options.instance_id = "sched-durable";
sched_options.durable_dir = (kTmp / "sched").string();
molo::tasks::Scheduler sched(queue, kv, clock, obs, sched_options);
molo::tasks::ScheduleEntry interval;
interval.key = "every-minute";
interval.task_name = "work";
interval.kind = molo::tasks::ScheduleKind::Interval;
interval.interval_secs = 60;
sched.put(interval);
molo::tasks::ScheduleEntry cron;
cron.key = "nightly";
cron.task_name = "work";
cron.kind = molo::tasks::ScheduleKind::Cron;
cron.cron = "0 2 * * *";
sched.put(cron);
molo::tasks::ScheduleEntry temp;
temp.key = "temp";
temp.task_name = "work";
temp.kind = molo::tasks::ScheduleKind::Interval;
sched.put(temp);
CHECK(sched.remove("temp"), "temp entry removed before reboot");
}
{
molo::tasks::SchedulerOptions sched_options;
sched_options.instance_id = "sched-durable";
sched_options.durable_dir = (kTmp / "sched").string();
molo::tasks::Scheduler sched(queue, kv, clock, obs, sched_options);
auto entries = sched.entries();
CHECK(entries.size() == 2, "two entries replay after reboot");
bool has_interval = false, has_cron = false;
for (const auto& entry : entries) {
if (entry.key == "every-minute" && entry.kind == molo::tasks::ScheduleKind::Interval) {
has_interval = true;
}
if (entry.key == "nightly" && entry.cron == "0 2 * * *") {
has_cron = true;
}
CHECK(entry.key != "temp", "removed entry stays removed");
}
CHECK(has_interval && has_cron, "entry kinds and crons survive");
}
std::cout << "PASS test_schedule_entries_survive_reboot" << std::endl;
}
int main() {
test_hard_kill_replays_unacked();
test_retry_state_survives_reboot();
test_compact_and_torn_tail();
test_schedule_entries_survive_reboot();
if (failures == 0) {
std::cout << "ALL DURABILITY TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}
@@ -0,0 +1,111 @@
// V7-8: two nodes behind one shared bus; broadcast plus catch-up.
#include <filesystem>
#include <iostream>
#include "supervisor/bus/stream_bus.hpp"
#include "supervisor/bus/topic_bus.hpp"
#include "supervisor/common/clock.hpp"
#include "supervisor/common/kv_store.hpp"
#include "supervisor/common/observer.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
const std::filesystem::path kTmp = "build/supervisor/tmp/test-multi";
// One stateless app node (S7-9): local sockets hang off the ephemeral bus,
// catch-up replays off the durable bus, sessions live in shared KV.
struct Node {
std::string id;
molo::bus::TopicBus::SubscriberId room_sub = 0;
int received_live = 0;
std::vector<std::string> caught_up;
};
void test_broadcast_reaches_both_nodes() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
// Shared backends both nodes attach to.
molo::bus::TopicBus live(clock);
molo::bus::StreamBus bus(kTmp / "bus", clock, obs);
bus.create_stream("roomlog");
molo::common::InMemoryKvStore kv(clock);
Node a;
a.id = "node-a";
Node b;
b.id = "node-b";
a.room_sub = live.subscribe("room:lobby");
b.room_sub = live.subscribe("room:lobby");
bus.create_consumer("roomlog", "node-a");
bus.create_consumer("roomlog", "node-b");
// Broadcast emit: publish once to the bus; every node re-emits to its
// local sockets (here: each node polls its own subscription).
auto broadcast = [&](const std::string& payload) {
const std::size_t delivered = live.publish("room:lobby", payload);
bus.publish("roomlog", payload); // durable copy for reconnect catch-up
return delivered;
};
CHECK(broadcast("hello") == 2, "one publish, two nodes");
for (Node* node : {&a, &b}) {
auto got = live.poll(node->room_sub, 10);
CHECK(got.size() == 1 && got[0].payload == "hello", "live delivery on " + node->id);
node->received_live += static_cast<int>(got.size());
// Nodes also drain the durable log to keep their cursors current.
for (const auto& msg : bus.poll("roomlog", node->id, 10)) {
bus.ack("roomlog", node->id, msg.seq);
}
}
// Shared session store: written via node A, visible via node B.
kv.set("session:u1", "{\"user\":\"u1\"}", 3600);
std::string session;
CHECK(kv.get("session:u1", session) && session == "{\"user\":\"u1\"}", "sessions shared");
// Kill node B: its live subscription dies with it.
CHECK(live.unsubscribe(b.room_sub), "node B dies");
CHECK(broadcast("msg-2") == 1, "dead node gets nothing live");
auto got_a = live.poll(a.room_sub, 10);
CHECK(got_a.size() == 1 && got_a[0].payload == "msg-2", "node A receives msg-2 live");
for (const auto& msg : bus.poll("roomlog", "node-a", 10)) {
bus.ack("roomlog", "node-a", msg.seq);
}
// Node B restarts: live messages sent while dead are gone (ephemeral
// tier), but the durable log replays them in order.
b.room_sub = live.subscribe("room:lobby");
CHECK(live.poll(b.room_sub, 10).empty(), "no ephemeral replay while dead");
for (const auto& msg : bus.poll("roomlog", "node-b", 10)) {
b.caught_up.push_back(msg.data);
bus.ack("roomlog", "node-b", msg.seq);
}
CHECK(b.caught_up.size() == 1 && b.caught_up[0] == "msg-2", "durable catch-up replays msg-2");
CHECK(bus.lag("roomlog", "node-b") == 0, "node B caught up");
std::cout << "PASS test_broadcast_reaches_both_nodes" << std::endl;
}
} // namespace
int main() {
test_broadcast_reaches_both_nodes();
if (failures == 0) {
std::cout << "ALL MULTI-INSTANCE TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}
+304
View File
@@ -0,0 +1,304 @@
// V7-7: two-instance replication, optimistic plus LWW plus hook conflicts.
#include <filesystem>
#include <iostream>
#include <thread>
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
#include "supervisor/replication/conflict_resolver.hpp"
#include "supervisor/replication/hybrid_clock.hpp"
#include "supervisor/replication/replicator.hpp"
#include "supervisor/replication/sync_session.hpp"
#include "supervisor/replication/version_vector.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
const std::filesystem::path kTmp = "build/supervisor/tmp/test-replication";
// In-process SyncPeer over another Replicator (production binds this to
// the bus transport; the sync logic is identical).
class ReplicatorSyncPeer : public molo::replication::SyncPeer {
public:
explicit ReplicatorSyncPeer(molo::replication::Replicator& remote) : remote_(remote) {}
std::vector<molo::replication::OplogEntry> fetch_since(
const molo::replication::VersionVector& have, std::size_t max_entries,
std::size_t max_bytes) override {
return remote_.entries_since(have.get(remote_.instance()) + 1, max_entries, max_bytes);
}
void submit(const std::vector<molo::replication::OplogEntry>& entries) override {
for (const auto& entry : entries) {
try {
remote_.apply_remote(entry);
} catch (const molo::replication::ConflictError&) {
// Per-entry independence: one loser's ConflictError never blocks
// the rest of the batch. The resolver already logged and counted it.
}
}
}
void peer_versions(molo::replication::VersionVector& out) const override {
remote_.versions(out);
}
std::uint64_t last_seq() const override { return remote_.last_seq(); }
std::string instance() const override { return remote_.instance(); }
private:
molo::replication::Replicator& remote_;
};
void sync_both(molo::replication::Replicator& a, molo::replication::Replicator& b,
std::shared_ptr<molo::common::Observer> obs) {
ReplicatorSyncPeer peer_b(b);
molo::replication::SyncSession(a, peer_b, obs).round_trip();
ReplicatorSyncPeer peer_a(a);
molo::replication::SyncSession(b, peer_a, obs).round_trip();
}
void test_optimistic_conflict_and_recovery() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = std::make_shared<molo::common::RecordingObserver>();
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs);
a.declare_model("orders", molo::replication::ConflictPolicy::Optimistic);
b.declare_model("orders", molo::replication::ConflictPolicy::Optimistic);
a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 1, "{\"v\":1}");
sync_both(a, b, obs);
CHECK(b.current("orders", "k1").has_value(), "base write replicates");
CHECK(obs->counter_value("molo_replication_conflicts_total") == 0, "no conflict on fast-forward");
// Concurrent write-write at the same version with different data.
clock.advance(10);
a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"a\"}");
b.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"b\"}");
// Direct resolver check: exactly one winner plus ConflictError semantics.
auto la = a.current("orders", "k1");
auto lb = b.current("orders", "k1");
auto outcome = a.resolver().resolve(*la, *lb);
CHECK(outcome.conflict && outcome.conflict_error, "write-write conflict flagged");
CHECK(outcome.winner_version == 2, "winner version carried");
bool threw = false;
try {
a.apply_remote(*lb);
} catch (const molo::replication::ConflictError& e) {
threw = true;
CHECK(e.winner_version() == 2, "ConflictError carries winner version");
CHECK(e.model() == "orders" && e.key() == "k1", "ConflictError names model and key");
}
CHECK(threw, "loser gets ConflictError");
// Sync keeps each incumbent; the conflict is counted, nothing is lost.
sync_both(a, b, obs);
CHECK(obs->counter_value("molo_replication_conflicts_total") > 0, "sync counts conflicts");
CHECK(a.resolver().conflict_log_size() > 0, "conflict logged");
// Documented recovery: the loser re-reads and retries with a higher version.
a.local_write("orders", "k1", molo::replication::OplogOp::Upsert, 3, "{\"by\":\"a3\"}");
sync_both(a, b, obs);
auto ca = a.current("orders", "k1");
auto cb = b.current("orders", "k1");
CHECK(ca.has_value() && cb.has_value(), "both converge");
if (ca.has_value() && cb.has_value()) {
CHECK(ca->version == 3 && cb->version == 3, "higher version wins everywhere");
CHECK(ca->data == cb->data, "converged data agrees");
}
std::cout << "PASS test_optimistic_conflict_and_recovery" << std::endl;
}
void test_lww_cache_conflict() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = molo::common::null_observer();
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs);
a.declare_model("cache", molo::replication::ConflictPolicy::LastWriterWins);
b.declare_model("cache", molo::replication::ConflictPolicy::LastWriterWins);
// Same frozen millisecond: hybrid counters tie at first write, so the
// instance id tie-break decides deterministically (node-b wins).
a.local_write("cache", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
b.local_write("cache", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"b\"}");
sync_both(a, b, obs);
auto ca = a.current("cache", "k");
auto cb = b.current("cache", "k");
CHECK(ca.has_value() && cb.has_value(), "both resolve LWW");
if (ca.has_value() && cb.has_value()) {
CHECK(ca->data == "{\"by\":\"b\"}" && cb->data == "{\"by\":\"b\"}",
"higher hybrid timestamp wins on both");
CHECK(ca->instance == "node-b", "winner instance recorded");
}
CHECK(a.resolver().conflict_log_size() == 1, "loser kept in conflict log");
auto log = a.resolver().conflict_log(10);
CHECK(!log.empty() && log[0].loser.data == "{\"by\":\"a\"}", "log holds the loser");
// A genuinely later write wins regardless of instance.
clock.advance(5);
a.local_write("cache", "k", molo::replication::OplogOp::Upsert, 2, "{\"by\":\"a2\"}");
sync_both(a, b, obs);
CHECK(b.current("cache", "k")->data == "{\"by\":\"a2\"}", "later wall clock wins");
std::cout << "PASS test_lww_cache_conflict" << std::endl;
}
void test_hook_policy_and_fallback() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = molo::common::null_observer();
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
// Hook that always prefers the remote side.
a.declare_model("special", molo::replication::ConflictPolicy::Hook,
[](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) {
return 1;
});
a.local_write("special", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
molo::replication::OplogEntry remote;
remote.instance = "node-b";
remote.ts = molo::replication::HybridTs{1760000000000LL, 0, "node-b"};
remote.model = "special";
remote.key = "k";
remote.op = molo::replication::OplogOp::Upsert;
remote.version = 1;
remote.data = "{\"by\":\"b\"}";
auto outcome = a.apply_remote(remote);
CHECK(outcome.remote_won && outcome.winner.data == "{\"by\":\"b\"}", "hook decision honored");
CHECK(a.resolver().conflict_log(10)[0].policy == "hook", "hook conflict logged");
// Throwing hook falls back to LWW immediately.
a.declare_model("broken", molo::replication::ConflictPolicy::Hook,
[](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) {
throw std::runtime_error("hook bug");
return 0;
});
a.local_write("broken", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
molo::replication::OplogEntry remote2 = remote;
remote2.model = "broken";
remote2.ts = molo::replication::HybridTs{1760000000000LL, 99, "node-b"};
auto outcome2 = a.apply_remote(remote2);
CHECK(outcome2.detail.find("hook fallback") != std::string::npos, "throw falls back to LWW");
CHECK(outcome2.remote_won, "LWW fallback picks the higher ts");
// Slow hook past the 5s budget falls back to LWW (S7-8).
a.declare_model("slow", molo::replication::ConflictPolicy::Hook,
[](const molo::replication::OplogEntry&, const molo::replication::OplogEntry&) {
std::this_thread::sleep_for(std::chrono::milliseconds(5200));
return 0;
});
a.local_write("slow", "k", molo::replication::OplogOp::Upsert, 1, "{\"by\":\"a\"}");
molo::replication::OplogEntry remote3 = remote;
remote3.model = "slow";
auto outcome3 = a.apply_remote(remote3);
CHECK(outcome3.detail.find("hook fallback") != std::string::npos, "5s timeout falls back");
std::cout << "PASS test_hook_policy_and_fallback" << std::endl;
}
void test_sync_batches_and_lag() {
namespace fs = std::filesystem;
fs::remove_all(kTmp);
molo::common::FrozenClock clock(1760000000000LL, 1000);
auto obs = std::make_shared<molo::common::RecordingObserver>();
molo::replication::Replicator a(kTmp / "a", "node-a", clock, obs);
molo::replication::Replicator b(kTmp / "b", "node-b", clock, obs);
for (int i = 0; i < 600; ++i) {
a.local_write("bulk", "k" + std::to_string(i), molo::replication::OplogOp::Upsert, 1, "{}");
}
ReplicatorSyncPeer peer_a(a);
molo::replication::SyncSession session(b, peer_a, obs);
auto s1 = session.round_trip();
CHECK(s1.pulled == 500, "first round honors the 500-entry batch cap");
CHECK(s1.lag == 100, "lag counts the remaining 100");
auto s2 = session.round_trip();
CHECK(s2.pulled == 100, "second round completes the catch-up");
CHECK(s2.lag == 0, "lag zero when caught up");
CHECK(b.current("bulk", "k599").has_value(), "tail entry arrived");
CHECK(obs->gauges().count("molo_replication_sync_lag") == 1, "lag gauge exported");
// Deletes replicate as tombstones.
a.local_write("bulk", "k0", molo::replication::OplogOp::Delete, 2, "{}");
session.round_trip();
auto tomb = b.current("bulk", "k0");
CHECK(tomb.has_value() && tomb->op == molo::replication::OplogOp::Delete, "delete replicates");
std::cout << "PASS test_sync_batches_and_lag" << std::endl;
}
void test_vectors_clocks_and_codecs() {
molo::replication::VersionVector v1, v2;
v1.update("a", 5);
v1.update("b", 3);
v2.update("a", 5);
v2.update("b", 4);
CHECK(v1.compare(v2) == molo::replication::VectorOrder::Before, "vector before");
CHECK(v2.compare(v1) == molo::replication::VectorOrder::After, "vector after");
v1.update("c", 1);
CHECK(v1.compare(v2) == molo::replication::VectorOrder::Concurrent, "vector concurrent");
v1.merge(v2);
CHECK(v1.get("b") == 4 && v1.get("c") == 1, "merge takes maxima");
molo::replication::VersionVector back;
CHECK(molo::replication::VersionVector::decode(v1.encode(), back), "vector codec roundtrip");
CHECK(back.compare(v1) == molo::replication::VectorOrder::Equal, "vector codec preserves");
molo::common::FrozenClock clock(1000, 1000);
molo::replication::HybridClock hybrid("n1", clock);
auto t1 = hybrid.now();
auto t2 = hybrid.now();
CHECK(t2.compare(t1) > 0, "same-ms counter orders");
clock.advance(10);
auto t3 = hybrid.now();
CHECK(t3.wall_ms == 1010 && t3.counter == 0, "wall advance resets the counter");
molo::replication::HybridTs parsed;
CHECK(molo::replication::HybridTs::decode(t3.encode(), parsed) && parsed == t3, "ts codec");
molo::replication::OplogEntry entry;
entry.seq = 7;
entry.instance = "n1";
entry.ts = t3;
entry.model = "m";
entry.key = "k";
entry.op = molo::replication::OplogOp::Upsert;
entry.version = 2;
entry.data = "{\"x\":1}";
molo::replication::OplogEntry entry_back;
CHECK(molo::replication::OplogEntry::decode(entry.encode(), entry_back), "oplog codec");
CHECK(entry_back.seq == 7 && entry_back.version == 2 && entry_back.data == "{\"x\":1}",
"oplog fields survive");
CHECK(!molo::replication::OplogEntry::decode("nope", entry_back), "oplog rejects garbage");
molo::replication::ConflictResolver resolver(clock, molo::common::null_observer());
CHECK(resolver.policy_for("undeclared") == molo::replication::ConflictPolicy::Optimistic,
"default policy is optimistic");
std::cout << "PASS test_vectors_clocks_and_codecs" << std::endl;
}
} // namespace
int main() {
test_optimistic_conflict_and_recovery();
test_lww_cache_conflict();
test_hook_policy_and_fallback();
test_sync_batches_and_lag();
test_vectors_clocks_and_codecs();
if (failures == 0) {
std::cout << "ALL REPLICATION TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}
+258
View File
@@ -0,0 +1,258 @@
// V7-4: scheduler intervals, cron, leader failover without duplicates.
#include <atomic>
#include <iostream>
#include "supervisor/common/clock.hpp"
#include "supervisor/common/kv_store.hpp"
#include "supervisor/common/observer.hpp"
#include "supervisor/tasks/cron.hpp"
#include "supervisor/tasks/scheduler.hpp"
#include "supervisor/tasks/task_queue.hpp"
#include "supervisor/tasks/task_registry.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
constexpr std::int64_t kT0 = 1760000000000LL; // fixed frozen origin
struct Harness {
molo::common::FrozenClock clock{kT0, 1000};
molo::common::InMemoryKvStore kv{clock};
molo::tasks::TaskQueue queue{molo::tasks::QueueOptions{}, clock,
molo::common::null_observer()};
molo::tasks::TaskRegistry registry;
std::atomic<long> runs{0};
Harness() {
registry.register_handler("tick", [&] {
struct H : molo::tasks::TaskHandler {
std::atomic<long>* runs;
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
++(*runs);
return molo::tasks::TaskOutcome::Success;
}
};
auto h = std::make_unique<H>();
h->runs = &runs;
return h;
});
}
int drain() {
int n = 0;
while (queue.run_one(registry, 0)) {
++n;
}
return n;
}
// Advances the frozen clock one second per scheduler tick, like the
// production 1s tick loop. Time jumps longer than the leader TTL would
// (correctly) drop leadership, so tests step through time instead.
void step(molo::tasks::Scheduler& sched, int seconds) {
for (int i = 0; i < seconds; ++i) {
clock.advance(1000);
sched.tick_once();
}
}
};
void test_interval_fires_per_tick() {
Harness h;
molo::tasks::SchedulerOptions options;
options.instance_id = "sched-a";
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options);
molo::tasks::ScheduleEntry entry;
entry.key = "every-minute";
entry.task_name = "tick";
entry.kind = molo::tasks::ScheduleKind::Interval;
entry.interval_secs = 60;
sched.put(entry);
sched.tick_once();
CHECK(sched.is_leader(), "single scheduler wins the lease");
CHECK(h.queue.stats().queued == 0, "no fire before the interval");
for (int i = 0; i < 5; ++i) {
h.step(sched, 60);
CHECK(h.drain() == 1, "one run per 60s tick");
}
CHECK(h.runs.load() == 5, "frozen clock advances 5 ticks, 5 runs");
sched.tick_once();
CHECK(h.drain() == 0, "no fire without time passing");
std::cout << "PASS test_interval_fires_per_tick" << std::endl;
}
void test_leader_failover_no_duplicates() {
Harness h;
molo::tasks::SchedulerOptions a_opts;
a_opts.instance_id = "sched-a";
a_opts.ttl_secs = 2;
a_opts.renew_secs = 1;
molo::tasks::Scheduler a(h.queue, h.kv, h.clock, molo::common::null_observer(), a_opts);
molo::tasks::SchedulerOptions b_opts;
b_opts.instance_id = "sched-b";
b_opts.ttl_secs = 2;
b_opts.renew_secs = 1;
molo::tasks::Scheduler b(h.queue, h.kv, h.clock, molo::common::null_observer(), b_opts);
molo::tasks::ScheduleEntry entry;
entry.key = "job";
entry.task_name = "tick";
entry.kind = molo::tasks::ScheduleKind::Interval;
entry.interval_secs = 60;
a.put(entry);
b.put(entry);
a.tick_once();
b.tick_once();
CHECK(a.is_leader() && !b.is_leader(), "first scheduler wins the lease");
for (int i = 0; i < 60; ++i) {
h.clock.advance(1000);
a.tick_once(); // leader fires at the window edge
b.tick_once(); // follower stays passive
}
CHECK(h.drain() == 1, "exactly one fire with two schedulers");
// Kill the leader: it stops ticking, its lease expires, the follower
// takes over within the TTL (V7-4 bound is 15s; TTL here is 2s).
for (int i = 0; i < 3; ++i) {
h.clock.advance(1000);
b.tick_once();
}
CHECK(b.is_leader(), "follower takes over after lease expiry");
const long runs_before = h.runs.load();
b.tick_once(); // takeover tick must not refire the shared last-fire window
CHECK(h.drain() == 0, "no duplicate run after failover");
CHECK(h.runs.load() == runs_before, "run count unchanged by failover");
h.step(b, 60);
CHECK(h.drain() == 1, "new leader fires the next window");
std::cout << "PASS test_leader_failover_no_duplicates" << std::endl;
}
void test_cron_entries() {
// Unit checks on the expression engine first.
molo::tasks::CronExpr expr;
std::string error;
CHECK(molo::tasks::CronExpr::parse("* * * * *", expr, error), "every-minute parses");
CHECK(molo::tasks::CronExpr::parse("*/15 9-17 * * 1-5", expr, error), "business hours parse");
CHECK(!molo::tasks::CronExpr::parse("* * * *", expr, error), "4 fields rejected");
CHECK(!molo::tasks::CronExpr::parse("61 * * * *", expr, error), "minute 61 rejected");
CHECK(molo::tasks::CronExpr::parse("0 0 * * 7", expr, error), "dow 7 means sunday");
// 2026-10-23 is a Friday. 12:00 UTC that day:
const std::int64_t friday_noon = 1792756800000LL; // 2026-10-23 12:00 UTC, a Friday
molo::tasks::CronExpr noon;
CHECK(molo::tasks::CronExpr::parse("0 12 * * *", noon, error), "noon parses");
CHECK(noon.matches(friday_noon + 30000), "noon matches inside the minute");
CHECK(!noon.matches(friday_noon + 60000), "noon misses the next minute");
CHECK(noon.next_after(friday_noon) == friday_noon + 86400000LL, "next noon is tomorrow");
// End to end through the scheduler with a frozen clock.
Harness h;
h.clock.set(friday_noon - 60000, 1000); // 11:59
molo::tasks::SchedulerOptions options;
options.instance_id = "sched-cron";
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options);
molo::tasks::ScheduleEntry entry;
entry.key = "noon-job";
entry.task_name = "tick";
entry.kind = molo::tasks::ScheduleKind::Cron;
entry.cron = "0 12 * * *";
sched.put(entry);
sched.tick_once();
CHECK(h.drain() == 0, "no fire at 11:59");
h.step(sched, 60); // step into 12:00
CHECK(h.drain() == 1, "fires at 12:00");
sched.tick_once();
CHECK(h.drain() == 0, "fires once per minute");
std::cout << "PASS test_cron_entries" << std::endl;
}
void test_once_and_missed_policy() {
Harness h;
auto obs = std::make_shared<molo::common::RecordingObserver>();
molo::tasks::SchedulerOptions options;
options.instance_id = "sched-once";
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, obs, options);
molo::tasks::ScheduleEntry skip;
skip.key = "once-skip";
skip.task_name = "tick";
skip.kind = molo::tasks::ScheduleKind::Once;
skip.at_ms = kT0;
skip.missed = molo::tasks::MissedPolicy::Skip;
molo::tasks::ScheduleEntry run;
run.key = "once-run";
run.task_name = "tick";
run.kind = molo::tasks::ScheduleKind::Once;
run.at_ms = kT0;
run.missed = molo::tasks::MissedPolicy::RunOnce;
sched.put(skip);
sched.put(run);
h.clock.advance(2 * 3600 * 1000); // miss both by 2h
sched.tick_once();
CHECK(h.drain() == 1, "run-once fires, skip drops");
CHECK(obs->event_count("tasks.schedule.missed") == 1, "miss recorded");
sched.tick_once();
CHECK(h.drain() == 0, "once entries never refire");
std::cout << "PASS test_once_and_missed_policy" << std::endl;
}
void test_interval_floor_and_bad_cron() {
Harness h;
molo::tasks::SchedulerOptions options;
options.instance_id = "sched-floor";
molo::tasks::Scheduler sched(h.queue, h.kv, h.clock, molo::common::null_observer(), options);
molo::tasks::ScheduleEntry fast;
fast.key = "fast";
fast.task_name = "tick";
fast.kind = molo::tasks::ScheduleKind::Interval;
fast.interval_secs = 5;
sched.put(fast);
CHECK(sched.entries()[0].interval_secs == 60, "interval floored at 60s");
molo::tasks::ScheduleEntry bad;
bad.key = "bad";
bad.task_name = "tick";
bad.kind = molo::tasks::ScheduleKind::Cron;
bad.cron = "not a cron";
bool threw = false;
try {
sched.put(bad);
} catch (const molo::tasks::TaskError&) {
threw = true;
}
CHECK(threw, "bad cron rejected at put");
CHECK(sched.remove("fast"), "remove works");
CHECK(!sched.remove("fast"), "double remove fails");
std::cout << "PASS test_interval_floor_and_bad_cron" << std::endl;
}
} // namespace
int main() {
test_interval_fires_per_tick();
test_leader_failover_no_duplicates();
test_cron_entries();
test_once_and_missed_policy();
test_interval_floor_and_bad_cron();
if (failures == 0) {
std::cout << "ALL SCHEDULER TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}
+262
View File
@@ -0,0 +1,262 @@
// V7-1: supervisor restarts, intensity caps, strategies, health, status.
#include <atomic>
#include <chrono>
#include <functional>
#include <iostream>
#include <thread>
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
#include "supervisor/supervisor/supervisor.hpp"
#include "supervisor/supervisor/supervisor_error.hpp"
#include "supervisor/supervisor/worker_group.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
bool wait_for(const std::function<bool()>& pred, long timeout_ms) {
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms);
while (std::chrono::steady_clock::now() < deadline) {
if (pred()) {
return true;
}
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
return pred();
}
int run_until_stopped(const std::atomic<bool>& stop) {
while (!stop.load()) {
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
return 0;
}
void test_three_workers_restart_and_status() {
molo::common::SystemClock clock;
auto obs = std::make_shared<molo::common::RecordingObserver>();
molo::supervisor::Supervisor sup(clock, obs);
molo::supervisor::GroupSpec spec;
spec.name = "web";
spec.size = 3;
spec.monitor_ms = 10;
spec.backoff_min_ms = 10;
spec.backoff_max_ms = 50;
const std::size_t idx = sup.add_group(spec, run_until_stopped);
CHECK(idx == 0, "first group index is 0");
sup.start_all();
CHECK(wait_for([&] {
const std::string status = sup.status_json();
return status.find("\"state\":\"running\"") != std::string::npos;
}, 2000), "workers reach running");
const std::string status = sup.status_json();
CHECK(status.find("\"name\":\"web\"") != std::string::npos, "status names the group");
CHECK(status.find("\"phase\":\"running\"") != std::string::npos, "status shows running phase");
CHECK(status.size() <= 1048576, "status under the 1 MiB cap");
CHECK(obs->counter_value("molo_supervisor_worker_starts_total") >= 3, "3 starts counted");
sup.shutdown(2000);
CHECK(sup.phase() == molo::supervisor::ShutdownPhase::Stopped, "shutdown reaches stopped");
std::cout << "PASS test_three_workers_restart_and_status" << std::endl;
}
void test_crash_restart_then_fatal() {
molo::common::SystemClock clock;
auto obs = std::make_shared<molo::common::RecordingObserver>();
molo::supervisor::Supervisor sup(clock, obs);
std::atomic<int> runs{0};
auto crasher = [&](const std::atomic<bool>&) {
++runs;
return 1; // always crashes
};
molo::supervisor::GroupSpec spec;
spec.name = "crasher";
spec.size = 1;
spec.monitor_ms = 5;
spec.backoff_min_ms = 10;
spec.backoff_max_ms = 20;
spec.restart.policy = molo::supervisor::RestartPolicy::OnFailure;
spec.restart.max_restarts = 2;
spec.restart.window_secs = 60;
sup.add_group(spec, crasher);
const auto t0 = std::chrono::steady_clock::now();
sup.start_all();
// Restart latency: the first restart must happen within 2s (V7-1).
CHECK(wait_for([&] { return runs.load() >= 2; }, 2000), "crasher restarted under 2s");
const auto restarted_ms =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - t0)
.count();
CHECK(restarted_ms < 2000, "restart latency under 2s");
// Intensity cap: initial + 2 restarts, then fatal.
CHECK(wait_for([&] { return sup.status_json().find("\"state\":\"fatal\"") != std::string::npos; },
5000),
"crasher goes fatal after the cap");
CHECK(runs.load() == 3, "exactly initial plus max_restarts runs");
CHECK(obs->counter_value("molo_supervisor_worker_fatal_total") == 1, "fatal counted");
sup.shutdown(1000);
std::cout << "PASS test_crash_restart_then_fatal" << std::endl;
}
void test_explicit_restart_clears_fatal() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::supervisor::WorkerGroup group(
[] {
molo::supervisor::GroupSpec spec;
spec.name = "flaky";
spec.monitor_ms = 5;
spec.backoff_min_ms = 5;
spec.backoff_max_ms = 10;
spec.restart.max_restarts = 1;
return spec;
}(),
[](const std::atomic<bool>&) { return 1; }, clock, obs);
group.start();
CHECK(wait_for([&] { return group.fatal(); }, 5000), "group goes fatal");
group.restart_child(0); // explicit restart clears Fatal
CHECK(wait_for([&] { return group.fatal(); }, 5000), "fatal again after crash loop");
group.stop();
std::cout << "PASS test_explicit_restart_clears_fatal" << std::endl;
}
void test_strategies_recover() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
const molo::supervisor::GroupStrategy strategies[] = {
molo::supervisor::GroupStrategy::OneForOne,
molo::supervisor::GroupStrategy::OneForAll,
molo::supervisor::GroupStrategy::RestForOne,
};
for (auto strategy : strategies) {
std::atomic<int> failures_left{1};
auto sometimes_fails = [&](const std::atomic<bool>& stop) {
if (failures_left.load() > 0 && failures_left.fetch_sub(1) == 1) {
return 1; // exactly one crash across the whole group
}
return run_until_stopped(stop);
};
molo::supervisor::GroupSpec spec;
spec.name = "strat";
spec.strategy = strategy;
spec.size = 2;
spec.monitor_ms = 5;
spec.backoff_min_ms = 5;
spec.backoff_max_ms = 10;
molo::supervisor::WorkerGroup group(spec, sometimes_fails, clock, obs);
group.start();
CHECK(wait_for([&] {
auto states = group.states();
return states.size() == 2 && states[0] == molo::supervisor::WorkerState::Running &&
states[1] == molo::supervisor::WorkerState::Running;
}, 5000), "strategy recovers to all running: " + molo::supervisor::to_string(strategy));
CHECK(!group.fatal(), "no fatal after single failure");
group.stop();
}
std::cout << "PASS test_strategies_recover" << std::endl;
}
void test_health_probe_triggers_restart() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::supervisor::GroupSpec spec;
spec.name = "sick";
spec.monitor_ms = 5;
spec.backoff_min_ms = 5;
spec.backoff_max_ms = 10;
spec.restart.max_restarts = 1;
spec.health.interval_ms = 20;
spec.health.max_failures = 2;
spec.probe = [] { return false; }; // always sick
std::atomic<int> runs{0};
auto worker = [&](const std::atomic<bool>& stop) {
++runs;
return run_until_stopped(stop);
};
molo::supervisor::WorkerGroup group(spec, worker, clock, obs);
group.start();
CHECK(wait_for([&] { return group.fatal(); }, 8000), "sick worker restarts then fatal");
CHECK(runs.load() >= 2, "unhealthy worker was restarted");
group.stop();
// Healthy probe: worker stays up.
molo::supervisor::GroupSpec good = spec;
good.name = "healthy";
good.probe = [] { return true; };
molo::supervisor::WorkerGroup group2(good, run_until_stopped, clock, obs);
group2.start();
std::this_thread::sleep_for(std::chrono::milliseconds(150));
auto states = group2.states();
CHECK(states[0] == molo::supervisor::WorkerState::Running, "healthy worker stays running");
CHECK(!group2.fatal(), "healthy group not fatal");
group2.stop();
std::cout << "PASS test_health_probe_triggers_restart" << std::endl;
}
void test_scale_and_restart_group() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::supervisor::Supervisor sup(clock, obs);
molo::supervisor::GroupSpec spec;
spec.name = "elastic";
spec.monitor_ms = 5;
spec.backoff_min_ms = 5;
const std::size_t idx = sup.add_group(spec, run_until_stopped);
sup.start_all();
sup.scale_group(idx, 4);
CHECK(wait_for([&] {
const std::string status = sup.status_json();
std::size_t count = 0, pos = 0;
while ((pos = status.find("\"state\":\"running\"", pos)) != std::string::npos) {
++count;
++pos;
}
return count == 4;
}, 3000), "scales to 4 running");
sup.scale_group(idx, 1);
CHECK(wait_for([&] { return sup.status_json().find("elastic#1") == std::string::npos; }, 3000),
"scales down to 1");
bool threw = false;
try {
sup.scale_group(idx, 65);
} catch (const molo::supervisor::SupervisorError&) {
threw = true;
}
CHECK(threw, "scale past 64 throws");
sup.restart_group(idx);
CHECK(wait_for([&] {
return sup.status_json().find("\"state\":\"running\"") != std::string::npos;
}, 3000), "group restarts");
sup.shutdown(2000);
std::cout << "PASS test_scale_and_restart_group" << std::endl;
}
} // namespace
int main() {
test_three_workers_restart_and_status();
test_crash_restart_then_fatal();
test_explicit_restart_clears_fatal();
test_strategies_recover();
test_health_probe_triggers_restart();
test_scale_and_restart_group();
if (failures == 0) {
std::cout << "ALL SUPERVISOR TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}
+348
View File
@@ -0,0 +1,348 @@
// V7-2, V7-3: task throughput, retries, backoff, dead letters, timeouts.
#include <atomic>
#include <chrono>
#include <iostream>
#include <mutex>
#include <thread>
#include <vector>
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
#include "supervisor/tasks/retry_policy.hpp"
#include "supervisor/tasks/task_envelope.hpp"
#include "supervisor/tasks/task_queue.hpp"
#include "supervisor/tasks/task_registry.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
class CountingHandler : public molo::tasks::TaskHandler {
public:
explicit CountingHandler(std::atomic<long>& count) : count_(count) {}
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
++count_;
return molo::tasks::TaskOutcome::Success;
}
private:
std::atomic<long>& count_;
};
void test_throughput_off_loop() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
std::atomic<long> done{0};
registry.register_handler("count", [&] { return std::make_unique<CountingHandler>(done); });
// Baseline scheduling latency of this thread.
auto probe = [] {
const auto t0 = std::chrono::steady_clock::now();
std::this_thread::sleep_for(std::chrono::milliseconds(5));
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
t0)
.count();
};
long baseline = 0;
for (int i = 0; i < 10; ++i) {
baseline += probe();
}
baseline /= 10;
std::atomic<bool> stop{false};
std::vector<std::thread> workers;
for (int i = 0; i < 4; ++i) {
workers.emplace_back([&] { queue.run_worker([&] { return stop.load(); }, registry); });
}
const int total = 500;
const auto t0 = std::chrono::steady_clock::now();
for (int i = 0; i < total; ++i) {
molo::tasks::TaskEnvelope env;
env.name = "count";
queue.enqueue(std::move(env));
}
const auto enqueue_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - t0)
.count();
long loaded = 0;
for (int i = 0; i < 10; ++i) {
loaded += probe();
}
loaded /= 10;
while (done.load() < total) {
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
stop = true;
for (auto& w : workers) {
w.join();
}
CHECK(done.load() == total, "all 500 tasks succeed");
CHECK(queue.stats().succeeded_total == total, "stats count all successes");
CHECK(enqueue_ms < 2000, "enqueue never blocks on execution");
CHECK(loaded < std::max<long>(50000, baseline * 10), "producer thread stays responsive");
std::cout << "PASS test_throughput_off_loop" << std::endl;
}
void test_retry_backoff_schedule() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
options.lease_ms = 60000;
options.retry = molo::tasks::RetrySpec{100, 1000, 0, 5}; // base 100ms, no jitter
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
std::vector<std::int64_t> runs;
std::mutex runs_mutex;
registry.register_handler("flaky", [&] {
struct H : molo::tasks::TaskHandler {
std::vector<std::int64_t>* runs;
std::mutex* mutex;
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
std::lock_guard<std::mutex> lock(*mutex);
runs->push_back(ctx.clock.steady_ms());
return runs->size() < 3 ? molo::tasks::TaskOutcome::RetryableFailure
: molo::tasks::TaskOutcome::Success;
}
};
auto h = std::make_unique<H>();
h->runs = &runs;
h->mutex = &runs_mutex;
return h;
});
molo::tasks::TaskEnvelope env;
env.name = "flaky";
env.max_retries = 5;
const std::string id = queue.enqueue(std::move(env));
for (int i = 0; i < 3; ++i) {
CHECK(queue.run_one(registry, 5000), "run_one executes the retry");
}
CHECK(runs.size() == 3, "handler ran 3 times");
if (runs.size() == 3) {
const long d1 = static_cast<long>(runs[1] - runs[0]);
const long d2 = static_cast<long>(runs[2] - runs[1]);
// delay(0)=100ms, delay(1)=200ms, within 20 percent plus 40ms slack.
CHECK(d1 >= 80 && d1 <= 100 * 1.2 + 40, "first backoff near 100ms");
CHECK(d2 >= 160 && d2 <= 200 * 1.2 + 40, "second backoff near 200ms");
}
CHECK(queue.stats().succeeded_total == 1, "eventual success counted");
CHECK(queue.stats().retried_total == 2, "two retries counted");
CHECK(!queue.get(id).has_value(), "acked task leaves the queue");
std::cout << "PASS test_retry_backoff_schedule" << std::endl;
}
void test_permanent_failure_dead_letters() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
molo::tasks::TaskRegistry registry;
registry.register_handler("doomed", [] {
struct H : molo::tasks::TaskHandler {
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
return molo::tasks::TaskOutcome::PermanentFailure;
}
};
return std::make_unique<H>();
});
molo::tasks::TaskEnvelope env;
env.name = "doomed";
const std::string id = queue.enqueue(std::move(env));
CHECK(queue.run_one(registry, 1000), "doomed task executes");
CHECK(queue.dead_letters().size() == 1, "permanent failure dead-letters");
auto entries = queue.dead_letters().list(10);
CHECK(entries[0].envelope.id == id, "dead entry keeps the task id");
CHECK(entries[0].reason == "handler-permanent", "dead entry keeps the reason");
// Unknown handler names dead-letter too, never crash the worker.
molo::tasks::TaskEnvelope ghost;
ghost.name = "no.such.handler";
queue.enqueue(std::move(ghost));
CHECK(queue.run_one(registry, 1000), "ghost task executes");
CHECK(queue.dead_letters().size() == 2, "unknown handler dead-letters");
std::cout << "PASS test_permanent_failure_dead_letters" << std::endl;
}
void test_idempotency_keys() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
molo::tasks::TaskRegistry registry;
std::atomic<long> done{0};
registry.register_handler("once", [&] { return std::make_unique<CountingHandler>(done); });
molo::tasks::TaskEnvelope a;
a.name = "once";
a.idempotency_key = "order-123";
const std::string first = queue.enqueue(a);
molo::tasks::TaskEnvelope b;
b.name = "once";
b.idempotency_key = "order-123";
const std::string second = queue.enqueue(std::move(b));
CHECK(first == second, "duplicate key returns the existing id");
CHECK(queue.stats().queued == 1, "duplicate does not re-enqueue");
CHECK(queue.run_one(registry, 1000), "task runs");
molo::tasks::TaskEnvelope c;
c.name = "once";
c.idempotency_key = "order-123";
CHECK(queue.enqueue(std::move(c)) == first, "key reserved after completion");
CHECK(done.load() == 1, "handler ran exactly once");
std::cout << "PASS test_idempotency_keys" << std::endl;
}
void test_caps_and_lease_expiry() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
options.max_depth = 2;
options.lease_ms = 60;
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskEnvelope big;
big.name = "big";
big.payload = std::string(1048576 + 1, 'x');
bool threw = false;
try {
queue.enqueue(std::move(big));
} catch (const molo::tasks::TaskError&) {
threw = true;
}
CHECK(threw, "oversize payload rejected");
molo::tasks::TaskEnvelope t1;
t1.name = "t";
queue.enqueue(std::move(t1));
molo::tasks::TaskEnvelope t2;
t2.name = "t";
queue.enqueue(std::move(t2));
threw = false;
try {
molo::tasks::TaskEnvelope t3;
t3.name = "t";
queue.enqueue(std::move(t3));
} catch (const molo::tasks::TaskError&) {
threw = true;
}
CHECK(threw, "depth cap enforced");
CHECK(!queue.heartbeat("nope"), "unknown heartbeat fails");
// Lease expiry on a fresh queue with a single task: the expired lease
// redelivers the same envelope with attempts incremented.
molo::tasks::QueueOptions exp_options;
exp_options.lease_ms = 60;
molo::tasks::TaskQueue exp_queue(exp_options, clock, obs);
molo::tasks::TaskEnvelope solo;
solo.name = "solo";
exp_queue.enqueue(std::move(solo));
auto leased = exp_queue.lease(0);
CHECK(leased.has_value(), "leases one task");
std::this_thread::sleep_for(std::chrono::milliseconds(120));
auto redelivered = exp_queue.lease(0);
CHECK(redelivered.has_value(), "expired lease redelivers");
if (redelivered.has_value()) {
CHECK(redelivered->envelope.id == leased->envelope.id, "same task redelivered");
CHECK(redelivered->envelope.attempts == 1, "expiry increments attempts");
CHECK(!exp_queue.ack(leased->lease_id), "stale lease cannot ack");
CHECK(exp_queue.ack(redelivered->lease_id), "fresh lease acks");
}
std::cout << "PASS test_caps_and_lease_expiry" << std::endl;
}
void test_hard_timeout_recycles() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
options.orphan_grace_ms = 2000;
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
std::atomic<bool> saw_cancel{false};
registry.register_handler("slow", [&] {
struct H : molo::tasks::TaskHandler {
std::atomic<bool>* saw_cancel;
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
std::this_thread::sleep_for(std::chrono::milliseconds(300));
if (ctx.cancel_requested.load()) {
saw_cancel->store(true);
}
return molo::tasks::TaskOutcome::Success;
}
};
auto h = std::make_unique<H>();
h->saw_cancel = &saw_cancel;
return h;
});
molo::tasks::TaskEnvelope env;
env.name = "slow";
env.timeout_soft_ms = 50;
env.timeout_hard_ms = 100;
env.max_retries = 0; // one strike, then dead
queue.enqueue(std::move(env));
const auto t0 = std::chrono::steady_clock::now();
CHECK(queue.run_one(registry, 1000), "slow task executes");
const auto took = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - t0)
.count();
CHECK(took < 2000, "hard timeout bounds the run");
CHECK(queue.recycle_requested(), "worker recycle requested");
CHECK(saw_cancel.load(), "soft timeout sets the cancel flag");
CHECK(queue.dead_letters().size() == 1, "exhausted task dead-letters");
auto entries = queue.dead_letters().list(10);
CHECK(entries[0].reason == "hard-timeout", "dead reason names the hard timeout");
std::cout << "PASS test_hard_timeout_recycles" << std::endl;
}
void test_envelope_json_roundtrip() {
molo::tasks::TaskEnvelope env;
env.id = "abc";
env.name = "billing.charge";
env.payload = "{\"amount\":42}";
env.attempts = 2;
env.max_retries = 5;
env.run_at_ms = 1700000000000LL;
env.timeout_soft_ms = 30000;
env.timeout_hard_ms = 60000;
env.idempotency_key = "k";
env.enqueued_at_ms = 1700000001000LL;
const std::string json = env.encode();
molo::tasks::TaskEnvelope back;
CHECK(molo::tasks::TaskEnvelope::decode(json, back), "envelope decodes");
CHECK(back.id == "abc" && back.name == "billing.charge", "identity survives");
CHECK(back.payload == "{\"amount\":42}", "payload survives");
CHECK(back.attempts == 2 && back.max_retries == 5, "counters survive");
CHECK(back.run_at_ms.has_value() && *back.run_at_ms == 1700000000000LL, "run_at survives");
CHECK(back.idempotency_key == "k", "key survives");
CHECK(!molo::tasks::TaskEnvelope::decode("{oops", back), "garbage rejected");
CHECK(!molo::tasks::TaskEnvelope::decode("{\"id\":1}", back), "typeless rejected");
CHECK(!molo::tasks::TaskEnvelope::generate_id().empty(), "ids generated");
CHECK(molo::tasks::TaskEnvelope::generate_id() != molo::tasks::TaskEnvelope::generate_id(),
"ids unique");
std::cout << "PASS test_envelope_json_roundtrip" << std::endl;
}
} // namespace
int main() {
test_throughput_off_loop();
test_retry_backoff_schedule();
test_permanent_failure_dead_letters();
test_idempotency_keys();
test_caps_and_lease_expiry();
test_hard_timeout_recycles();
test_envelope_json_roundtrip();
if (failures == 0) {
std::cout << "ALL TASK TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}