// V7-2, V7-3: task throughput, retries, backoff, dead letters, timeouts. #include #include #include #include #include #include #include "supervisor/common/clock.hpp" #include "supervisor/common/observer.hpp" #include "supervisor/tasks/retry_policy.hpp" #include "supervisor/tasks/task_envelope.hpp" #include "supervisor/tasks/task_queue.hpp" #include "supervisor/tasks/task_registry.hpp" namespace { int failures = 0; #define CHECK(cond, msg) \ do { \ if (!(cond)) { \ std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ ++failures; \ } \ } while (0) class CountingHandler : public molo::tasks::TaskHandler { public: explicit CountingHandler(std::atomic& count) : count_(count) {} molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override { ++count_; return molo::tasks::TaskOutcome::Success; } private: std::atomic& count_; }; void test_throughput_off_loop() { molo::common::SystemClock clock; auto obs = molo::common::null_observer(); molo::tasks::QueueOptions options; molo::tasks::TaskQueue queue(options, clock, obs); molo::tasks::TaskRegistry registry; std::atomic done{0}; registry.register_handler("count", [&] { return std::make_unique(done); }); // Baseline scheduling latency of this thread. auto probe = [] { const auto t0 = std::chrono::steady_clock::now(); std::this_thread::sleep_for(std::chrono::milliseconds(5)); return std::chrono::duration_cast(std::chrono::steady_clock::now() - t0) .count(); }; long baseline = 0; for (int i = 0; i < 10; ++i) { baseline += probe(); } baseline /= 10; std::atomic stop{false}; std::vector workers; for (int i = 0; i < 4; ++i) { workers.emplace_back([&] { queue.run_worker([&] { return stop.load(); }, registry); }); } const int total = 500; const auto t0 = std::chrono::steady_clock::now(); for (int i = 0; i < total; ++i) { molo::tasks::TaskEnvelope env; env.name = "count"; queue.enqueue(std::move(env)); } const auto enqueue_ms = std::chrono::duration_cast( std::chrono::steady_clock::now() - t0) .count(); long loaded = 0; for (int i = 0; i < 10; ++i) { loaded += probe(); } loaded /= 10; while (done.load() < total) { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } stop = true; for (auto& w : workers) { w.join(); } CHECK(done.load() == total, "all 500 tasks succeed"); CHECK(queue.stats().succeeded_total == total, "stats count all successes"); CHECK(enqueue_ms < 2000, "enqueue never blocks on execution"); CHECK(loaded < std::max(50000, baseline * 10), "producer thread stays responsive"); std::cout << "PASS test_throughput_off_loop" << std::endl; } void test_retry_backoff_schedule() { molo::common::SystemClock clock; auto obs = molo::common::null_observer(); molo::tasks::QueueOptions options; options.lease_ms = 60000; options.retry = molo::tasks::RetrySpec{100, 1000, 0, 5}; // base 100ms, no jitter molo::tasks::TaskQueue queue(options, clock, obs); molo::tasks::TaskRegistry registry; std::vector runs; std::mutex runs_mutex; registry.register_handler("flaky", [&] { struct H : molo::tasks::TaskHandler { std::vector* runs; std::mutex* mutex; molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override { std::lock_guard lock(*mutex); runs->push_back(ctx.clock.steady_ms()); return runs->size() < 3 ? molo::tasks::TaskOutcome::RetryableFailure : molo::tasks::TaskOutcome::Success; } }; auto h = std::make_unique(); h->runs = &runs; h->mutex = &runs_mutex; return h; }); molo::tasks::TaskEnvelope env; env.name = "flaky"; env.max_retries = 5; const std::string id = queue.enqueue(std::move(env)); for (int i = 0; i < 3; ++i) { CHECK(queue.run_one(registry, 5000), "run_one executes the retry"); } CHECK(runs.size() == 3, "handler ran 3 times"); if (runs.size() == 3) { const long d1 = static_cast(runs[1] - runs[0]); const long d2 = static_cast(runs[2] - runs[1]); // delay(0)=100ms, delay(1)=200ms, within 20 percent plus 40ms slack. CHECK(d1 >= 80 && d1 <= 100 * 1.2 + 40, "first backoff near 100ms"); CHECK(d2 >= 160 && d2 <= 200 * 1.2 + 40, "second backoff near 200ms"); } CHECK(queue.stats().succeeded_total == 1, "eventual success counted"); CHECK(queue.stats().retried_total == 2, "two retries counted"); CHECK(!queue.get(id).has_value(), "acked task leaves the queue"); std::cout << "PASS test_retry_backoff_schedule" << std::endl; } void test_permanent_failure_dead_letters() { molo::common::SystemClock clock; auto obs = molo::common::null_observer(); molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs); molo::tasks::TaskRegistry registry; registry.register_handler("doomed", [] { struct H : molo::tasks::TaskHandler { molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override { return molo::tasks::TaskOutcome::PermanentFailure; } }; return std::make_unique(); }); molo::tasks::TaskEnvelope env; env.name = "doomed"; const std::string id = queue.enqueue(std::move(env)); CHECK(queue.run_one(registry, 1000), "doomed task executes"); CHECK(queue.dead_letters().size() == 1, "permanent failure dead-letters"); auto entries = queue.dead_letters().list(10); CHECK(entries[0].envelope.id == id, "dead entry keeps the task id"); CHECK(entries[0].reason == "handler-permanent", "dead entry keeps the reason"); // Unknown handler names dead-letter too, never crash the worker. molo::tasks::TaskEnvelope ghost; ghost.name = "no.such.handler"; queue.enqueue(std::move(ghost)); CHECK(queue.run_one(registry, 1000), "ghost task executes"); CHECK(queue.dead_letters().size() == 2, "unknown handler dead-letters"); std::cout << "PASS test_permanent_failure_dead_letters" << std::endl; } void test_idempotency_keys() { molo::common::SystemClock clock; auto obs = molo::common::null_observer(); molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs); molo::tasks::TaskRegistry registry; std::atomic done{0}; registry.register_handler("once", [&] { return std::make_unique(done); }); molo::tasks::TaskEnvelope a; a.name = "once"; a.idempotency_key = "order-123"; const std::string first = queue.enqueue(a); molo::tasks::TaskEnvelope b; b.name = "once"; b.idempotency_key = "order-123"; const std::string second = queue.enqueue(std::move(b)); CHECK(first == second, "duplicate key returns the existing id"); CHECK(queue.stats().queued == 1, "duplicate does not re-enqueue"); CHECK(queue.run_one(registry, 1000), "task runs"); molo::tasks::TaskEnvelope c; c.name = "once"; c.idempotency_key = "order-123"; CHECK(queue.enqueue(std::move(c)) == first, "key reserved after completion"); CHECK(done.load() == 1, "handler ran exactly once"); std::cout << "PASS test_idempotency_keys" << std::endl; } void test_caps_and_lease_expiry() { molo::common::SystemClock clock; auto obs = molo::common::null_observer(); molo::tasks::QueueOptions options; options.max_depth = 2; options.lease_ms = 60; molo::tasks::TaskQueue queue(options, clock, obs); molo::tasks::TaskEnvelope big; big.name = "big"; big.payload = std::string(1048576 + 1, 'x'); bool threw = false; try { queue.enqueue(std::move(big)); } catch (const molo::tasks::TaskError&) { threw = true; } CHECK(threw, "oversize payload rejected"); molo::tasks::TaskEnvelope t1; t1.name = "t"; queue.enqueue(std::move(t1)); molo::tasks::TaskEnvelope t2; t2.name = "t"; queue.enqueue(std::move(t2)); threw = false; try { molo::tasks::TaskEnvelope t3; t3.name = "t"; queue.enqueue(std::move(t3)); } catch (const molo::tasks::TaskError&) { threw = true; } CHECK(threw, "depth cap enforced"); CHECK(!queue.heartbeat("nope"), "unknown heartbeat fails"); // Lease expiry on a fresh queue with a single task: the expired lease // redelivers the same envelope with attempts incremented. molo::tasks::QueueOptions exp_options; exp_options.lease_ms = 60; molo::tasks::TaskQueue exp_queue(exp_options, clock, obs); molo::tasks::TaskEnvelope solo; solo.name = "solo"; exp_queue.enqueue(std::move(solo)); auto leased = exp_queue.lease(0); CHECK(leased.has_value(), "leases one task"); std::this_thread::sleep_for(std::chrono::milliseconds(120)); auto redelivered = exp_queue.lease(0); CHECK(redelivered.has_value(), "expired lease redelivers"); if (redelivered.has_value()) { CHECK(redelivered->envelope.id == leased->envelope.id, "same task redelivered"); CHECK(redelivered->envelope.attempts == 1, "expiry increments attempts"); CHECK(!exp_queue.ack(leased->lease_id), "stale lease cannot ack"); CHECK(exp_queue.ack(redelivered->lease_id), "fresh lease acks"); } std::cout << "PASS test_caps_and_lease_expiry" << std::endl; } void test_hard_timeout_recycles() { molo::common::SystemClock clock; auto obs = molo::common::null_observer(); molo::tasks::QueueOptions options; options.orphan_grace_ms = 2000; molo::tasks::TaskQueue queue(options, clock, obs); molo::tasks::TaskRegistry registry; std::atomic saw_cancel{false}; registry.register_handler("slow", [&] { struct H : molo::tasks::TaskHandler { std::atomic* saw_cancel; molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override { std::this_thread::sleep_for(std::chrono::milliseconds(300)); if (ctx.cancel_requested.load()) { saw_cancel->store(true); } return molo::tasks::TaskOutcome::Success; } }; auto h = std::make_unique(); h->saw_cancel = &saw_cancel; return h; }); molo::tasks::TaskEnvelope env; env.name = "slow"; env.timeout_soft_ms = 50; env.timeout_hard_ms = 100; env.max_retries = 0; // one strike, then dead queue.enqueue(std::move(env)); const auto t0 = std::chrono::steady_clock::now(); CHECK(queue.run_one(registry, 1000), "slow task executes"); const auto took = std::chrono::duration_cast( std::chrono::steady_clock::now() - t0) .count(); CHECK(took < 2000, "hard timeout bounds the run"); CHECK(queue.recycle_requested(), "worker recycle requested"); CHECK(saw_cancel.load(), "soft timeout sets the cancel flag"); CHECK(queue.dead_letters().size() == 1, "exhausted task dead-letters"); auto entries = queue.dead_letters().list(10); CHECK(entries[0].reason == "hard-timeout", "dead reason names the hard timeout"); std::cout << "PASS test_hard_timeout_recycles" << std::endl; } void test_envelope_json_roundtrip() { molo::tasks::TaskEnvelope env; env.id = "abc"; env.name = "billing.charge"; env.payload = "{\"amount\":42}"; env.attempts = 2; env.max_retries = 5; env.run_at_ms = 1700000000000LL; env.timeout_soft_ms = 30000; env.timeout_hard_ms = 60000; env.idempotency_key = "k"; env.enqueued_at_ms = 1700000001000LL; const std::string json = env.encode(); molo::tasks::TaskEnvelope back; CHECK(molo::tasks::TaskEnvelope::decode(json, back), "envelope decodes"); CHECK(back.id == "abc" && back.name == "billing.charge", "identity survives"); CHECK(back.payload == "{\"amount\":42}", "payload survives"); CHECK(back.attempts == 2 && back.max_retries == 5, "counters survive"); CHECK(back.run_at_ms.has_value() && *back.run_at_ms == 1700000000000LL, "run_at survives"); CHECK(back.idempotency_key == "k", "key survives"); CHECK(!molo::tasks::TaskEnvelope::decode("{oops", back), "garbage rejected"); CHECK(!molo::tasks::TaskEnvelope::decode("{\"id\":1}", back), "typeless rejected"); CHECK(!molo::tasks::TaskEnvelope::generate_id().empty(), "ids generated"); CHECK(molo::tasks::TaskEnvelope::generate_id() != molo::tasks::TaskEnvelope::generate_id(), "ids unique"); std::cout << "PASS test_envelope_json_roundtrip" << std::endl; } } // namespace int main() { test_throughput_off_loop(); test_retry_backoff_schedule(); test_permanent_failure_dead_letters(); test_idempotency_keys(); test_caps_and_lease_expiry(); test_hard_timeout_recycles(); test_envelope_json_roundtrip(); if (failures == 0) { std::cout << "ALL TASK TESTS PASSED" << std::endl; return 0; } std::cout << failures << " FAILURES" << std::endl; return 1; }