Files
moloapp/tests/src/supervisor/test_tasks.cpp
T

349 lines
13 KiB
C++
Raw Normal View History

// V7-2, V7-3: task throughput, retries, backoff, dead letters, timeouts.
#include <atomic>
#include <chrono>
#include <iostream>
#include <mutex>
#include <thread>
#include <vector>
#include "supervisor/common/clock.hpp"
#include "supervisor/common/observer.hpp"
#include "supervisor/tasks/retry_policy.hpp"
#include "supervisor/tasks/task_envelope.hpp"
#include "supervisor/tasks/task_queue.hpp"
#include "supervisor/tasks/task_registry.hpp"
namespace {
int failures = 0;
#define CHECK(cond, msg) \
do { \
if (!(cond)) { \
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
++failures; \
} \
} while (0)
class CountingHandler : public molo::tasks::TaskHandler {
public:
explicit CountingHandler(std::atomic<long>& count) : count_(count) {}
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
++count_;
return molo::tasks::TaskOutcome::Success;
}
private:
std::atomic<long>& count_;
};
void test_throughput_off_loop() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
std::atomic<long> done{0};
registry.register_handler("count", [&] { return std::make_unique<CountingHandler>(done); });
// Baseline scheduling latency of this thread.
auto probe = [] {
const auto t0 = std::chrono::steady_clock::now();
std::this_thread::sleep_for(std::chrono::milliseconds(5));
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
t0)
.count();
};
long baseline = 0;
for (int i = 0; i < 10; ++i) {
baseline += probe();
}
baseline /= 10;
std::atomic<bool> stop{false};
std::vector<std::thread> workers;
for (int i = 0; i < 4; ++i) {
workers.emplace_back([&] { queue.run_worker([&] { return stop.load(); }, registry); });
}
const int total = 500;
const auto t0 = std::chrono::steady_clock::now();
for (int i = 0; i < total; ++i) {
molo::tasks::TaskEnvelope env;
env.name = "count";
queue.enqueue(std::move(env));
}
const auto enqueue_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - t0)
.count();
long loaded = 0;
for (int i = 0; i < 10; ++i) {
loaded += probe();
}
loaded /= 10;
while (done.load() < total) {
std::this_thread::sleep_for(std::chrono::milliseconds(5));
}
stop = true;
for (auto& w : workers) {
w.join();
}
CHECK(done.load() == total, "all 500 tasks succeed");
CHECK(queue.stats().succeeded_total == total, "stats count all successes");
CHECK(enqueue_ms < 2000, "enqueue never blocks on execution");
CHECK(loaded < std::max<long>(50000, baseline * 10), "producer thread stays responsive");
std::cout << "PASS test_throughput_off_loop" << std::endl;
}
void test_retry_backoff_schedule() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
options.lease_ms = 60000;
options.retry = molo::tasks::RetrySpec{100, 1000, 0, 5}; // base 100ms, no jitter
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
std::vector<std::int64_t> runs;
std::mutex runs_mutex;
registry.register_handler("flaky", [&] {
struct H : molo::tasks::TaskHandler {
std::vector<std::int64_t>* runs;
std::mutex* mutex;
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
std::lock_guard<std::mutex> lock(*mutex);
runs->push_back(ctx.clock.steady_ms());
return runs->size() < 3 ? molo::tasks::TaskOutcome::RetryableFailure
: molo::tasks::TaskOutcome::Success;
}
};
auto h = std::make_unique<H>();
h->runs = &runs;
h->mutex = &runs_mutex;
return h;
});
molo::tasks::TaskEnvelope env;
env.name = "flaky";
env.max_retries = 5;
const std::string id = queue.enqueue(std::move(env));
for (int i = 0; i < 3; ++i) {
CHECK(queue.run_one(registry, 5000), "run_one executes the retry");
}
CHECK(runs.size() == 3, "handler ran 3 times");
if (runs.size() == 3) {
const long d1 = static_cast<long>(runs[1] - runs[0]);
const long d2 = static_cast<long>(runs[2] - runs[1]);
// delay(0)=100ms, delay(1)=200ms, within 20 percent plus 40ms slack.
CHECK(d1 >= 80 && d1 <= 100 * 1.2 + 40, "first backoff near 100ms");
CHECK(d2 >= 160 && d2 <= 200 * 1.2 + 40, "second backoff near 200ms");
}
CHECK(queue.stats().succeeded_total == 1, "eventual success counted");
CHECK(queue.stats().retried_total == 2, "two retries counted");
CHECK(!queue.get(id).has_value(), "acked task leaves the queue");
std::cout << "PASS test_retry_backoff_schedule" << std::endl;
}
void test_permanent_failure_dead_letters() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
molo::tasks::TaskRegistry registry;
registry.register_handler("doomed", [] {
struct H : molo::tasks::TaskHandler {
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
return molo::tasks::TaskOutcome::PermanentFailure;
}
};
return std::make_unique<H>();
});
molo::tasks::TaskEnvelope env;
env.name = "doomed";
const std::string id = queue.enqueue(std::move(env));
CHECK(queue.run_one(registry, 1000), "doomed task executes");
CHECK(queue.dead_letters().size() == 1, "permanent failure dead-letters");
auto entries = queue.dead_letters().list(10);
CHECK(entries[0].envelope.id == id, "dead entry keeps the task id");
CHECK(entries[0].reason == "handler-permanent", "dead entry keeps the reason");
// Unknown handler names dead-letter too, never crash the worker.
molo::tasks::TaskEnvelope ghost;
ghost.name = "no.such.handler";
queue.enqueue(std::move(ghost));
CHECK(queue.run_one(registry, 1000), "ghost task executes");
CHECK(queue.dead_letters().size() == 2, "unknown handler dead-letters");
std::cout << "PASS test_permanent_failure_dead_letters" << std::endl;
}
void test_idempotency_keys() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
molo::tasks::TaskRegistry registry;
std::atomic<long> done{0};
registry.register_handler("once", [&] { return std::make_unique<CountingHandler>(done); });
molo::tasks::TaskEnvelope a;
a.name = "once";
a.idempotency_key = "order-123";
const std::string first = queue.enqueue(a);
molo::tasks::TaskEnvelope b;
b.name = "once";
b.idempotency_key = "order-123";
const std::string second = queue.enqueue(std::move(b));
CHECK(first == second, "duplicate key returns the existing id");
CHECK(queue.stats().queued == 1, "duplicate does not re-enqueue");
CHECK(queue.run_one(registry, 1000), "task runs");
molo::tasks::TaskEnvelope c;
c.name = "once";
c.idempotency_key = "order-123";
CHECK(queue.enqueue(std::move(c)) == first, "key reserved after completion");
CHECK(done.load() == 1, "handler ran exactly once");
std::cout << "PASS test_idempotency_keys" << std::endl;
}
void test_caps_and_lease_expiry() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
options.max_depth = 2;
options.lease_ms = 60;
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskEnvelope big;
big.name = "big";
big.payload = std::string(1048576 + 1, 'x');
bool threw = false;
try {
queue.enqueue(std::move(big));
} catch (const molo::tasks::TaskError&) {
threw = true;
}
CHECK(threw, "oversize payload rejected");
molo::tasks::TaskEnvelope t1;
t1.name = "t";
queue.enqueue(std::move(t1));
molo::tasks::TaskEnvelope t2;
t2.name = "t";
queue.enqueue(std::move(t2));
threw = false;
try {
molo::tasks::TaskEnvelope t3;
t3.name = "t";
queue.enqueue(std::move(t3));
} catch (const molo::tasks::TaskError&) {
threw = true;
}
CHECK(threw, "depth cap enforced");
CHECK(!queue.heartbeat("nope"), "unknown heartbeat fails");
// Lease expiry on a fresh queue with a single task: the expired lease
// redelivers the same envelope with attempts incremented.
molo::tasks::QueueOptions exp_options;
exp_options.lease_ms = 60;
molo::tasks::TaskQueue exp_queue(exp_options, clock, obs);
molo::tasks::TaskEnvelope solo;
solo.name = "solo";
exp_queue.enqueue(std::move(solo));
auto leased = exp_queue.lease(0);
CHECK(leased.has_value(), "leases one task");
std::this_thread::sleep_for(std::chrono::milliseconds(120));
auto redelivered = exp_queue.lease(0);
CHECK(redelivered.has_value(), "expired lease redelivers");
if (redelivered.has_value()) {
CHECK(redelivered->envelope.id == leased->envelope.id, "same task redelivered");
CHECK(redelivered->envelope.attempts == 1, "expiry increments attempts");
CHECK(!exp_queue.ack(leased->lease_id), "stale lease cannot ack");
CHECK(exp_queue.ack(redelivered->lease_id), "fresh lease acks");
}
std::cout << "PASS test_caps_and_lease_expiry" << std::endl;
}
void test_hard_timeout_recycles() {
molo::common::SystemClock clock;
auto obs = molo::common::null_observer();
molo::tasks::QueueOptions options;
options.orphan_grace_ms = 2000;
molo::tasks::TaskQueue queue(options, clock, obs);
molo::tasks::TaskRegistry registry;
std::atomic<bool> saw_cancel{false};
registry.register_handler("slow", [&] {
struct H : molo::tasks::TaskHandler {
std::atomic<bool>* saw_cancel;
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
std::this_thread::sleep_for(std::chrono::milliseconds(300));
if (ctx.cancel_requested.load()) {
saw_cancel->store(true);
}
return molo::tasks::TaskOutcome::Success;
}
};
auto h = std::make_unique<H>();
h->saw_cancel = &saw_cancel;
return h;
});
molo::tasks::TaskEnvelope env;
env.name = "slow";
env.timeout_soft_ms = 50;
env.timeout_hard_ms = 100;
env.max_retries = 0; // one strike, then dead
queue.enqueue(std::move(env));
const auto t0 = std::chrono::steady_clock::now();
CHECK(queue.run_one(registry, 1000), "slow task executes");
const auto took = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - t0)
.count();
CHECK(took < 2000, "hard timeout bounds the run");
CHECK(queue.recycle_requested(), "worker recycle requested");
CHECK(saw_cancel.load(), "soft timeout sets the cancel flag");
CHECK(queue.dead_letters().size() == 1, "exhausted task dead-letters");
auto entries = queue.dead_letters().list(10);
CHECK(entries[0].reason == "hard-timeout", "dead reason names the hard timeout");
std::cout << "PASS test_hard_timeout_recycles" << std::endl;
}
void test_envelope_json_roundtrip() {
molo::tasks::TaskEnvelope env;
env.id = "abc";
env.name = "billing.charge";
env.payload = "{\"amount\":42}";
env.attempts = 2;
env.max_retries = 5;
env.run_at_ms = 1700000000000LL;
env.timeout_soft_ms = 30000;
env.timeout_hard_ms = 60000;
env.idempotency_key = "k";
env.enqueued_at_ms = 1700000001000LL;
const std::string json = env.encode();
molo::tasks::TaskEnvelope back;
CHECK(molo::tasks::TaskEnvelope::decode(json, back), "envelope decodes");
CHECK(back.id == "abc" && back.name == "billing.charge", "identity survives");
CHECK(back.payload == "{\"amount\":42}", "payload survives");
CHECK(back.attempts == 2 && back.max_retries == 5, "counters survive");
CHECK(back.run_at_ms.has_value() && *back.run_at_ms == 1700000000000LL, "run_at survives");
CHECK(back.idempotency_key == "k", "key survives");
CHECK(!molo::tasks::TaskEnvelope::decode("{oops", back), "garbage rejected");
CHECK(!molo::tasks::TaskEnvelope::decode("{\"id\":1}", back), "typeless rejected");
CHECK(!molo::tasks::TaskEnvelope::generate_id().empty(), "ids generated");
CHECK(molo::tasks::TaskEnvelope::generate_id() != molo::tasks::TaskEnvelope::generate_id(),
"ids unique");
std::cout << "PASS test_envelope_json_roundtrip" << std::endl;
}
} // namespace
int main() {
test_throughput_off_loop();
test_retry_backoff_schedule();
test_permanent_failure_dead_letters();
test_idempotency_keys();
test_caps_and_lease_expiry();
test_hard_timeout_recycles();
test_envelope_json_roundtrip();
if (failures == 0) {
std::cout << "ALL TASK TESTS PASSED" << std::endl;
return 0;
}
std::cout << failures << " FAILURES" << std::endl;
return 1;
}