Full C++23 framework tree: runtime, observability, security, data, app, realtime, supervisor, clients, CLI, example plus enterprise tenancy, audit, operability, governance, supplychain. Postgres via docker compose for tests.
349 lines
13 KiB
C++
349 lines
13 KiB
C++
// V7-2, V7-3: task throughput, retries, backoff, dead letters, timeouts.
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <iostream>
|
|
#include <mutex>
|
|
#include <thread>
|
|
#include <vector>
|
|
|
|
#include "supervisor/common/clock.hpp"
|
|
#include "supervisor/common/observer.hpp"
|
|
#include "supervisor/tasks/retry_policy.hpp"
|
|
#include "supervisor/tasks/task_envelope.hpp"
|
|
#include "supervisor/tasks/task_queue.hpp"
|
|
#include "supervisor/tasks/task_registry.hpp"
|
|
|
|
namespace {
|
|
|
|
int failures = 0;
|
|
|
|
#define CHECK(cond, msg) \
|
|
do { \
|
|
if (!(cond)) { \
|
|
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
|
++failures; \
|
|
} \
|
|
} while (0)
|
|
|
|
class CountingHandler : public molo::tasks::TaskHandler {
|
|
public:
|
|
explicit CountingHandler(std::atomic<long>& count) : count_(count) {}
|
|
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
|
|
++count_;
|
|
return molo::tasks::TaskOutcome::Success;
|
|
}
|
|
|
|
private:
|
|
std::atomic<long>& count_;
|
|
};
|
|
|
|
void test_throughput_off_loop() {
|
|
molo::common::SystemClock clock;
|
|
auto obs = molo::common::null_observer();
|
|
molo::tasks::QueueOptions options;
|
|
molo::tasks::TaskQueue queue(options, clock, obs);
|
|
molo::tasks::TaskRegistry registry;
|
|
std::atomic<long> done{0};
|
|
registry.register_handler("count", [&] { return std::make_unique<CountingHandler>(done); });
|
|
|
|
// Baseline scheduling latency of this thread.
|
|
auto probe = [] {
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
|
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() -
|
|
t0)
|
|
.count();
|
|
};
|
|
long baseline = 0;
|
|
for (int i = 0; i < 10; ++i) {
|
|
baseline += probe();
|
|
}
|
|
baseline /= 10;
|
|
|
|
std::atomic<bool> stop{false};
|
|
std::vector<std::thread> workers;
|
|
for (int i = 0; i < 4; ++i) {
|
|
workers.emplace_back([&] { queue.run_worker([&] { return stop.load(); }, registry); });
|
|
}
|
|
const int total = 500;
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
for (int i = 0; i < total; ++i) {
|
|
molo::tasks::TaskEnvelope env;
|
|
env.name = "count";
|
|
queue.enqueue(std::move(env));
|
|
}
|
|
const auto enqueue_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - t0)
|
|
.count();
|
|
long loaded = 0;
|
|
for (int i = 0; i < 10; ++i) {
|
|
loaded += probe();
|
|
}
|
|
loaded /= 10;
|
|
while (done.load() < total) {
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
|
}
|
|
stop = true;
|
|
for (auto& w : workers) {
|
|
w.join();
|
|
}
|
|
CHECK(done.load() == total, "all 500 tasks succeed");
|
|
CHECK(queue.stats().succeeded_total == total, "stats count all successes");
|
|
CHECK(enqueue_ms < 2000, "enqueue never blocks on execution");
|
|
CHECK(loaded < std::max<long>(50000, baseline * 10), "producer thread stays responsive");
|
|
std::cout << "PASS test_throughput_off_loop" << std::endl;
|
|
}
|
|
|
|
void test_retry_backoff_schedule() {
|
|
molo::common::SystemClock clock;
|
|
auto obs = molo::common::null_observer();
|
|
molo::tasks::QueueOptions options;
|
|
options.lease_ms = 60000;
|
|
options.retry = molo::tasks::RetrySpec{100, 1000, 0, 5}; // base 100ms, no jitter
|
|
molo::tasks::TaskQueue queue(options, clock, obs);
|
|
molo::tasks::TaskRegistry registry;
|
|
std::vector<std::int64_t> runs;
|
|
std::mutex runs_mutex;
|
|
registry.register_handler("flaky", [&] {
|
|
struct H : molo::tasks::TaskHandler {
|
|
std::vector<std::int64_t>* runs;
|
|
std::mutex* mutex;
|
|
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
|
|
std::lock_guard<std::mutex> lock(*mutex);
|
|
runs->push_back(ctx.clock.steady_ms());
|
|
return runs->size() < 3 ? molo::tasks::TaskOutcome::RetryableFailure
|
|
: molo::tasks::TaskOutcome::Success;
|
|
}
|
|
};
|
|
auto h = std::make_unique<H>();
|
|
h->runs = &runs;
|
|
h->mutex = &runs_mutex;
|
|
return h;
|
|
});
|
|
|
|
molo::tasks::TaskEnvelope env;
|
|
env.name = "flaky";
|
|
env.max_retries = 5;
|
|
const std::string id = queue.enqueue(std::move(env));
|
|
for (int i = 0; i < 3; ++i) {
|
|
CHECK(queue.run_one(registry, 5000), "run_one executes the retry");
|
|
}
|
|
CHECK(runs.size() == 3, "handler ran 3 times");
|
|
if (runs.size() == 3) {
|
|
const long d1 = static_cast<long>(runs[1] - runs[0]);
|
|
const long d2 = static_cast<long>(runs[2] - runs[1]);
|
|
// delay(0)=100ms, delay(1)=200ms, within 20 percent plus 40ms slack.
|
|
CHECK(d1 >= 80 && d1 <= 100 * 1.2 + 40, "first backoff near 100ms");
|
|
CHECK(d2 >= 160 && d2 <= 200 * 1.2 + 40, "second backoff near 200ms");
|
|
}
|
|
CHECK(queue.stats().succeeded_total == 1, "eventual success counted");
|
|
CHECK(queue.stats().retried_total == 2, "two retries counted");
|
|
CHECK(!queue.get(id).has_value(), "acked task leaves the queue");
|
|
std::cout << "PASS test_retry_backoff_schedule" << std::endl;
|
|
}
|
|
|
|
void test_permanent_failure_dead_letters() {
|
|
molo::common::SystemClock clock;
|
|
auto obs = molo::common::null_observer();
|
|
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
|
|
molo::tasks::TaskRegistry registry;
|
|
registry.register_handler("doomed", [] {
|
|
struct H : molo::tasks::TaskHandler {
|
|
molo::tasks::TaskOutcome run(molo::tasks::TaskContext&) override {
|
|
return molo::tasks::TaskOutcome::PermanentFailure;
|
|
}
|
|
};
|
|
return std::make_unique<H>();
|
|
});
|
|
molo::tasks::TaskEnvelope env;
|
|
env.name = "doomed";
|
|
const std::string id = queue.enqueue(std::move(env));
|
|
CHECK(queue.run_one(registry, 1000), "doomed task executes");
|
|
CHECK(queue.dead_letters().size() == 1, "permanent failure dead-letters");
|
|
auto entries = queue.dead_letters().list(10);
|
|
CHECK(entries[0].envelope.id == id, "dead entry keeps the task id");
|
|
CHECK(entries[0].reason == "handler-permanent", "dead entry keeps the reason");
|
|
|
|
// Unknown handler names dead-letter too, never crash the worker.
|
|
molo::tasks::TaskEnvelope ghost;
|
|
ghost.name = "no.such.handler";
|
|
queue.enqueue(std::move(ghost));
|
|
CHECK(queue.run_one(registry, 1000), "ghost task executes");
|
|
CHECK(queue.dead_letters().size() == 2, "unknown handler dead-letters");
|
|
std::cout << "PASS test_permanent_failure_dead_letters" << std::endl;
|
|
}
|
|
|
|
void test_idempotency_keys() {
|
|
molo::common::SystemClock clock;
|
|
auto obs = molo::common::null_observer();
|
|
molo::tasks::TaskQueue queue(molo::tasks::QueueOptions{}, clock, obs);
|
|
molo::tasks::TaskRegistry registry;
|
|
std::atomic<long> done{0};
|
|
registry.register_handler("once", [&] { return std::make_unique<CountingHandler>(done); });
|
|
|
|
molo::tasks::TaskEnvelope a;
|
|
a.name = "once";
|
|
a.idempotency_key = "order-123";
|
|
const std::string first = queue.enqueue(a);
|
|
molo::tasks::TaskEnvelope b;
|
|
b.name = "once";
|
|
b.idempotency_key = "order-123";
|
|
const std::string second = queue.enqueue(std::move(b));
|
|
CHECK(first == second, "duplicate key returns the existing id");
|
|
CHECK(queue.stats().queued == 1, "duplicate does not re-enqueue");
|
|
CHECK(queue.run_one(registry, 1000), "task runs");
|
|
molo::tasks::TaskEnvelope c;
|
|
c.name = "once";
|
|
c.idempotency_key = "order-123";
|
|
CHECK(queue.enqueue(std::move(c)) == first, "key reserved after completion");
|
|
CHECK(done.load() == 1, "handler ran exactly once");
|
|
std::cout << "PASS test_idempotency_keys" << std::endl;
|
|
}
|
|
|
|
void test_caps_and_lease_expiry() {
|
|
molo::common::SystemClock clock;
|
|
auto obs = molo::common::null_observer();
|
|
molo::tasks::QueueOptions options;
|
|
options.max_depth = 2;
|
|
options.lease_ms = 60;
|
|
molo::tasks::TaskQueue queue(options, clock, obs);
|
|
|
|
molo::tasks::TaskEnvelope big;
|
|
big.name = "big";
|
|
big.payload = std::string(1048576 + 1, 'x');
|
|
bool threw = false;
|
|
try {
|
|
queue.enqueue(std::move(big));
|
|
} catch (const molo::tasks::TaskError&) {
|
|
threw = true;
|
|
}
|
|
CHECK(threw, "oversize payload rejected");
|
|
|
|
molo::tasks::TaskEnvelope t1;
|
|
t1.name = "t";
|
|
queue.enqueue(std::move(t1));
|
|
molo::tasks::TaskEnvelope t2;
|
|
t2.name = "t";
|
|
queue.enqueue(std::move(t2));
|
|
threw = false;
|
|
try {
|
|
molo::tasks::TaskEnvelope t3;
|
|
t3.name = "t";
|
|
queue.enqueue(std::move(t3));
|
|
} catch (const molo::tasks::TaskError&) {
|
|
threw = true;
|
|
}
|
|
CHECK(threw, "depth cap enforced");
|
|
CHECK(!queue.heartbeat("nope"), "unknown heartbeat fails");
|
|
|
|
// Lease expiry on a fresh queue with a single task: the expired lease
|
|
// redelivers the same envelope with attempts incremented.
|
|
molo::tasks::QueueOptions exp_options;
|
|
exp_options.lease_ms = 60;
|
|
molo::tasks::TaskQueue exp_queue(exp_options, clock, obs);
|
|
molo::tasks::TaskEnvelope solo;
|
|
solo.name = "solo";
|
|
exp_queue.enqueue(std::move(solo));
|
|
auto leased = exp_queue.lease(0);
|
|
CHECK(leased.has_value(), "leases one task");
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(120));
|
|
auto redelivered = exp_queue.lease(0);
|
|
CHECK(redelivered.has_value(), "expired lease redelivers");
|
|
if (redelivered.has_value()) {
|
|
CHECK(redelivered->envelope.id == leased->envelope.id, "same task redelivered");
|
|
CHECK(redelivered->envelope.attempts == 1, "expiry increments attempts");
|
|
CHECK(!exp_queue.ack(leased->lease_id), "stale lease cannot ack");
|
|
CHECK(exp_queue.ack(redelivered->lease_id), "fresh lease acks");
|
|
}
|
|
std::cout << "PASS test_caps_and_lease_expiry" << std::endl;
|
|
}
|
|
|
|
void test_hard_timeout_recycles() {
|
|
molo::common::SystemClock clock;
|
|
auto obs = molo::common::null_observer();
|
|
molo::tasks::QueueOptions options;
|
|
options.orphan_grace_ms = 2000;
|
|
molo::tasks::TaskQueue queue(options, clock, obs);
|
|
molo::tasks::TaskRegistry registry;
|
|
std::atomic<bool> saw_cancel{false};
|
|
registry.register_handler("slow", [&] {
|
|
struct H : molo::tasks::TaskHandler {
|
|
std::atomic<bool>* saw_cancel;
|
|
molo::tasks::TaskOutcome run(molo::tasks::TaskContext& ctx) override {
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(300));
|
|
if (ctx.cancel_requested.load()) {
|
|
saw_cancel->store(true);
|
|
}
|
|
return molo::tasks::TaskOutcome::Success;
|
|
}
|
|
};
|
|
auto h = std::make_unique<H>();
|
|
h->saw_cancel = &saw_cancel;
|
|
return h;
|
|
});
|
|
molo::tasks::TaskEnvelope env;
|
|
env.name = "slow";
|
|
env.timeout_soft_ms = 50;
|
|
env.timeout_hard_ms = 100;
|
|
env.max_retries = 0; // one strike, then dead
|
|
queue.enqueue(std::move(env));
|
|
const auto t0 = std::chrono::steady_clock::now();
|
|
CHECK(queue.run_one(registry, 1000), "slow task executes");
|
|
const auto took = std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
std::chrono::steady_clock::now() - t0)
|
|
.count();
|
|
CHECK(took < 2000, "hard timeout bounds the run");
|
|
CHECK(queue.recycle_requested(), "worker recycle requested");
|
|
CHECK(saw_cancel.load(), "soft timeout sets the cancel flag");
|
|
CHECK(queue.dead_letters().size() == 1, "exhausted task dead-letters");
|
|
auto entries = queue.dead_letters().list(10);
|
|
CHECK(entries[0].reason == "hard-timeout", "dead reason names the hard timeout");
|
|
std::cout << "PASS test_hard_timeout_recycles" << std::endl;
|
|
}
|
|
|
|
void test_envelope_json_roundtrip() {
|
|
molo::tasks::TaskEnvelope env;
|
|
env.id = "abc";
|
|
env.name = "billing.charge";
|
|
env.payload = "{\"amount\":42}";
|
|
env.attempts = 2;
|
|
env.max_retries = 5;
|
|
env.run_at_ms = 1700000000000LL;
|
|
env.timeout_soft_ms = 30000;
|
|
env.timeout_hard_ms = 60000;
|
|
env.idempotency_key = "k";
|
|
env.enqueued_at_ms = 1700000001000LL;
|
|
const std::string json = env.encode();
|
|
molo::tasks::TaskEnvelope back;
|
|
CHECK(molo::tasks::TaskEnvelope::decode(json, back), "envelope decodes");
|
|
CHECK(back.id == "abc" && back.name == "billing.charge", "identity survives");
|
|
CHECK(back.payload == "{\"amount\":42}", "payload survives");
|
|
CHECK(back.attempts == 2 && back.max_retries == 5, "counters survive");
|
|
CHECK(back.run_at_ms.has_value() && *back.run_at_ms == 1700000000000LL, "run_at survives");
|
|
CHECK(back.idempotency_key == "k", "key survives");
|
|
CHECK(!molo::tasks::TaskEnvelope::decode("{oops", back), "garbage rejected");
|
|
CHECK(!molo::tasks::TaskEnvelope::decode("{\"id\":1}", back), "typeless rejected");
|
|
CHECK(!molo::tasks::TaskEnvelope::generate_id().empty(), "ids generated");
|
|
CHECK(molo::tasks::TaskEnvelope::generate_id() != molo::tasks::TaskEnvelope::generate_id(),
|
|
"ids unique");
|
|
std::cout << "PASS test_envelope_json_roundtrip" << std::endl;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
int main() {
|
|
test_throughput_off_loop();
|
|
test_retry_backoff_schedule();
|
|
test_permanent_failure_dead_letters();
|
|
test_idempotency_keys();
|
|
test_caps_and_lease_expiry();
|
|
test_hard_timeout_recycles();
|
|
test_envelope_json_roundtrip();
|
|
if (failures == 0) {
|
|
std::cout << "ALL TASK TESTS PASSED" << std::endl;
|
|
return 0;
|
|
}
|
|
std::cout << failures << " FAILURES" << std::endl;
|
|
return 1;
|
|
}
|