diff --git a/src/supervisor/TOPOLOGY.md b/src/supervisor/TOPOLOGY.md index 68a7b2f..db484d2 100644 --- a/src/supervisor/TOPOLOGY.md +++ b/src/supervisor/TOPOLOGY.md @@ -82,8 +82,13 @@ the queue dedupes them and no duplicate run happens. ## Known v1 limits -- Thread workers only; OS-process isolation is a v2 transport behind the - same `WorkFunc` interface (O7-2). +- Workers are threads. An OS program is supervised by wrapping it in + `make_process_work(ProcessSpec)` (O7-2): the `WorkFunc` forks and execs + the program, SIGTERM then SIGKILL after a grace on stop, and the child + gets a parent-death SIGTERM so it never outlives the supervisor, even + on SIGKILL of the parent. Restarts, backoff, and intensity caps come + from the owning `WorkerGroup` like any other worker. Gap: no + per-process resource limits (rlimit/cgroup) yet. - In-house backends only; no Redis-protocol compatibility (O7-1). - At-least-once everywhere; no exactly-once primitive (O7-3). Handlers must be idempotent (R7-2). diff --git a/src/supervisor/common/caps.hpp b/src/supervisor/common/caps.hpp index 0536c32..9079b6d 100644 --- a/src/supervisor/common/caps.hpp +++ b/src/supervisor/common/caps.hpp @@ -44,6 +44,9 @@ struct Caps { static constexpr int kJsonMaxDepth = 64; static constexpr int kMaxStreamsPerBus = 1024; static constexpr int kMaxConsumersPerStream = 1024; + static constexpr std::size_t kProcessArgvMax = 256; + static constexpr long kProcessStopGraceMs = 3000; // SIGTERM, then SIGKILL + static constexpr long kProcessPollMs = 50; }; } // namespace molo::common diff --git a/src/supervisor/supervisor.mk b/src/supervisor/supervisor.mk index d30767d..996f0eb 100644 --- a/src/supervisor/supervisor.mk +++ b/src/supervisor/supervisor.mk @@ -29,6 +29,7 @@ SUP_SRCS = \ $(SUP_ROOT)/tasks/scheduler.cpp \ $(SUP_ROOT)/supervisor/restart_policy.cpp \ $(SUP_ROOT)/supervisor/worker_process.cpp \ + $(SUP_ROOT)/supervisor/process_work.cpp \ $(SUP_ROOT)/supervisor/health_check.cpp \ $(SUP_ROOT)/supervisor/worker_group.cpp \ $(SUP_ROOT)/supervisor/shutdown_coordinator.cpp \ @@ -46,6 +47,7 @@ SUP_OBJS = $(SUP_SRCS:$(SUP_ROOT)/%.cpp=$(SUP_BUILD)/%.o) SUP_TESTS = \ test_supervisor \ + test_process_work \ test_tasks \ test_scheduler \ test_bus \ diff --git a/src/supervisor/supervisor/process_work.cpp b/src/supervisor/supervisor/process_work.cpp new file mode 100644 index 0000000..7c9a7dc --- /dev/null +++ b/src/supervisor/supervisor/process_work.cpp @@ -0,0 +1,96 @@ +#include "supervisor/supervisor/process_work.hpp" + +#include +#include +#include +#include + +#include +#include + +#include "supervisor/common/caps.hpp" + +namespace molo::supervisor { + +namespace { + +int decode(int status) { + if (WIFEXITED(status)) { + return WEXITSTATUS(status); + } + if (WIFSIGNALED(status)) { + return 128 + WTERMSIG(status); + } + return 1; +} + +void sleep_poll(long ms) { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); +} + +// Terminates the child's process group and reaps it. +void terminate(pid_t pid, long grace_ms) { + kill(-pid, SIGTERM); + int status = 0; + long waited = 0; + while (waitpid(pid, &status, WNOHANG) == 0) { + if (waited >= grace_ms) { + kill(-pid, SIGKILL); + waitpid(pid, &status, 0); + return; + } + sleep_poll(common::Caps::kProcessPollMs); + waited += common::Caps::kProcessPollMs; + } +} + +} // namespace + +WorkFunc make_process_work(ProcessSpec spec) { + return [spec = std::move(spec)](const std::atomic& stop) -> int { + if (spec.argv.empty() || spec.argv.size() > common::Caps::kProcessArgvMax) { + return 1; + } + const long grace = + spec.stop_grace_ms > 0 ? spec.stop_grace_ms : common::Caps::kProcessStopGraceMs; + std::vector args; + for (const auto& a : spec.argv) { + args.push_back(const_cast(a.c_str())); + } + args.push_back(nullptr); + + const pid_t parent = getpid(); + const pid_t pid = fork(); + if (pid < 0) { + return 1; + } + if (pid == 0) { + setpgid(0, 0); // own group so grandchildren die with the child + prctl(PR_SET_PDEATHSIG, SIGTERM); + if (getppid() != parent) { + _exit(1); // parent already gone + } + execvp(args[0], args.data()); + _exit(127); + } + setpgid(pid, pid); // also from the parent side, avoids the exec race + + int status = 0; + while (true) { + if (stop.load()) { + terminate(pid, grace); + return 0; + } + const pid_t r = waitpid(pid, &status, WNOHANG); + if (r == pid) { + return decode(status); + } + if (r < 0) { + return 1; + } + sleep_poll(common::Caps::kProcessPollMs); + } + }; +} + +} // namespace molo::supervisor diff --git a/src/supervisor/supervisor/process_work.hpp b/src/supervisor/supervisor/process_work.hpp new file mode 100644 index 0000000..b673a56 --- /dev/null +++ b/src/supervisor/supervisor/process_work.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +#include "supervisor/supervisor/worker_process.hpp" + +namespace molo::supervisor { + +// An OS program to supervise (O7-2). argv[0] is resolved through PATH. +struct ProcessSpec { + std::vector argv; // 1..Caps::kProcessArgvMax entries + long stop_grace_ms = 0; // 0 means Caps::kProcessStopGraceMs +}; + +// Returns a WorkFunc that runs the program as a child process and blocks +// until it exits or stop is requested. Guarantees: +// - stop: SIGTERM to the child's process group, SIGKILL after the grace; +// the child is always reaped before the WorkFunc returns. +// - parent death (even SIGKILL): the child receives SIGTERM (Linux). +// - exit code: the child's code, 128+signal when killed by a signal, 127 +// when exec fails, 1 on invalid spec or fork failure, 0 after a +// requested stop. WorkerGroup applies restart policy and backoff. +WorkFunc make_process_work(ProcessSpec spec); + +} // namespace molo::supervisor diff --git a/tests/src/supervisor/test_process_work.cpp b/tests/src/supervisor/test_process_work.cpp new file mode 100644 index 0000000..0070ef7 --- /dev/null +++ b/tests/src/supervisor/test_process_work.cpp @@ -0,0 +1,121 @@ +// O7-2: OS-process workers: lifecycle, exit codes, restart, no orphans. +#include + +#include +#include +#include +#include +#include +#include + +#include "supervisor/common/clock.hpp" +#include "supervisor/common/observer.hpp" +#include "supervisor/supervisor/process_work.hpp" +#include "supervisor/supervisor/supervisor.hpp" + +namespace { + +int failures = 0; + +#define CHECK(cond, msg) \ + do { \ + if (!(cond)) { \ + std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \ + ++failures; \ + } \ + } while (0) + +using molo::supervisor::ProcessSpec; +using molo::supervisor::make_process_work; + +bool wait_for(const std::function& pred, long timeout_ms) { + const auto end = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (std::chrono::steady_clock::now() < end) { + if (pred()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return pred(); +} + +std::string read_file(const std::string& path) { + std::ifstream in(path); + std::string s((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + return s; +} + +bool pid_alive(long pid) { return pid > 0 && kill(static_cast(pid), 0) == 0; } + +void test_exit_codes() { + std::atomic stop{false}; + CHECK(make_process_work({{"sh", "-c", "exit 0"}})(stop) == 0, "clean exit"); + CHECK(make_process_work({{"sh", "-c", "exit 3"}})(stop) == 3, "exit code passthrough"); + CHECK(make_process_work({{"sh", "-c", "kill -9 $$"}})(stop) == 128 + 9, "signal exit"); + CHECK(make_process_work({{"/nonexistent/prog"}})(stop) == 127, "exec failure"); + CHECK(make_process_work({{}})(stop) == 1, "empty argv rejected"); +} + +void test_stop_kills_process_group() { + const std::string pidfile = "/tmp/molo_pw_group.pid"; + std::remove(pidfile.c_str()); + std::atomic stop{false}; + ProcessSpec spec{{"sh", "-c", "sleep 300 & echo $! > " + pidfile + "; wait"}, 500}; + int code = -1; + std::thread t([&] { code = make_process_work(spec)(stop); }); + CHECK(wait_for([&] { return !read_file(pidfile).empty(); }, 3000), "grandchild started"); + const long gpid = std::atol(read_file(pidfile).c_str()); + CHECK(pid_alive(gpid), "grandchild alive"); + stop = true; + t.join(); + CHECK(code == 0, "requested stop returns 0"); + CHECK(wait_for([&] { return !pid_alive(gpid); }, 2000), "grandchild killed with group"); +} + +void test_sigterm_ignored_gets_sigkill() { + std::atomic stop{false}; + ProcessSpec spec{{"sh", "-c", "trap '' TERM; while true; do sleep 1; done"}, 300}; + std::thread t([&] { make_process_work(spec)(stop); }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + const auto t0 = std::chrono::steady_clock::now(); + stop = true; + t.join(); + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + CHECK(ms < 3000, "stubborn child force-killed after grace"); +} + +void test_supervisor_restarts_and_shutdown_reaps() { + const std::string pidfile = "/tmp/molo_pw_sup.pid"; + std::remove(pidfile.c_str()); + molo::common::SystemClock clock; + auto obs = std::make_shared(); + long pid1 = 0; + { + molo::supervisor::Supervisor sup(clock, obs); + molo::supervisor::GroupSpec spec; + spec.name = "proc"; + spec.backoff_min_ms = 50; + spec.backoff_max_ms = 100; + sup.add_group(spec, make_process_work({{"sh", "-c", "echo $$ > " + pidfile + "; exec sleep 300"}})); + sup.start_all(); + CHECK(wait_for([&] { return !read_file(pidfile).empty(); }, 3000), "child started"); + pid1 = std::atol(read_file(pidfile).c_str()); + std::remove(pidfile.c_str()); + kill(static_cast(pid1), SIGKILL); // crash it + CHECK(wait_for([&] { return !read_file(pidfile).empty(); }, 5000), "restarted after crash"); + const long pid2 = std::atol(read_file(pidfile).c_str()); + CHECK(pid2 != pid1 && pid_alive(pid2), "new child alive"); + sup.shutdown(0); + CHECK(wait_for([&] { return !pid_alive(pid2); }, 3000), "shutdown reaps child"); + } +} + +} // namespace + +int main() { + test_exit_codes(); + test_stop_kills_process_group(); + test_sigterm_ignored_gets_sigkill(); + test_supervisor_restarts_and_shutdown_reaps(); + if (failures == 0) std::cout << "process_work: all passed" << std::endl; + return failures == 0 ? 0 : 1; +}