feat(supervisor): process workers via make_process_work with group kill and parent-death cleanup
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -82,8 +82,13 @@ the queue dedupes them and no duplicate run happens.
|
||||
|
||||
## Known v1 limits
|
||||
|
||||
- Thread workers only; OS-process isolation is a v2 transport behind the
|
||||
same `WorkFunc` interface (O7-2).
|
||||
- Workers are threads. An OS program is supervised by wrapping it in
|
||||
`make_process_work(ProcessSpec)` (O7-2): the `WorkFunc` forks and execs
|
||||
the program, SIGTERM then SIGKILL after a grace on stop, and the child
|
||||
gets a parent-death SIGTERM so it never outlives the supervisor, even
|
||||
on SIGKILL of the parent. Restarts, backoff, and intensity caps come
|
||||
from the owning `WorkerGroup` like any other worker. Gap: no
|
||||
per-process resource limits (rlimit/cgroup) yet.
|
||||
- In-house backends only; no Redis-protocol compatibility (O7-1).
|
||||
- At-least-once everywhere; no exactly-once primitive (O7-3). Handlers
|
||||
must be idempotent (R7-2).
|
||||
|
||||
@@ -44,6 +44,9 @@ struct Caps {
|
||||
static constexpr int kJsonMaxDepth = 64;
|
||||
static constexpr int kMaxStreamsPerBus = 1024;
|
||||
static constexpr int kMaxConsumersPerStream = 1024;
|
||||
static constexpr std::size_t kProcessArgvMax = 256;
|
||||
static constexpr long kProcessStopGraceMs = 3000; // SIGTERM, then SIGKILL
|
||||
static constexpr long kProcessPollMs = 50;
|
||||
};
|
||||
|
||||
} // namespace molo::common
|
||||
|
||||
@@ -29,6 +29,7 @@ SUP_SRCS = \
|
||||
$(SUP_ROOT)/tasks/scheduler.cpp \
|
||||
$(SUP_ROOT)/supervisor/restart_policy.cpp \
|
||||
$(SUP_ROOT)/supervisor/worker_process.cpp \
|
||||
$(SUP_ROOT)/supervisor/process_work.cpp \
|
||||
$(SUP_ROOT)/supervisor/health_check.cpp \
|
||||
$(SUP_ROOT)/supervisor/worker_group.cpp \
|
||||
$(SUP_ROOT)/supervisor/shutdown_coordinator.cpp \
|
||||
@@ -46,6 +47,7 @@ SUP_OBJS = $(SUP_SRCS:$(SUP_ROOT)/%.cpp=$(SUP_BUILD)/%.o)
|
||||
|
||||
SUP_TESTS = \
|
||||
test_supervisor \
|
||||
test_process_work \
|
||||
test_tasks \
|
||||
test_scheduler \
|
||||
test_bus \
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#include "supervisor/supervisor/process_work.hpp"
|
||||
|
||||
#include <signal.h>
|
||||
#include <sys/prctl.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
#include "supervisor/common/caps.hpp"
|
||||
|
||||
namespace molo::supervisor {
|
||||
|
||||
namespace {
|
||||
|
||||
int decode(int status) {
|
||||
if (WIFEXITED(status)) {
|
||||
return WEXITSTATUS(status);
|
||||
}
|
||||
if (WIFSIGNALED(status)) {
|
||||
return 128 + WTERMSIG(status);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
void sleep_poll(long ms) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(ms));
|
||||
}
|
||||
|
||||
// Terminates the child's process group and reaps it.
|
||||
void terminate(pid_t pid, long grace_ms) {
|
||||
kill(-pid, SIGTERM);
|
||||
int status = 0;
|
||||
long waited = 0;
|
||||
while (waitpid(pid, &status, WNOHANG) == 0) {
|
||||
if (waited >= grace_ms) {
|
||||
kill(-pid, SIGKILL);
|
||||
waitpid(pid, &status, 0);
|
||||
return;
|
||||
}
|
||||
sleep_poll(common::Caps::kProcessPollMs);
|
||||
waited += common::Caps::kProcessPollMs;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
WorkFunc make_process_work(ProcessSpec spec) {
|
||||
return [spec = std::move(spec)](const std::atomic<bool>& stop) -> int {
|
||||
if (spec.argv.empty() || spec.argv.size() > common::Caps::kProcessArgvMax) {
|
||||
return 1;
|
||||
}
|
||||
const long grace =
|
||||
spec.stop_grace_ms > 0 ? spec.stop_grace_ms : common::Caps::kProcessStopGraceMs;
|
||||
std::vector<char*> args;
|
||||
for (const auto& a : spec.argv) {
|
||||
args.push_back(const_cast<char*>(a.c_str()));
|
||||
}
|
||||
args.push_back(nullptr);
|
||||
|
||||
const pid_t parent = getpid();
|
||||
const pid_t pid = fork();
|
||||
if (pid < 0) {
|
||||
return 1;
|
||||
}
|
||||
if (pid == 0) {
|
||||
setpgid(0, 0); // own group so grandchildren die with the child
|
||||
prctl(PR_SET_PDEATHSIG, SIGTERM);
|
||||
if (getppid() != parent) {
|
||||
_exit(1); // parent already gone
|
||||
}
|
||||
execvp(args[0], args.data());
|
||||
_exit(127);
|
||||
}
|
||||
setpgid(pid, pid); // also from the parent side, avoids the exec race
|
||||
|
||||
int status = 0;
|
||||
while (true) {
|
||||
if (stop.load()) {
|
||||
terminate(pid, grace);
|
||||
return 0;
|
||||
}
|
||||
const pid_t r = waitpid(pid, &status, WNOHANG);
|
||||
if (r == pid) {
|
||||
return decode(status);
|
||||
}
|
||||
if (r < 0) {
|
||||
return 1;
|
||||
}
|
||||
sleep_poll(common::Caps::kProcessPollMs);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace molo::supervisor
|
||||
@@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "supervisor/supervisor/worker_process.hpp"
|
||||
|
||||
namespace molo::supervisor {
|
||||
|
||||
// An OS program to supervise (O7-2). argv[0] is resolved through PATH.
|
||||
struct ProcessSpec {
|
||||
std::vector<std::string> argv; // 1..Caps::kProcessArgvMax entries
|
||||
long stop_grace_ms = 0; // 0 means Caps::kProcessStopGraceMs
|
||||
};
|
||||
|
||||
// Returns a WorkFunc that runs the program as a child process and blocks
|
||||
// until it exits or stop is requested. Guarantees:
|
||||
// - stop: SIGTERM to the child's process group, SIGKILL after the grace;
|
||||
// the child is always reaped before the WorkFunc returns.
|
||||
// - parent death (even SIGKILL): the child receives SIGTERM (Linux).
|
||||
// - exit code: the child's code, 128+signal when killed by a signal, 127
|
||||
// when exec fails, 1 on invalid spec or fork failure, 0 after a
|
||||
// requested stop. WorkerGroup applies restart policy and backoff.
|
||||
WorkFunc make_process_work(ProcessSpec spec);
|
||||
|
||||
} // namespace molo::supervisor
|
||||
@@ -0,0 +1,121 @@
|
||||
// O7-2: OS-process workers: lifecycle, exit codes, restart, no orphans.
|
||||
#include <signal.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <thread>
|
||||
|
||||
#include "supervisor/common/clock.hpp"
|
||||
#include "supervisor/common/observer.hpp"
|
||||
#include "supervisor/supervisor/process_work.hpp"
|
||||
#include "supervisor/supervisor/supervisor.hpp"
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
#define CHECK(cond, msg) \
|
||||
do { \
|
||||
if (!(cond)) { \
|
||||
std::cout << "FAIL " << __LINE__ << ": " << (msg) << std::endl; \
|
||||
++failures; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
using molo::supervisor::ProcessSpec;
|
||||
using molo::supervisor::make_process_work;
|
||||
|
||||
bool wait_for(const std::function<bool()>& pred, long timeout_ms) {
|
||||
const auto end = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms);
|
||||
while (std::chrono::steady_clock::now() < end) {
|
||||
if (pred()) return true;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
}
|
||||
return pred();
|
||||
}
|
||||
|
||||
std::string read_file(const std::string& path) {
|
||||
std::ifstream in(path);
|
||||
std::string s((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
|
||||
return s;
|
||||
}
|
||||
|
||||
bool pid_alive(long pid) { return pid > 0 && kill(static_cast<pid_t>(pid), 0) == 0; }
|
||||
|
||||
void test_exit_codes() {
|
||||
std::atomic<bool> stop{false};
|
||||
CHECK(make_process_work({{"sh", "-c", "exit 0"}})(stop) == 0, "clean exit");
|
||||
CHECK(make_process_work({{"sh", "-c", "exit 3"}})(stop) == 3, "exit code passthrough");
|
||||
CHECK(make_process_work({{"sh", "-c", "kill -9 $$"}})(stop) == 128 + 9, "signal exit");
|
||||
CHECK(make_process_work({{"/nonexistent/prog"}})(stop) == 127, "exec failure");
|
||||
CHECK(make_process_work({{}})(stop) == 1, "empty argv rejected");
|
||||
}
|
||||
|
||||
void test_stop_kills_process_group() {
|
||||
const std::string pidfile = "/tmp/molo_pw_group.pid";
|
||||
std::remove(pidfile.c_str());
|
||||
std::atomic<bool> stop{false};
|
||||
ProcessSpec spec{{"sh", "-c", "sleep 300 & echo $! > " + pidfile + "; wait"}, 500};
|
||||
int code = -1;
|
||||
std::thread t([&] { code = make_process_work(spec)(stop); });
|
||||
CHECK(wait_for([&] { return !read_file(pidfile).empty(); }, 3000), "grandchild started");
|
||||
const long gpid = std::atol(read_file(pidfile).c_str());
|
||||
CHECK(pid_alive(gpid), "grandchild alive");
|
||||
stop = true;
|
||||
t.join();
|
||||
CHECK(code == 0, "requested stop returns 0");
|
||||
CHECK(wait_for([&] { return !pid_alive(gpid); }, 2000), "grandchild killed with group");
|
||||
}
|
||||
|
||||
void test_sigterm_ignored_gets_sigkill() {
|
||||
std::atomic<bool> stop{false};
|
||||
ProcessSpec spec{{"sh", "-c", "trap '' TERM; while true; do sleep 1; done"}, 300};
|
||||
std::thread t([&] { make_process_work(spec)(stop); });
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
stop = true;
|
||||
t.join();
|
||||
const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - t0).count();
|
||||
CHECK(ms < 3000, "stubborn child force-killed after grace");
|
||||
}
|
||||
|
||||
void test_supervisor_restarts_and_shutdown_reaps() {
|
||||
const std::string pidfile = "/tmp/molo_pw_sup.pid";
|
||||
std::remove(pidfile.c_str());
|
||||
molo::common::SystemClock clock;
|
||||
auto obs = std::make_shared<molo::common::NullObserver>();
|
||||
long pid1 = 0;
|
||||
{
|
||||
molo::supervisor::Supervisor sup(clock, obs);
|
||||
molo::supervisor::GroupSpec spec;
|
||||
spec.name = "proc";
|
||||
spec.backoff_min_ms = 50;
|
||||
spec.backoff_max_ms = 100;
|
||||
sup.add_group(spec, make_process_work({{"sh", "-c", "echo $$ > " + pidfile + "; exec sleep 300"}}));
|
||||
sup.start_all();
|
||||
CHECK(wait_for([&] { return !read_file(pidfile).empty(); }, 3000), "child started");
|
||||
pid1 = std::atol(read_file(pidfile).c_str());
|
||||
std::remove(pidfile.c_str());
|
||||
kill(static_cast<pid_t>(pid1), SIGKILL); // crash it
|
||||
CHECK(wait_for([&] { return !read_file(pidfile).empty(); }, 5000), "restarted after crash");
|
||||
const long pid2 = std::atol(read_file(pidfile).c_str());
|
||||
CHECK(pid2 != pid1 && pid_alive(pid2), "new child alive");
|
||||
sup.shutdown(0);
|
||||
CHECK(wait_for([&] { return !pid_alive(pid2); }, 3000), "shutdown reaps child");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
test_exit_codes();
|
||||
test_stop_kills_process_group();
|
||||
test_sigterm_ignored_gets_sigkill();
|
||||
test_supervisor_restarts_and_shutdown_reaps();
|
||||
if (failures == 0) std::cout << "process_work: all passed" << std::endl;
|
||||
return failures == 0 ? 0 : 1;
|
||||
}
|
||||
Reference in New Issue
Block a user