feat(supervisor): process workers via make_process_work with group kill and parent-death cleanup

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-24 03:10:28 +02:00
co-authored by Claude Sonnet 5.5
parent 62fdd08da8
commit 97b4b5f574
6 changed files with 255 additions and 2 deletions
+7 -2
View File
@@ -82,8 +82,13 @@ the queue dedupes them and no duplicate run happens.
## Known v1 limits
- Thread workers only; OS-process isolation is a v2 transport behind the
same `WorkFunc` interface (O7-2).
- Workers are threads. An OS program is supervised by wrapping it in
`make_process_work(ProcessSpec)` (O7-2): the `WorkFunc` forks and execs
the program, SIGTERM then SIGKILL after a grace on stop, and the child
gets a parent-death SIGTERM so it never outlives the supervisor, even
on SIGKILL of the parent. Restarts, backoff, and intensity caps come
from the owning `WorkerGroup` like any other worker. Gap: no
per-process resource limits (rlimit/cgroup) yet.
- In-house backends only; no Redis-protocol compatibility (O7-1).
- At-least-once everywhere; no exactly-once primitive (O7-3). Handlers
must be idempotent (R7-2).
+3
View File
@@ -44,6 +44,9 @@ struct Caps {
static constexpr int kJsonMaxDepth = 64;
static constexpr int kMaxStreamsPerBus = 1024;
static constexpr int kMaxConsumersPerStream = 1024;
static constexpr std::size_t kProcessArgvMax = 256;
static constexpr long kProcessStopGraceMs = 3000; // SIGTERM, then SIGKILL
static constexpr long kProcessPollMs = 50;
};
} // namespace molo::common
+2
View File
@@ -29,6 +29,7 @@ SUP_SRCS = \
$(SUP_ROOT)/tasks/scheduler.cpp \
$(SUP_ROOT)/supervisor/restart_policy.cpp \
$(SUP_ROOT)/supervisor/worker_process.cpp \
$(SUP_ROOT)/supervisor/process_work.cpp \
$(SUP_ROOT)/supervisor/health_check.cpp \
$(SUP_ROOT)/supervisor/worker_group.cpp \
$(SUP_ROOT)/supervisor/shutdown_coordinator.cpp \
@@ -46,6 +47,7 @@ SUP_OBJS = $(SUP_SRCS:$(SUP_ROOT)/%.cpp=$(SUP_BUILD)/%.o)
SUP_TESTS = \
test_supervisor \
test_process_work \
test_tasks \
test_scheduler \
test_bus \
@@ -0,0 +1,96 @@
#include "supervisor/supervisor/process_work.hpp"
#include <signal.h>
#include <sys/prctl.h>
#include <sys/wait.h>
#include <unistd.h>
#include <chrono>
#include <thread>
#include "supervisor/common/caps.hpp"
namespace molo::supervisor {
namespace {
int decode(int status) {
if (WIFEXITED(status)) {
return WEXITSTATUS(status);
}
if (WIFSIGNALED(status)) {
return 128 + WTERMSIG(status);
}
return 1;
}
void sleep_poll(long ms) {
std::this_thread::sleep_for(std::chrono::milliseconds(ms));
}
// Terminates the child's process group and reaps it.
void terminate(pid_t pid, long grace_ms) {
kill(-pid, SIGTERM);
int status = 0;
long waited = 0;
while (waitpid(pid, &status, WNOHANG) == 0) {
if (waited >= grace_ms) {
kill(-pid, SIGKILL);
waitpid(pid, &status, 0);
return;
}
sleep_poll(common::Caps::kProcessPollMs);
waited += common::Caps::kProcessPollMs;
}
}
} // namespace
WorkFunc make_process_work(ProcessSpec spec) {
return [spec = std::move(spec)](const std::atomic<bool>& stop) -> int {
if (spec.argv.empty() || spec.argv.size() > common::Caps::kProcessArgvMax) {
return 1;
}
const long grace =
spec.stop_grace_ms > 0 ? spec.stop_grace_ms : common::Caps::kProcessStopGraceMs;
std::vector<char*> args;
for (const auto& a : spec.argv) {
args.push_back(const_cast<char*>(a.c_str()));
}
args.push_back(nullptr);
const pid_t parent = getpid();
const pid_t pid = fork();
if (pid < 0) {
return 1;
}
if (pid == 0) {
setpgid(0, 0); // own group so grandchildren die with the child
prctl(PR_SET_PDEATHSIG, SIGTERM);
if (getppid() != parent) {
_exit(1); // parent already gone
}
execvp(args[0], args.data());
_exit(127);
}
setpgid(pid, pid); // also from the parent side, avoids the exec race
int status = 0;
while (true) {
if (stop.load()) {
terminate(pid, grace);
return 0;
}
const pid_t r = waitpid(pid, &status, WNOHANG);
if (r == pid) {
return decode(status);
}
if (r < 0) {
return 1;
}
sleep_poll(common::Caps::kProcessPollMs);
}
};
}
} // namespace molo::supervisor
@@ -0,0 +1,26 @@
#pragma once
#include <string>
#include <vector>
#include "supervisor/supervisor/worker_process.hpp"
namespace molo::supervisor {
// An OS program to supervise (O7-2). argv[0] is resolved through PATH.
struct ProcessSpec {
std::vector<std::string> argv; // 1..Caps::kProcessArgvMax entries
long stop_grace_ms = 0; // 0 means Caps::kProcessStopGraceMs
};
// Returns a WorkFunc that runs the program as a child process and blocks
// until it exits or stop is requested. Guarantees:
// - stop: SIGTERM to the child's process group, SIGKILL after the grace;
// the child is always reaped before the WorkFunc returns.
// - parent death (even SIGKILL): the child receives SIGTERM (Linux).
// - exit code: the child's code, 128+signal when killed by a signal, 127
// when exec fails, 1 on invalid spec or fork failure, 0 after a
// requested stop. WorkerGroup applies restart policy and backoff.
WorkFunc make_process_work(ProcessSpec spec);
} // namespace molo::supervisor