doos 0.1.0: user-level overlayfs containers with slirp networking
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
+392
@@ -0,0 +1,392 @@
|
||||
#include "sandbox.hpp"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <net/if.h>
|
||||
#include <sched.h>
|
||||
#include <signal.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/mount.h>
|
||||
#include <sys/prctl.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/statvfs.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "spec.hpp"
|
||||
#include "util.hpp"
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
namespace doos {
|
||||
|
||||
namespace {
|
||||
|
||||
volatile sig_atomic_t g_forwardTo = 0;
|
||||
|
||||
void forwardSignal(int sig) {
|
||||
if (g_forwardTo > 0) ::kill(g_forwardTo, sig);
|
||||
}
|
||||
|
||||
void installForwarding(pid_t target, std::initializer_list<int> sigs) {
|
||||
g_forwardTo = target;
|
||||
struct sigaction sa;
|
||||
std::memset(&sa, 0, sizeof sa);
|
||||
sa.sa_handler = forwardSignal;
|
||||
sigemptyset(&sa.sa_mask);
|
||||
for (int s : sigs) ::sigaction(s, &sa, nullptr);
|
||||
}
|
||||
|
||||
void doMount(const char* src, const std::string& target, const char* type, unsigned long flags,
|
||||
const std::string& data = "") {
|
||||
if (::mount(src, target.c_str(), type, flags, data.empty() ? nullptr : data.c_str()) != 0) {
|
||||
throw SysError(std::string("mount ") + (type ? type : "bind") + " " + target);
|
||||
}
|
||||
}
|
||||
|
||||
// Remount read-only while preserving the locked flags a userns cannot change.
|
||||
void remountReadOnly(const std::string& path) {
|
||||
doMount(path.c_str(), path, nullptr, MS_BIND | MS_REC);
|
||||
struct statvfs sv;
|
||||
if (::statvfs(path.c_str(), &sv) != 0) throw SysError("statvfs " + path);
|
||||
unsigned long flags = MS_BIND | MS_REMOUNT | MS_RDONLY;
|
||||
if (sv.f_flag & ST_NOSUID) flags |= MS_NOSUID;
|
||||
if (sv.f_flag & ST_NODEV) flags |= MS_NODEV;
|
||||
if (sv.f_flag & ST_NOEXEC) flags |= MS_NOEXEC;
|
||||
if (sv.f_flag & ST_NOATIME) flags |= MS_NOATIME;
|
||||
if (sv.f_flag & ST_NODIRATIME) flags |= MS_NODIRATIME;
|
||||
if (sv.f_flag & ST_RELATIME) flags |= MS_RELATIME;
|
||||
doMount(nullptr, path, nullptr, flags);
|
||||
}
|
||||
|
||||
void touch(const std::string& path) {
|
||||
int fd = ::open(path.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0644);
|
||||
if (fd < 0) throw SysError("create " + path);
|
||||
::close(fd);
|
||||
}
|
||||
|
||||
void bringLoopbackUp() {
|
||||
int s = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0);
|
||||
if (s < 0) throw SysError("socket");
|
||||
Fd guard(s);
|
||||
ifreq ifr;
|
||||
std::memset(&ifr, 0, sizeof ifr);
|
||||
std::strncpy(ifr.ifr_name, "lo", IFNAMSIZ - 1);
|
||||
if (::ioctl(s, SIOCGIFFLAGS, &ifr) != 0) throw SysError("SIOCGIFFLAGS lo");
|
||||
ifr.ifr_flags |= IFF_UP;
|
||||
if (::ioctl(s, SIOCSIFFLAGS, &ifr) != 0) throw SysError("SIOCSIFFLAGS lo");
|
||||
}
|
||||
|
||||
std::string sub(const fs::path& root, std::string_view hostDir) { return (root / hostDir.substr(1)).string(); }
|
||||
|
||||
void mountOverlay(const Container& c, const fs::path& root, std::string_view hostDir) {
|
||||
std::string dir(hostDir);
|
||||
std::string opts = "lowerdir=" + dir + ",upperdir=" + (c.layer(dir) / "upper").string() +
|
||||
",workdir=" + (c.layer(dir) / "work").string() + ",userxattr";
|
||||
doMount("overlay", sub(root, hostDir), "overlay", MS_NODEV | MS_NOSUID, opts);
|
||||
}
|
||||
|
||||
void bindHostDevice(const fs::path& root, const std::string& dev) {
|
||||
std::string target = (root / "dev" / dev).string();
|
||||
touch(target);
|
||||
doMount(("/dev/" + dev).c_str(), target, nullptr, MS_BIND);
|
||||
}
|
||||
|
||||
void setupRoot(const Container& c) {
|
||||
fs::path root = c.rootfs();
|
||||
doMount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE);
|
||||
doMount(root.c_str(), root.string(), nullptr, MS_BIND | MS_REC);
|
||||
|
||||
// resolv.conf lives in the /etc upperdir so it matches the network mode on every start.
|
||||
fs::path resolv = c.layer("/etc") / "upper" / "resolv.conf";
|
||||
std::error_code ec;
|
||||
fs::remove(resolv, ec);
|
||||
fsutil::writeFile(resolv, Network::resolvConf(c.net()));
|
||||
|
||||
for (auto d : spec::kOverlayDirs) {
|
||||
std::error_code e;
|
||||
if (fs::is_directory(std::string(d), e)) mountOverlay(c, root, d);
|
||||
}
|
||||
for (auto d : spec::kCompatDirs) {
|
||||
std::error_code e;
|
||||
std::string host(d);
|
||||
auto st = fs::symlink_status(host, e);
|
||||
if (e) continue;
|
||||
if (fs::is_symlink(st)) {
|
||||
fs::path target = fs::read_symlink(host);
|
||||
fs::path link = root / host.substr(1);
|
||||
fs::remove(link, e);
|
||||
fs::create_symlink(target, link);
|
||||
} else if (fs::is_directory(st)) {
|
||||
mountOverlay(c, root, d);
|
||||
}
|
||||
}
|
||||
for (auto d : spec::kPrivateDirs) doMount(c.privateDir(std::string(d)).c_str(), sub(root, d), nullptr, MS_BIND);
|
||||
|
||||
// tmpfs mounts
|
||||
doMount("tmpfs", sub(root, "/tmp"), "tmpfs", MS_NOSUID | MS_NODEV, "mode=1777,size=" + std::string(spec::kTmpSize));
|
||||
doMount("tmpfs", sub(root, "/run"), "tmpfs", MS_NOSUID | MS_NODEV, "mode=755,size=" + std::string(spec::kRunSize));
|
||||
|
||||
// /dev
|
||||
doMount("tmpfs", sub(root, "/dev"), "tmpfs", MS_NOSUID | MS_STRICTATIME, "mode=755,size=64k");
|
||||
for (auto d : spec::kDevices) bindHostDevice(root, std::string(d));
|
||||
fsutil::mkdirs(root / "dev/pts");
|
||||
fsutil::mkdirs(root / "dev/shm", 01777);
|
||||
doMount("devpts", sub(root, "/dev/pts"), "devpts", MS_NOSUID | MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620");
|
||||
doMount("tmpfs", sub(root, "/dev/shm"), "tmpfs", MS_NOSUID | MS_NODEV | MS_NOEXEC,
|
||||
"mode=1777,size=" + std::string(spec::kShmSize));
|
||||
fs::create_symlink("pts/ptmx", root / "dev/ptmx");
|
||||
fs::create_symlink("/proc/self/fd", root / "dev/fd");
|
||||
fs::create_symlink("/proc/self/fd/0", root / "dev/stdin");
|
||||
fs::create_symlink("/proc/self/fd/1", root / "dev/stdout");
|
||||
fs::create_symlink("/proc/self/fd/2", root / "dev/stderr");
|
||||
|
||||
// /proc (our own pid namespace), with the dangerous parts read-only or hidden.
|
||||
doMount("proc", sub(root, "/proc"), "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC);
|
||||
for (auto p : spec::kProcReadOnly) remountReadOnly(sub(root, p));
|
||||
for (auto p : spec::kProcMasked) doMount((root / "dev/null").c_str(), sub(root, p), nullptr, MS_BIND);
|
||||
|
||||
// Switch root, then drop the old one.
|
||||
if (::chdir(root.c_str()) != 0) throw SysError("chdir newroot");
|
||||
if (::syscall(SYS_pivot_root, ".", ".") != 0) throw SysError("pivot_root");
|
||||
if (::umount2(".", MNT_DETACH) != 0) throw SysError("umount old root");
|
||||
if (::chdir("/") != 0) throw SysError("chdir /");
|
||||
}
|
||||
|
||||
void applyLimitsAndEnv(const Container& c) {
|
||||
if (::prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) throw SysError("no_new_privs");
|
||||
rlimit core{0, 0};
|
||||
::setrlimit(RLIMIT_CORE, &core);
|
||||
rlimit nofile;
|
||||
if (::getrlimit(RLIMIT_NOFILE, &nofile) == 0) {
|
||||
if (nofile.rlim_max > spec::kMaxOpenFiles) nofile.rlim_max = spec::kMaxOpenFiles;
|
||||
if (nofile.rlim_cur > nofile.rlim_max) nofile.rlim_cur = nofile.rlim_max;
|
||||
::setrlimit(RLIMIT_NOFILE, &nofile);
|
||||
}
|
||||
const char* term = std::getenv("TERM");
|
||||
std::string termv = term && *term ? term : "xterm";
|
||||
const char* lang = std::getenv("LANG");
|
||||
std::string langv = lang ? lang : "";
|
||||
::clearenv();
|
||||
::setenv("PATH", std::string(spec::kContainerPath).c_str(), 1);
|
||||
::setenv("HOME", std::string(spec::kHome).c_str(), 1);
|
||||
::setenv("USER", "root", 1);
|
||||
::setenv("LOGNAME", "root", 1);
|
||||
::setenv("TERM", termv.c_str(), 1);
|
||||
::setenv("HOSTNAME", c.name().c_str(), 1);
|
||||
::setenv("container", "doos", 1);
|
||||
::setenv("PS1", ("(doos:" + c.name() + ") \\u@\\h:\\w\\$ ").c_str(), 1);
|
||||
if (!langv.empty()) ::setenv("LANG", langv.c_str(), 1);
|
||||
}
|
||||
|
||||
[[noreturn]] void execCommand(std::vector<std::string> cmd) {
|
||||
if (cmd.empty()) cmd = {::access("/bin/bash", X_OK) == 0 ? "/bin/bash" : "/bin/sh"};
|
||||
std::vector<char*> argv;
|
||||
for (auto& a : cmd) argv.push_back(a.data());
|
||||
argv.push_back(nullptr);
|
||||
::execvp(argv[0], argv.data());
|
||||
::dprintf(2, "doos: exec %s: %s\n", argv[0], std::strerror(errno));
|
||||
::_exit(127);
|
||||
}
|
||||
|
||||
int exitCodeOf(int status) {
|
||||
if (WIFEXITED(status)) return WEXITSTATUS(status);
|
||||
if (WIFSIGNALED(status)) return 128 + WTERMSIG(status);
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct ChildCtx {
|
||||
const Container* container;
|
||||
const std::vector<std::string>* command;
|
||||
int syncFd;
|
||||
int lockFd; // inherited copy must be closed or the flock outlives unlock()
|
||||
};
|
||||
|
||||
// Runs as pid 1 of the new pid namespace.
|
||||
int containerInit(void* arg) {
|
||||
auto* ctx = static_cast<ChildCtx*>(arg);
|
||||
try {
|
||||
::prctl(PR_SET_PDEATHSIG, SIGKILL);
|
||||
::close(ctx->lockFd);
|
||||
char go = 0;
|
||||
if (::read(ctx->syncFd, &go, 1) != 1 || go != 'G') ::_exit(125);
|
||||
::close(ctx->syncFd);
|
||||
|
||||
const Container& c = *ctx->container;
|
||||
setupRoot(c);
|
||||
if (::sethostname(c.name().c_str(), c.name().size()) != 0) throw SysError("sethostname");
|
||||
if (Network::isolated(c.net())) bringLoopbackUp();
|
||||
applyLimitsAndEnv(c);
|
||||
|
||||
pid_t worker = ::fork();
|
||||
if (worker < 0) throw SysError("fork");
|
||||
if (worker == 0) execCommand(*ctx->command);
|
||||
|
||||
// Minimal init: forward termination signals, reap everything, mirror the worker's status.
|
||||
installForwarding(worker, {SIGTERM, SIGINT, SIGHUP, SIGQUIT});
|
||||
int code = 1;
|
||||
for (;;) {
|
||||
int status = 0;
|
||||
pid_t p = ::waitpid(-1, &status, 0);
|
||||
if (p < 0) {
|
||||
if (errno == EINTR) continue;
|
||||
break;
|
||||
}
|
||||
if (p == worker) {
|
||||
code = exitCodeOf(status);
|
||||
break;
|
||||
}
|
||||
}
|
||||
::kill(-1, SIGKILL);
|
||||
::_exit(code);
|
||||
} catch (const std::exception& e) {
|
||||
::dprintf(2, "doos: %s\n", e.what());
|
||||
::_exit(125);
|
||||
}
|
||||
}
|
||||
|
||||
void writeMap(pid_t pid, const char* file, const std::string& content) {
|
||||
std::string path = "/proc/" + std::to_string(pid) + "/" + file;
|
||||
int fd = ::open(path.c_str(), O_WRONLY | O_CLOEXEC);
|
||||
if (fd < 0) throw SysError("open " + path);
|
||||
Fd guard(fd);
|
||||
if (::write(fd, content.data(), content.size()) != static_cast<ssize_t>(content.size())) {
|
||||
throw SysError("write " + path +
|
||||
" (user namespaces blocked? see apparmor/doos and docs/RESEARCH.md)");
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int Sandbox::run() {
|
||||
if (command_.size() > spec::kMaxCommandArgs) throw std::runtime_error("too many command arguments");
|
||||
// Decide start-or-join under the lock so concurrent attaches agree.
|
||||
FileLock lock(container_.lockFile(), spec::kLockTimeoutMs);
|
||||
if (auto pid = container_.runningPid()) {
|
||||
return join(*pid, lock);
|
||||
}
|
||||
return start(lock);
|
||||
}
|
||||
|
||||
int Sandbox::start(FileLock& lock) {
|
||||
NetMode net = container_.net();
|
||||
int flags = CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWIPC | SIGCHLD;
|
||||
if (Network::isolated(net)) flags |= CLONE_NEWNET;
|
||||
|
||||
int sync[2];
|
||||
if (::pipe2(sync, O_CLOEXEC) != 0) throw SysError("pipe");
|
||||
Fd syncRead(sync[0]), syncWrite(sync[1]);
|
||||
|
||||
std::vector<char> stack(1 << 20);
|
||||
ChildCtx ctx{&container_, &command_, syncRead.get(), lock.fd()};
|
||||
pid_t child = ::clone(containerInit, stack.data() + stack.size(), flags, &ctx);
|
||||
if (child < 0) throw SysError("clone (user namespaces available?)");
|
||||
|
||||
// From here on the child is ours: make sure it never outlives a failed setup.
|
||||
struct Reaper {
|
||||
pid_t pid;
|
||||
bool armed = true;
|
||||
~Reaper() {
|
||||
if (armed) {
|
||||
::kill(pid, SIGKILL);
|
||||
::waitpid(pid, nullptr, 0);
|
||||
}
|
||||
}
|
||||
} reaper{child};
|
||||
|
||||
writeMap(child, "setgroups", "deny");
|
||||
writeMap(child, "uid_map", "0 " + std::to_string(::geteuid()) + " 1\n");
|
||||
writeMap(child, "gid_map", "0 " + std::to_string(::getegid()) + " 1\n");
|
||||
|
||||
SlirpSession slirp;
|
||||
if (net == NetMode::Slirp) slirp.start(child);
|
||||
|
||||
container_.recordPid(child);
|
||||
struct PidGuard {
|
||||
const Container& c;
|
||||
~PidGuard() { c.clearPid(); }
|
||||
} pidGuard{container_};
|
||||
|
||||
char go = 'G';
|
||||
if (::write(syncWrite.get(), &go, 1) != 1) throw SysError("sync child");
|
||||
syncWrite.reset();
|
||||
syncRead.reset();
|
||||
|
||||
// The lock only protects the start decision; joiners must not wait for the container's lifetime.
|
||||
lock.unlock();
|
||||
struct sigaction ign;
|
||||
std::memset(&ign, 0, sizeof ign);
|
||||
ign.sa_handler = SIG_IGN;
|
||||
::sigaction(SIGINT, &ign, nullptr);
|
||||
::sigaction(SIGQUIT, &ign, nullptr);
|
||||
installForwarding(child, {SIGTERM, SIGHUP});
|
||||
|
||||
int status = 0;
|
||||
while (::waitpid(child, &status, 0) < 0) {
|
||||
if (errno != EINTR) throw SysError("waitpid");
|
||||
}
|
||||
reaper.armed = false;
|
||||
return exitCodeOf(status);
|
||||
}
|
||||
|
||||
int Sandbox::join(pid_t initPid, FileLock& lock) {
|
||||
std::string base = "/proc/" + std::to_string(initPid);
|
||||
// Order matters: user first (grants the capabilities to join the rest).
|
||||
const char* names[] = {"user", "mnt", "uts", "ipc", "net", "pid"};
|
||||
const int kinds[] = {CLONE_NEWUSER, CLONE_NEWNS, CLONE_NEWUTS, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWPID};
|
||||
Fd fds[6];
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
if (container_.net() == NetMode::Host && kinds[i] == CLONE_NEWNET) continue;
|
||||
fds[i].reset(::open((base + "/ns/" + names[i]).c_str(), O_RDONLY | O_CLOEXEC));
|
||||
if (!fds[i].valid()) throw SysError(std::string("open ns/") + names[i]);
|
||||
}
|
||||
Fd rootFd(::open((base + "/root").c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
|
||||
if (!rootFd.valid()) throw SysError("open container root");
|
||||
|
||||
lock.unlock();
|
||||
pid_t child = ::fork();
|
||||
if (child < 0) throw SysError("fork");
|
||||
if (child == 0) {
|
||||
try {
|
||||
for (int i = 0; i < 6; ++i) {
|
||||
if (fds[i].valid() && ::setns(fds[i].get(), kinds[i]) != 0) {
|
||||
throw SysError(std::string("setns ") + names[i]);
|
||||
}
|
||||
}
|
||||
if (::fchdir(rootFd.get()) != 0 || ::chroot(".") != 0) throw SysError("chroot into container");
|
||||
if (::chdir(std::string(spec::kHome).c_str()) != 0) throw SysError("chdir home");
|
||||
// setns(pid) only affects children, so fork once more.
|
||||
pid_t inner = ::fork();
|
||||
if (inner < 0) throw SysError("fork");
|
||||
if (inner == 0) {
|
||||
applyLimitsAndEnv(container_);
|
||||
execCommand(command_);
|
||||
}
|
||||
int status = 0;
|
||||
installForwarding(inner, {SIGTERM, SIGHUP});
|
||||
while (::waitpid(inner, &status, 0) < 0) {
|
||||
if (errno != EINTR) break;
|
||||
}
|
||||
::_exit(exitCodeOf(status));
|
||||
} catch (const std::exception& e) {
|
||||
::dprintf(2, "doos: %s\n", e.what());
|
||||
::_exit(125);
|
||||
}
|
||||
}
|
||||
struct sigaction ign;
|
||||
std::memset(&ign, 0, sizeof ign);
|
||||
ign.sa_handler = SIG_IGN;
|
||||
::sigaction(SIGINT, &ign, nullptr);
|
||||
::sigaction(SIGQUIT, &ign, nullptr);
|
||||
installForwarding(child, {SIGTERM, SIGHUP});
|
||||
int status = 0;
|
||||
while (::waitpid(child, &status, 0) < 0) {
|
||||
if (errno != EINTR) throw SysError("waitpid");
|
||||
}
|
||||
return exitCodeOf(status);
|
||||
}
|
||||
|
||||
} // namespace doos
|
||||
Reference in New Issue
Block a user