#include "sandbox.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "cgroup.hpp" #include "hardening.hpp" #include "spec.hpp" #include "util.hpp" namespace fs = std::filesystem; namespace doos { namespace { volatile sig_atomic_t g_forwardTo = 0; volatile sig_atomic_t g_terminating = 0; void forwardSignal(int sig) { g_terminating = 1; if (g_forwardTo > 0) ::kill(g_forwardTo, sig); } void installForwarding(pid_t target, std::initializer_list sigs) { g_forwardTo = target; struct sigaction sa; std::memset(&sa, 0, sizeof sa); sa.sa_handler = forwardSignal; sigemptyset(&sa.sa_mask); for (int s : sigs) ::sigaction(s, &sa, nullptr); } void doMount(const char* src, const std::string& target, const char* type, unsigned long flags, const std::string& data = "") { if (::mount(src, target.c_str(), type, flags, data.empty() ? nullptr : data.c_str()) != 0) { throw SysError(std::string("mount ") + (type ? type : "bind") + " " + target); } } // Remount read-only while preserving the locked flags a userns cannot change. void remountReadOnly(const std::string& path) { doMount(path.c_str(), path, nullptr, MS_BIND | MS_REC); struct statvfs sv; if (::statvfs(path.c_str(), &sv) != 0) throw SysError("statvfs " + path); unsigned long flags = MS_BIND | MS_REMOUNT | MS_RDONLY; if (sv.f_flag & ST_NOSUID) flags |= MS_NOSUID; if (sv.f_flag & ST_NODEV) flags |= MS_NODEV; if (sv.f_flag & ST_NOEXEC) flags |= MS_NOEXEC; if (sv.f_flag & ST_NOATIME) flags |= MS_NOATIME; if (sv.f_flag & ST_NODIRATIME) flags |= MS_NODIRATIME; if (sv.f_flag & ST_RELATIME) flags |= MS_RELATIME; doMount(nullptr, path, nullptr, flags); } void touch(const std::string& path) { int fd = ::open(path.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0644); if (fd < 0) throw SysError("create " + path); ::close(fd); } void bringLoopbackUp() { int s = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0); if (s < 0) throw SysError("socket"); Fd guard(s); ifreq ifr; std::memset(&ifr, 0, sizeof ifr); std::strncpy(ifr.ifr_name, "lo", IFNAMSIZ - 1); if (::ioctl(s, SIOCGIFFLAGS, &ifr) != 0) throw SysError("SIOCGIFFLAGS lo"); ifr.ifr_flags |= IFF_UP; if (::ioctl(s, SIOCSIFFLAGS, &ifr) != 0) throw SysError("SIOCSIFFLAGS lo"); } std::string sub(const fs::path& root, std::string_view hostDir) { return (root / hostDir.substr(1)).string(); } void mountOverlay(const Container& c, const fs::path& root, std::string_view hostDir) { std::string dir(hostDir); std::string opts = "lowerdir=" + dir + ",upperdir=" + (c.layer(dir) / "upper").string() + ",workdir=" + (c.layer(dir) / "work").string() + ",userxattr"; doMount("overlay", sub(root, hostDir), "overlay", MS_NODEV | MS_NOSUID, opts); } void bindHostDevice(const fs::path& root, const std::string& dev) { std::string target = (root / "dev" / dev).string(); touch(target); doMount(("/dev/" + dev).c_str(), target, nullptr, MS_BIND); } // apt drops to its _apt user for downloads; with one mappable id that user does not exist. void writeAptSandboxOverride(const Container& c, const fs::path& lowerEtc) { std::error_code ec; if (c.ids() != IdMode::Single || !fs::is_directory(lowerEtc / "apt", ec)) return; fs::path dir = c.etcDir() / "apt/apt.conf.d"; fsutil::mkdirs(dir); fsutil::writeFile(dir / "99doos-sandbox", "APT::Sandbox::User \"root\";\n"); } void writeResolvConf(const Container& c) { fs::path resolv = c.etcDir() / "resolv.conf"; fsutil::mkdirs(resolv.parent_path()); std::error_code ec; fs::remove(resolv, ec); fsutil::writeFile(resolv, Network::resolvConf(c.net())); } void mountHostLayers(const Container& c, const fs::path& root) { for (auto d : spec::kOverlayDirs) { std::error_code e; if (fs::is_directory(std::string(d), e)) mountOverlay(c, root, d); } for (auto d : spec::kCompatDirs) { std::error_code e; std::string host(d); auto st = fs::symlink_status(host, e); if (e) continue; if (fs::is_symlink(st)) { fs::path target = fs::read_symlink(host); fs::path link = root / host.substr(1); fs::remove(link, e); fs::create_symlink(target, link); } else if (fs::is_directory(st)) { mountOverlay(c, root, d); } } for (auto d : spec::kPrivateDirs) doMount(c.privateDir(std::string(d)).c_str(), sub(root, d), nullptr, MS_BIND); // overlays that live inside a private dir must be mounted after it for (auto d : spec::kVarOverlayDirs) { std::error_code e; if (fs::is_directory(std::string(d), e)) mountOverlay(c, root, d); } } void mountImageRoot(const Container& c, const fs::path& root, const fs::path& imageRoot) { std::string opts = "lowerdir=" + imageRoot.string() + ",upperdir=" + (c.layer("/root") / "upper").string() + ",workdir=" + (c.layer("/root") / "work").string() + ",userxattr"; doMount("overlay", root.string(), "overlay", MS_NODEV | MS_NOSUID, opts); for (auto d : spec::kEmptyDirs) fsutil::mkdirs(sub(root, d)); } void setupRoot(const Container& c, const fs::path& imageRoot) { fs::path root = c.rootfs(); doMount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE); writeResolvConf(c); if (c.imageMode()) { writeAptSandboxOverride(c, imageRoot / "etc"); mountImageRoot(c, root, imageRoot); } else { writeAptSandboxOverride(c, "/etc"); doMount(root.c_str(), root.string(), nullptr, MS_BIND | MS_REC); mountHostLayers(c, root); } // tmpfs mounts doMount("tmpfs", sub(root, "/tmp"), "tmpfs", MS_NOSUID | MS_NODEV, "mode=1777,size=" + std::string(spec::kTmpSize)); doMount("tmpfs", sub(root, "/run"), "tmpfs", MS_NOSUID | MS_NODEV, "mode=755,size=" + std::string(spec::kRunSize)); // /dev doMount("tmpfs", sub(root, "/dev"), "tmpfs", MS_NOSUID | MS_STRICTATIME, "mode=755,size=64k"); for (auto d : spec::kDevices) bindHostDevice(root, std::string(d)); fsutil::mkdirs(root / "dev/pts"); fsutil::mkdirs(root / "dev/shm", 01777); doMount("devpts", sub(root, "/dev/pts"), "devpts", MS_NOSUID | MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620"); doMount("tmpfs", sub(root, "/dev/shm"), "tmpfs", MS_NOSUID | MS_NODEV | MS_NOEXEC, "mode=1777,size=" + std::string(spec::kShmSize)); fs::create_symlink("pts/ptmx", root / "dev/ptmx"); fs::create_symlink("/proc/self/fd", root / "dev/fd"); fs::create_symlink("/proc/self/fd/0", root / "dev/stdin"); fs::create_symlink("/proc/self/fd/1", root / "dev/stdout"); fs::create_symlink("/proc/self/fd/2", root / "dev/stderr"); // /proc (our own pid namespace), with the dangerous parts read-only or hidden. doMount("proc", sub(root, "/proc"), "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC); for (auto p : spec::kProcReadOnly) remountReadOnly(sub(root, p)); for (auto p : spec::kProcMasked) doMount((root / "dev/null").c_str(), sub(root, p), nullptr, MS_BIND); // Switch root, then drop the old one. if (::chdir(root.c_str()) != 0) throw SysError("chdir newroot"); if (::syscall(SYS_pivot_root, ".", ".") != 0) throw SysError("pivot_root"); if (::umount2(".", MNT_DETACH) != 0) throw SysError("umount old root"); if (::chdir("/") != 0) throw SysError("chdir /"); } void applyLimitsAndEnv(const Container& c) { if (::prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) throw SysError("no_new_privs"); rlimit core{0, 0}; ::setrlimit(RLIMIT_CORE, &core); // No single file can outgrow the size cap, which bounds the overshoot between two usage scans. rlimit fsize{c.sizeCap(), c.sizeCap()}; ::setrlimit(RLIMIT_FSIZE, &fsize); rlimit nofile; if (::getrlimit(RLIMIT_NOFILE, &nofile) == 0) { if (nofile.rlim_max > spec::kMaxOpenFiles) nofile.rlim_max = spec::kMaxOpenFiles; if (nofile.rlim_cur > nofile.rlim_max) nofile.rlim_cur = nofile.rlim_max; ::setrlimit(RLIMIT_NOFILE, &nofile); } if (!c.relaxed()) Hardening::apply(); const char* term = std::getenv("TERM"); std::string termv = term && *term ? term : "xterm"; const char* lang = std::getenv("LANG"); std::string langv = lang ? lang : ""; ::clearenv(); ::setenv("PATH", std::string(spec::kContainerPath).c_str(), 1); ::setenv("HOME", std::string(spec::kHome).c_str(), 1); ::setenv("USER", "root", 1); ::setenv("LOGNAME", "root", 1); ::setenv("TERM", termv.c_str(), 1); ::setenv("HOSTNAME", c.name().c_str(), 1); ::setenv("container", "doos", 1); ::setenv("PS1", ("(doos:" + c.name() + ") \\u@\\h:\\w\\$ ").c_str(), 1); // The host's locale usually does not exist inside an image; C.UTF-8 is built into glibc and musl. if (c.imageMode()) ::setenv("LANG", "C.UTF-8", 1); else if (!langv.empty()) ::setenv("LANG", langv.c_str(), 1); } [[noreturn]] void execCommand(std::vector cmd) { if (cmd.empty()) cmd = {::access("/bin/bash", X_OK) == 0 ? "/bin/bash" : "/bin/sh"}; std::vector argv; for (auto& a : cmd) argv.push_back(a.data()); argv.push_back(nullptr); ::execvp(argv[0], argv.data()); ::dprintf(2, "doos: exec %s: %s\n", argv[0], std::strerror(errno)); ::_exit(127); } int exitCodeOf(int status) { if (WIFEXITED(status)) return WEXITSTATUS(status); if (WIFSIGNALED(status)) return 128 + WTERMSIG(status); return 1; } struct ChildCtx { const Container* container; const std::vector* command; int syncFd; fs::path imageRoot; int lockFd; // inherited copy must be closed or the flock outlives unlock() }; // Runs as pid 1 of the new pid namespace. int containerInit(void* arg) { auto* ctx = static_cast(arg); try { ::prctl(PR_SET_PDEATHSIG, SIGKILL); ::close(ctx->lockFd); char go = 0; if (::read(ctx->syncFd, &go, 1) != 1 || go != 'G') ::_exit(125); ::close(ctx->syncFd); const Container& c = *ctx->container; setupRoot(c, ctx->imageRoot); if (::sethostname(c.name().c_str(), c.name().size()) != 0) throw SysError("sethostname"); if (Network::isolated(c.net())) bringLoopbackUp(); applyLimitsAndEnv(c); pid_t worker = ::fork(); if (worker < 0) throw SysError("fork"); if (worker == 0) execCommand(*ctx->command); // Minimal init: forward termination signals, reap everything, mirror the worker's status. installForwarding(worker, {SIGTERM, SIGINT, SIGHUP, SIGQUIT}); int code = 1; for (;;) { int status = 0; pid_t p = ::waitpid(-1, &status, 0); if (p < 0) { if (errno == EINTR) continue; break; } if (p == worker) { code = exitCodeOf(status); break; } } ::kill(-1, SIGKILL); ::_exit(code); } catch (const std::exception& e) { ::dprintf(2, "doos: %s\n", e.what()); auto* se = dynamic_cast(&e); std::ifstream restrictUserns("/proc/sys/kernel/apparmor_restrict_unprivileged_userns"); int restricted = 0; if (restrictUserns) restrictUserns >> restricted; if (se && (se->code() == EPERM || se->code() == EACCES) && restricted == 1) { ::dprintf(2, "doos: AppArmor restricts user namespaces for this binary. Install a profile once with:\n %s\n", Sandbox::apparmorInstallHint().c_str()); } ::_exit(125); } } } // namespace namespace { fs::path selfPath() { std::error_code ec; fs::path self = fs::read_symlink("/proc/self/exe", ec); if (ec) throw std::runtime_error("cannot resolve the path of this binary: " + ec.message()); std::string path = self.string(); if (path.find_first_of("{}[]*?\"\n,") != std::string::npos) { throw std::runtime_error("binary path contains characters AppArmor treats as globs: " + path); } return self; } // One profile per binary path, so installing doos in a second place does not replace the first profile. std::string profileName(const fs::path& self) { std::string n = "doos" + self.string(); for (auto& c : n) { if (c == '/') c = '-'; } return n; } } // namespace std::string Sandbox::apparmorProfile() { fs::path self = selfPath(); return "# Lets doos use unprivileged user namespaces where kernel.apparmor_restrict_unprivileged_userns=1.\n" "# Generated by: doos apparmor (profile is for exactly this path)\n" "abi ,\n" "include \n" "\n" "profile " + profileName(self) + " " + self.string() + " flags=(unconfined) {\n" " userns,\n" " @{exec_path} mr,\n" "}\n"; } std::string Sandbox::apparmorInstallHint() { std::string file = "/etc/apparmor.d/" + profileName(selfPath()); return "doos apparmor | sudo tee " + file + " >/dev/null && sudo apparmor_parser -r " + file; } int Sandbox::run() { if (command_.size() > spec::kMaxCommandArgs) throw std::runtime_error("too many command arguments"); // Decide start-or-join under the lock so concurrent attaches agree. FileLock lock(container_.lockFile(), spec::kLockTimeoutMs); if (auto pid = container_.runningPid()) { return join(*pid, lock); } return start(lock); } int Sandbox::start(FileLock& lock) { NetMode net = container_.net(); fs::path imageRoot; if (container_.imageMode()) { Store store = Store::fromEnvironment(); imageRoot = store.images().rootfs(container_.image()); if (!fs::is_directory(imageRoot)) { throw std::runtime_error("image '" + container_.image() + "' is missing; add it again with: doos image add"); } } int flags = CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWIPC | SIGCHLD; if (Network::isolated(net)) flags |= CLONE_NEWNET; int sync[2]; if (::pipe2(sync, O_CLOEXEC) != 0) throw SysError("pipe"); Fd syncRead(sync[0]), syncWrite(sync[1]); std::vector stack(1 << 20); ChildCtx ctx{&container_, &command_, syncRead.get(), imageRoot, lock.fd()}; pid_t child = ::clone(containerInit, stack.data() + stack.size(), flags, &ctx); if (child < 0) throw SysError("clone (user namespaces available?)"); // From here on the child is ours: make sure it never outlives a failed setup. struct Reaper { pid_t pid; bool armed = true; ~Reaper() { if (armed) { ::kill(pid, SIGKILL); ::waitpid(pid, nullptr, 0); } } } reaper{child}; IdMap::apply(child, container_.ids()); SlirpSession slirp; PastaSession pasta; if (net == NetMode::Slirp) slirp.start(child); if (net == NetMode::Pasta) pasta.start(child); Limits limits{container_.memoryCap(), container_.pidsCap(), container_.milliCpus()}; std::string cgroupWhy, notEnforced; std::unique_ptr cgroup = Cgroup::create(container_.name(), limits, cgroupWhy, notEnforced); if (cgroup) { cgroup->addMain(child); if (!notEnforced.empty()) std::fprintf(stderr, "doos: limits not enforced (controller missing): %s\n", notEnforced.c_str()); } else { std::fprintf(stderr, "doos: memory, pids and cpu limits are not enforced and the size cap only warns: %s\n", cgroupWhy.c_str()); } container_.recordPid(child); struct PidGuard { const Container& c; ~PidGuard() { c.clearPid(); } } pidGuard{container_}; char go = 'G'; if (::write(syncWrite.get(), &go, 1) != 1) throw SysError("sync child"); syncWrite.reset(); syncRead.reset(); // The lock only protects the start decision; joiners must not wait for the container's lifetime. lock.unlock(); struct sigaction ign; std::memset(&ign, 0, sizeof ign); ign.sa_handler = SIG_IGN; ::sigaction(SIGINT, &ign, nullptr); ::sigaction(SIGQUIT, &ign, nullptr); installForwarding(child, {SIGTERM, SIGHUP}); int status = watch(child, cgroup.get()); reaper.armed = false; if (cgroup && cgroup->oomKills() > 0) { std::fprintf(stderr, "doos: '%s' was killed by its memory limit (%s); raise it with --memory\n", container_.name().c_str(), formatSize(container_.memoryCap()).c_str()); } if (cgroup && cgroup->pidFailures() > 0) { std::fprintf(stderr, "doos: '%s' hit its process limit (%u) %llu times; raise it with --pids\n", container_.name().c_str(), container_.pidsCap(), static_cast(cgroup->pidFailures())); } return exitCodeOf(status); } // Waits for the container's init while enforcing the size cap: warn at kSizeWarnPercent, freeze at the cap, // thaw again once usage drops below kSizeThawPercent. Nothing is ever deleted by this loop. int Sandbox::watch(pid_t child, Cgroup* cgroup) { using Clock = std::chrono::steady_clock; long fd = ::syscall(SYS_pidfd_open, child, 0); Fd pidfd(static_cast(fd)); const std::uint64_t cap = container_.sizeCap(); bool warned = false; auto nextScan = Clock::now() + std::chrono::milliseconds(spec::kSizeCheckMinIntervalMs); for (;;) { auto left = std::chrono::duration_cast(nextScan - Clock::now()).count(); int timeout = static_cast(std::max(left, 50)); if (pidfd.valid()) { pollfd p{pidfd.get(), POLLIN, 0}; int r = ::poll(&p, 1, timeout); if (r > 0) break; if (r < 0 && errno != EINTR) break; } else { int st = 0; if (::waitpid(child, &st, WNOHANG) == child) return st; ::poll(nullptr, 0, std::min(timeout, 1000)); } if (g_terminating && cgroup && cgroup->frozen()) cgroup->freeze(false); // a frozen init cannot see SIGTERM if (Clock::now() < nextScan) continue; auto began = Clock::now(); std::size_t skipped = 0; std::uint64_t used = container_.usedBytes(skipped); // The scan may use at most 1/kSizeScanDutyFactor of the time: big trees are scanned less often. auto took = Clock::now() - began; nextScan = Clock::now() + std::max(std::chrono::milliseconds(spec::kSizeCheckMinIntervalMs), std::chrono::duration_cast(took) * spec::kSizeScanDutyFactor); const char* name = container_.name().c_str(); if (used >= cap) { if (cgroup && !cgroup->frozen()) { cgroup->freeze(true); std::fprintf(stderr, "doos: '%s' reached its size cap (%s of %s): FROZEN. Free space with 'doos attach %s -- ...'; it thaws below %d%%.\n", name, formatSize(used).c_str(), formatSize(cap).c_str(), name, spec::kSizeThawPercent); } else if (!cgroup && !warned) { std::fprintf(stderr, "doos: '%s' is over its size cap (%s of %s), cannot freeze\n", name, formatSize(used).c_str(), formatSize(cap).c_str()); warned = true; } } else if (used * 100 >= cap * spec::kSizeWarnPercent) { if (!warned) { std::fprintf(stderr, "doos: '%s' uses %s of its %s size cap\n", name, formatSize(used).c_str(), formatSize(cap).c_str()); warned = true; } } else { warned = false; } if (cgroup && cgroup->frozen() && used * 100 < cap * spec::kSizeThawPercent) { cgroup->freeze(false); std::fprintf(stderr, "doos: '%s' thawed (%s used)\n", name, formatSize(used).c_str()); } } int status = 0; while (::waitpid(child, &status, 0) < 0) { if (errno != EINTR) throw SysError("waitpid"); } return status; } int Sandbox::join(pid_t initPid, FileLock& lock) { std::string base = "/proc/" + std::to_string(initPid); // Order matters: user first (grants the capabilities to join the rest). const char* names[] = {"user", "mnt", "uts", "ipc", "net", "pid"}; const int kinds[] = {CLONE_NEWUSER, CLONE_NEWNS, CLONE_NEWUTS, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWPID}; Fd fds[6]; for (int i = 0; i < 6; ++i) { if (container_.net() == NetMode::Host && kinds[i] == CLONE_NEWNET) continue; fds[i].reset(::open((base + "/ns/" + names[i]).c_str(), O_RDONLY | O_CLOEXEC)); if (!fds[i].valid()) throw SysError(std::string("open ns/") + names[i]); } Fd rootFd(::open((base + "/root").c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)); if (!rootFd.valid()) throw SysError("open container root"); lock.unlock(); // Later attaches live in the `shell` leaf: same limits as the container, but never frozen. std::unique_ptr cgroup = Cgroup::open(container_.name()); pid_t child = ::fork(); if (child < 0) throw SysError("fork"); if (child == 0) { try { if (cgroup) { try { cgroup->addShell(::getpid()); } catch (const std::exception&) { /* limits only; keep going */ } } for (int i = 0; i < 6; ++i) { if (fds[i].valid() && ::setns(fds[i].get(), kinds[i]) != 0) { throw SysError(std::string("setns ") + names[i]); } } if (::fchdir(rootFd.get()) != 0 || ::chroot(".") != 0) throw SysError("chroot into container"); if (::chdir(std::string(spec::kHome).c_str()) != 0) throw SysError("chdir home"); // setns(pid) only affects children, so fork once more. pid_t inner = ::fork(); if (inner < 0) throw SysError("fork"); if (inner == 0) { applyLimitsAndEnv(container_); execCommand(command_); } int status = 0; installForwarding(inner, {SIGTERM, SIGHUP}); while (::waitpid(inner, &status, 0) < 0) { if (errno != EINTR) break; } ::_exit(exitCodeOf(status)); } catch (const std::exception& e) { ::dprintf(2, "doos: %s\n", e.what()); ::_exit(125); } } struct sigaction ign; std::memset(&ign, 0, sizeof ign); ign.sa_handler = SIG_IGN; ::sigaction(SIGINT, &ign, nullptr); ::sigaction(SIGQUIT, &ign, nullptr); installForwarding(child, {SIGTERM, SIGHUP}); int status = 0; while (::waitpid(child, &status, 0) < 0) { if (errno != EINTR) throw SysError("waitpid"); } return exitCodeOf(status); } } // namespace doos