doos/src/sandbox.cpp

#include "sandbox.hpp"

#include <fcntl.h>
#include <net/if.h>
#include <sched.h>
#include <signal.h>
#include <sys/ioctl.h>
#include <sys/mount.h>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/syscall.h>
#include <sys/wait.h>
#include <unistd.h>

#include <cstring>

#include "spec.hpp"
#include "util.hpp"

namespace fs = std::filesystem;

namespace doos {

namespace {

volatile sig_atomic_t g_forwardTo = 0;

void forwardSignal(int sig) {
    if (g_forwardTo > 0) ::kill(g_forwardTo, sig);
}

void installForwarding(pid_t target, std::initializer_list<int> sigs) {
    g_forwardTo = target;
    struct sigaction sa;
    std::memset(&sa, 0, sizeof sa);
    sa.sa_handler = forwardSignal;
    sigemptyset(&sa.sa_mask);
    for (int s : sigs) ::sigaction(s, &sa, nullptr);
}

void doMount(const char* src, const std::string& target, const char* type, unsigned long flags,
             const std::string& data = "") {
    if (::mount(src, target.c_str(), type, flags, data.empty() ? nullptr : data.c_str()) != 0) {
        throw SysError(std::string("mount ") + (type ? type : "bind") + " " + target);
    }
}

// Remount read-only while preserving the locked flags a userns cannot change.
void remountReadOnly(const std::string& path) {
    doMount(path.c_str(), path, nullptr, MS_BIND | MS_REC);
    struct statvfs sv;
    if (::statvfs(path.c_str(), &sv) != 0) throw SysError("statvfs " + path);
    unsigned long flags = MS_BIND | MS_REMOUNT | MS_RDONLY;
    if (sv.f_flag & ST_NOSUID) flags |= MS_NOSUID;
    if (sv.f_flag & ST_NODEV) flags |= MS_NODEV;
    if (sv.f_flag & ST_NOEXEC) flags |= MS_NOEXEC;
    if (sv.f_flag & ST_NOATIME) flags |= MS_NOATIME;
    if (sv.f_flag & ST_NODIRATIME) flags |= MS_NODIRATIME;
    if (sv.f_flag & ST_RELATIME) flags |= MS_RELATIME;
    doMount(nullptr, path, nullptr, flags);
}

void touch(const std::string& path) {
    int fd = ::open(path.c_str(), O_CREAT | O_WRONLY | O_CLOEXEC, 0644);
    if (fd < 0) throw SysError("create " + path);
    ::close(fd);
}

void bringLoopbackUp() {
    int s = ::socket(AF_INET, SOCK_DGRAM | SOCK_CLOEXEC, 0);
    if (s < 0) throw SysError("socket");
    Fd guard(s);
    ifreq ifr;
    std::memset(&ifr, 0, sizeof ifr);
    std::strncpy(ifr.ifr_name, "lo", IFNAMSIZ - 1);
    if (::ioctl(s, SIOCGIFFLAGS, &ifr) != 0) throw SysError("SIOCGIFFLAGS lo");
    ifr.ifr_flags |= IFF_UP;
    if (::ioctl(s, SIOCSIFFLAGS, &ifr) != 0) throw SysError("SIOCSIFFLAGS lo");
}

std::string sub(const fs::path& root, std::string_view hostDir) { return (root / hostDir.substr(1)).string(); }

void mountOverlay(const Container& c, const fs::path& root, std::string_view hostDir) {
    std::string dir(hostDir);
    std::string opts = "lowerdir=" + dir + ",upperdir=" + (c.layer(dir) / "upper").string() +
                       ",workdir=" + (c.layer(dir) / "work").string() + ",userxattr";
    doMount("overlay", sub(root, hostDir), "overlay", MS_NODEV | MS_NOSUID, opts);
}

void bindHostDevice(const fs::path& root, const std::string& dev) {
    std::string target = (root / "dev" / dev).string();
    touch(target);
    doMount(("/dev/" + dev).c_str(), target, nullptr, MS_BIND);
}

void setupRoot(const Container& c) {
    fs::path root = c.rootfs();
    doMount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE);
    doMount(root.c_str(), root.string(), nullptr, MS_BIND | MS_REC);

    // resolv.conf lives in the /etc upperdir so it matches the network mode on every start.
    fs::path resolv = c.layer("/etc") / "upper" / "resolv.conf";
    std::error_code ec;
    fs::remove(resolv, ec);
    fsutil::writeFile(resolv, Network::resolvConf(c.net()));

    for (auto d : spec::kOverlayDirs) {
        std::error_code e;
        if (fs::is_directory(std::string(d), e)) mountOverlay(c, root, d);
    }
    for (auto d : spec::kCompatDirs) {
        std::error_code e;
        std::string host(d);
        auto st = fs::symlink_status(host, e);
        if (e) continue;
        if (fs::is_symlink(st)) {
            fs::path target = fs::read_symlink(host);
            fs::path link = root / host.substr(1);
            fs::remove(link, e);
            fs::create_symlink(target, link);
        } else if (fs::is_directory(st)) {
            mountOverlay(c, root, d);
        }
    }
    for (auto d : spec::kPrivateDirs) doMount(c.privateDir(std::string(d)).c_str(), sub(root, d), nullptr, MS_BIND);

    // tmpfs mounts
    doMount("tmpfs", sub(root, "/tmp"), "tmpfs", MS_NOSUID | MS_NODEV, "mode=1777,size=" + std::string(spec::kTmpSize));
    doMount("tmpfs", sub(root, "/run"), "tmpfs", MS_NOSUID | MS_NODEV, "mode=755,size=" + std::string(spec::kRunSize));

    // /dev
    doMount("tmpfs", sub(root, "/dev"), "tmpfs", MS_NOSUID | MS_STRICTATIME, "mode=755,size=64k");
    for (auto d : spec::kDevices) bindHostDevice(root, std::string(d));
    fsutil::mkdirs(root / "dev/pts");
    fsutil::mkdirs(root / "dev/shm", 01777);
    doMount("devpts", sub(root, "/dev/pts"), "devpts", MS_NOSUID | MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620");
    doMount("tmpfs", sub(root, "/dev/shm"), "tmpfs", MS_NOSUID | MS_NODEV | MS_NOEXEC,
            "mode=1777,size=" + std::string(spec::kShmSize));
    fs::create_symlink("pts/ptmx", root / "dev/ptmx");
    fs::create_symlink("/proc/self/fd", root / "dev/fd");
    fs::create_symlink("/proc/self/fd/0", root / "dev/stdin");
    fs::create_symlink("/proc/self/fd/1", root / "dev/stdout");
    fs::create_symlink("/proc/self/fd/2", root / "dev/stderr");

    // /proc (our own pid namespace), with the dangerous parts read-only or hidden.
    doMount("proc", sub(root, "/proc"), "proc", MS_NOSUID | MS_NODEV | MS_NOEXEC);
    for (auto p : spec::kProcReadOnly) remountReadOnly(sub(root, p));
    for (auto p : spec::kProcMasked) doMount((root / "dev/null").c_str(), sub(root, p), nullptr, MS_BIND);

    // Switch root, then drop the old one.
    if (::chdir(root.c_str()) != 0) throw SysError("chdir newroot");
    if (::syscall(SYS_pivot_root, ".", ".") != 0) throw SysError("pivot_root");
    if (::umount2(".", MNT_DETACH) != 0) throw SysError("umount old root");
    if (::chdir("/") != 0) throw SysError("chdir /");
}

void applyLimitsAndEnv(const Container& c) {
    if (::prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) throw SysError("no_new_privs");
    rlimit core{0, 0};
    ::setrlimit(RLIMIT_CORE, &core);
    rlimit nofile;
    if (::getrlimit(RLIMIT_NOFILE, &nofile) == 0) {
        if (nofile.rlim_max > spec::kMaxOpenFiles) nofile.rlim_max = spec::kMaxOpenFiles;
        if (nofile.rlim_cur > nofile.rlim_max) nofile.rlim_cur = nofile.rlim_max;
        ::setrlimit(RLIMIT_NOFILE, &nofile);
    }
    const char* term = std::getenv("TERM");
    std::string termv = term && *term ? term : "xterm";
    const char* lang = std::getenv("LANG");
    std::string langv = lang ? lang : "";
    ::clearenv();
    ::setenv("PATH", std::string(spec::kContainerPath).c_str(), 1);
    ::setenv("HOME", std::string(spec::kHome).c_str(), 1);
    ::setenv("USER", "root", 1);
    ::setenv("LOGNAME", "root", 1);
    ::setenv("TERM", termv.c_str(), 1);
    ::setenv("HOSTNAME", c.name().c_str(), 1);
    ::setenv("container", "doos", 1);
    ::setenv("PS1", ("(doos:" + c.name() + ") \\u@\\h:\\w\\$ ").c_str(), 1);
    if (!langv.empty()) ::setenv("LANG", langv.c_str(), 1);
}

[[noreturn]] void execCommand(std::vector<std::string> cmd) {
    if (cmd.empty()) cmd = {::access("/bin/bash", X_OK) == 0 ? "/bin/bash" : "/bin/sh"};
    std::vector<char*> argv;
    for (auto& a : cmd) argv.push_back(a.data());
    argv.push_back(nullptr);
    ::execvp(argv[0], argv.data());
    ::dprintf(2, "doos: exec %s: %s\n", argv[0], std::strerror(errno));
    ::_exit(127);
}

int exitCodeOf(int status) {
    if (WIFEXITED(status)) return WEXITSTATUS(status);
    if (WIFSIGNALED(status)) return 128 + WTERMSIG(status);
    return 1;
}

struct ChildCtx {
    const Container* container;
    const std::vector<std::string>* command;
    int syncFd;
    int lockFd;  // inherited copy must be closed or the flock outlives unlock()
};

// Runs as pid 1 of the new pid namespace.
int containerInit(void* arg) {
    auto* ctx = static_cast<ChildCtx*>(arg);
    try {
        ::prctl(PR_SET_PDEATHSIG, SIGKILL);
        ::close(ctx->lockFd);
        char go = 0;
        if (::read(ctx->syncFd, &go, 1) != 1 || go != 'G') ::_exit(125);
        ::close(ctx->syncFd);

        const Container& c = *ctx->container;
        setupRoot(c);
        if (::sethostname(c.name().c_str(), c.name().size()) != 0) throw SysError("sethostname");
        if (Network::isolated(c.net())) bringLoopbackUp();
        applyLimitsAndEnv(c);

        pid_t worker = ::fork();
        if (worker < 0) throw SysError("fork");
        if (worker == 0) execCommand(*ctx->command);

        // Minimal init: forward termination signals, reap everything, mirror the worker's status.
        installForwarding(worker, {SIGTERM, SIGINT, SIGHUP, SIGQUIT});
        int code = 1;
        for (;;) {
            int status = 0;
            pid_t p = ::waitpid(-1, &status, 0);
            if (p < 0) {
                if (errno == EINTR) continue;
                break;
            }
            if (p == worker) {
                code = exitCodeOf(status);
                break;
            }
        }
        ::kill(-1, SIGKILL);
        ::_exit(code);
    } catch (const std::exception& e) {
        ::dprintf(2, "doos: %s\n", e.what());
        ::_exit(125);
    }
}

void writeMap(pid_t pid, const char* file, const std::string& content) {
    std::string path = "/proc/" + std::to_string(pid) + "/" + file;
    int fd = ::open(path.c_str(), O_WRONLY | O_CLOEXEC);
    if (fd < 0) throw SysError("open " + path);
    Fd guard(fd);
    if (::write(fd, content.data(), content.size()) != static_cast<ssize_t>(content.size())) {
        throw SysError("write " + path +
                       " (user namespaces blocked? see apparmor/doos and docs/RESEARCH.md)");
    }
}

}  // namespace

int Sandbox::run() {
    if (command_.size() > spec::kMaxCommandArgs) throw std::runtime_error("too many command arguments");
    // Decide start-or-join under the lock so concurrent attaches agree.
    FileLock lock(container_.lockFile(), spec::kLockTimeoutMs);
    if (auto pid = container_.runningPid()) {
        return join(*pid, lock);
    }
    return start(lock);
}

int Sandbox::start(FileLock& lock) {
    NetMode net = container_.net();
    int flags = CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWPID | CLONE_NEWUTS | CLONE_NEWIPC | SIGCHLD;
    if (Network::isolated(net)) flags |= CLONE_NEWNET;

    int sync[2];
    if (::pipe2(sync, O_CLOEXEC) != 0) throw SysError("pipe");
    Fd syncRead(sync[0]), syncWrite(sync[1]);

    std::vector<char> stack(1 << 20);
    ChildCtx ctx{&container_, &command_, syncRead.get(), lock.fd()};
    pid_t child = ::clone(containerInit, stack.data() + stack.size(), flags, &ctx);
    if (child < 0) throw SysError("clone (user namespaces available?)");

    // From here on the child is ours: make sure it never outlives a failed setup.
    struct Reaper {
        pid_t pid;
        bool armed = true;
        ~Reaper() {
            if (armed) {
                ::kill(pid, SIGKILL);
                ::waitpid(pid, nullptr, 0);
            }
        }
    } reaper{child};

    writeMap(child, "setgroups", "deny");
    writeMap(child, "uid_map", "0 " + std::to_string(::geteuid()) + " 1\n");
    writeMap(child, "gid_map", "0 " + std::to_string(::getegid()) + " 1\n");

    SlirpSession slirp;
    if (net == NetMode::Slirp) slirp.start(child);

    container_.recordPid(child);
    struct PidGuard {
        const Container& c;
        ~PidGuard() { c.clearPid(); }
    } pidGuard{container_};

    char go = 'G';
    if (::write(syncWrite.get(), &go, 1) != 1) throw SysError("sync child");
    syncWrite.reset();
    syncRead.reset();

    // The lock only protects the start decision; joiners must not wait for the container's lifetime.
    lock.unlock();
    struct sigaction ign;
    std::memset(&ign, 0, sizeof ign);
    ign.sa_handler = SIG_IGN;
    ::sigaction(SIGINT, &ign, nullptr);
    ::sigaction(SIGQUIT, &ign, nullptr);
    installForwarding(child, {SIGTERM, SIGHUP});

    int status = 0;
    while (::waitpid(child, &status, 0) < 0) {
        if (errno != EINTR) throw SysError("waitpid");
    }
    reaper.armed = false;
    return exitCodeOf(status);
}

int Sandbox::join(pid_t initPid, FileLock& lock) {
    std::string base = "/proc/" + std::to_string(initPid);
    // Order matters: user first (grants the capabilities to join the rest).
    const char* names[] = {"user", "mnt", "uts", "ipc", "net", "pid"};
    const int kinds[] = {CLONE_NEWUSER, CLONE_NEWNS, CLONE_NEWUTS, CLONE_NEWIPC, CLONE_NEWNET, CLONE_NEWPID};
    Fd fds[6];
    for (int i = 0; i < 6; ++i) {
        if (container_.net() == NetMode::Host && kinds[i] == CLONE_NEWNET) continue;
        fds[i].reset(::open((base + "/ns/" + names[i]).c_str(), O_RDONLY | O_CLOEXEC));
        if (!fds[i].valid()) throw SysError(std::string("open ns/") + names[i]);
    }
    Fd rootFd(::open((base + "/root").c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
    if (!rootFd.valid()) throw SysError("open container root");

    lock.unlock();
    pid_t child = ::fork();
    if (child < 0) throw SysError("fork");
    if (child == 0) {
        try {
            for (int i = 0; i < 6; ++i) {
                if (fds[i].valid() && ::setns(fds[i].get(), kinds[i]) != 0) {
                    throw SysError(std::string("setns ") + names[i]);
                }
            }
            if (::fchdir(rootFd.get()) != 0 || ::chroot(".") != 0) throw SysError("chroot into container");
            if (::chdir(std::string(spec::kHome).c_str()) != 0) throw SysError("chdir home");
            // setns(pid) only affects children, so fork once more.
            pid_t inner = ::fork();
            if (inner < 0) throw SysError("fork");
            if (inner == 0) {
                applyLimitsAndEnv(container_);
                execCommand(command_);
            }
            int status = 0;
            installForwarding(inner, {SIGTERM, SIGHUP});
            while (::waitpid(inner, &status, 0) < 0) {
                if (errno != EINTR) break;
            }
            ::_exit(exitCodeOf(status));
        } catch (const std::exception& e) {
            ::dprintf(2, "doos: %s\n", e.what());
            ::_exit(125);
        }
    }
    struct sigaction ign;
    std::memset(&ign, 0, sizeof ign);
    ign.sa_handler = SIG_IGN;
    ::sigaction(SIGINT, &ign, nullptr);
    ::sigaction(SIGQUIT, &ign, nullptr);
    installForwarding(child, {SIGTERM, SIGHUP});
    int status = 0;
    while (::waitpid(child, &status, 0) < 0) {
        if (errno != EINTR) throw SysError("waitpid");
    }
    return exitCodeOf(status);
}

}  // namespace doos