From 836573a9a2d38935e254702826d250ea39824a1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Oct 2017 12:22:29 +0100 Subject: [PATCH 01/87] Dynamically allocate UIDs Rather than rely on a nixbld group, we now allocate UIDs/GIDs dynamically starting at a configurable ID (872415232 by default). Also, we allocate 2^18 UIDs and GIDs per build, and run the build as root in its UID namespace. (This should not be the default since it breaks some builds. We probably should enable this conditional on a requiredSystemFeature.) The goal is to be able to run (NixOS) containers in a build. However, this will also require some cgroup initialisation. The 2^18 UIDs/GIDs is intended to provide enough ID space to run multiple containers per build, e.g. for distributed NixOS tests. --- src/libstore/build.cc | 59 +++++++++++++++++++++++++++++++++++------ src/libstore/globals.hh | 10 +++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 347fe1b99..c853f609d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -513,7 +513,6 @@ private: AutoCloseFD fdUserLock; bool isEnabled = false; - string user; uid_t uid = 0; gid_t gid = 0; std::vector supplementaryGIDs; @@ -523,9 +522,9 @@ public: void kill(); - string getUser() { return user; } uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } + gid_t getGID() { assert(gid); return gid; } + uint32_t getIDCount() { return 1; } std::vector getSupplementaryGIDs() { return supplementaryGIDs; } bool findFreeUser(); @@ -537,13 +536,16 @@ public: UserLock::UserLock() { +#if 0 assert(settings.buildUsersGroup != ""); createDirs(settings.nixStateDir + "/userpool"); +#endif } bool UserLock::findFreeUser() { if (enabled()) return true; +#if 0 /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) @@ -607,12 +609,46 @@ bool UserLock::findFreeUser() { } } + return false; +#endif + + assert(settings.startId > 0); + assert(settings.startId % settings.idsPerBuild == 0); + assert(settings.uidCount % settings.idsPerBuild == 0); + assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + + // FIXME: check whether the id range overlaps any known users + + size_t nrSlots = settings.uidCount / settings.idsPerBuild; + + for (size_t i = 0; i < nrSlots; i++) { + debug("trying user slot '%d'", i); + + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%1%'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + uid = settings.startId + i * settings.idsPerBuild; + gid = settings.startId + i * settings.idsPerBuild; + return true; + } + } + return false; } void UserLock::kill() { + // FIXME: use a cgroup to kill all processes in the build? +#if 0 killUser(uid); +#endif } @@ -1523,7 +1559,7 @@ void DerivationGoal::tryLocalBuild() { /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { + if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) { #if defined(__linux__) || defined(__APPLE__) if (!buildUser) buildUser = std::make_unique(); @@ -2129,7 +2165,7 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - if (mkdir(chrootRootDir.c_str(), 0750) == -1) + if (mkdir(chrootRootDir.c_str(), 0755) == -1) throw SysError("cannot create '%1%'", chrootRootDir); if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) @@ -2444,14 +2480,15 @@ void DerivationGoal::startBuilder() the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + uint32_t nrIds = settings.idsPerBuild; // FIXME writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); + fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds)); - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds)); /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ @@ -3306,10 +3343,16 @@ void DerivationGoal::runChild() /* Switch to the sandbox uid/gid in the user namespace, which corresponds to the build user or calling user in the parent namespace. */ +#if 0 if (setgid(sandboxGid) == -1) throw SysError("setgid failed"); if (setuid(sandboxUid) == -1) throw SysError("setuid failed"); +#endif + if (setgid(0) == -1) + throw SysError("setgid failed"); + if (setuid(0) == -1) + throw SysError("setuid failed"); setUser = false; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 58cf08763..7dc842bca 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -148,6 +148,16 @@ public: Setting buildUsersGroup{this, "", "build-users-group", "The Unix group that contains the build users."}; + #if __linux__ + const uint32_t idsPerBuild = 1 << 18; + + Setting startId{this, 872415232, "start-id", + "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"}; + + Setting uidCount{this, idsPerBuild * 128, "id-count", + "The number of UIDs/GIDs to use for dynamic ID allocation."}; + #endif + Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; From c3e0a68c7eeeb4f491c0464392b2146ddec4305a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 May 2020 13:52:41 +0200 Subject: [PATCH 02/87] canonicalisePathMetaData(): Support a UID range --- src/libstore/build.cc | 12 +++++++++--- src/libstore/local-store.cc | 27 +++++++++++++++++---------- src/libstore/local-store.hh | 15 ++++++++++++--- src/nix-store/nix-store.cc | 2 +- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c853f609d..4e654e8ad 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -524,7 +524,7 @@ public: uid_t getUID() { assert(uid); return uid; } gid_t getGID() { assert(gid); return gid; } - uint32_t getIDCount() { return 1; } + uint32_t getIDCount() { return settings.idsPerBuild; } std::vector getSupplementaryGIDs() { return supplementaryGIDs; } bool findFreeUser(); @@ -3744,7 +3744,10 @@ void DerivationGoal::registerOutputs() /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) : std::nullopt, + inodesSeen); /* FIXME: this is in-memory. */ StringSink sink; @@ -3819,7 +3822,10 @@ void DerivationGoal::registerOutputs() /* Get rid of all weird permissions. This also checks that all files are owned by the build user, if applicable. */ canonicalisePathMetaData(actualPath, - buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); + buildUser && !rewritten + ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) + : std::nullopt, + inodesSeen); /* For this output path, find the references to other paths contained in it. Compute the SHA-256 NAR hash at the same diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eed225349..80ebe903f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -424,7 +424,10 @@ void canonicaliseTimestampAndPermissions(const Path & path) } -static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) { checkInterrupt(); @@ -475,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe However, ignore files that we chown'ed ourselves previously to ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) throw BuildError("invalid ownership on file '%1%'", path); @@ -509,14 +512,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (S_ISDIR(st.st_mode)) { DirEntries entries = readDirectory(path); for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); } } -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) { - canonicalisePathMetaData_(path, fromUid, inodesSeen); + canonicalisePathMetaData_(path, uidRange, inodesSeen); /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ @@ -531,10 +537,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino } -void canonicalisePathMetaData(const Path & path, uid_t fromUid) +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) { InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); + canonicalisePathMetaData(path, uidRange, inodesSeen); } @@ -1021,7 +1028,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, autoGC(); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); optimisePath(realPath); // FIXME: combine with hashPath() @@ -1064,7 +1071,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } else writeFile(realPath, dump); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); /* Register the SHA-256 hash of the NAR serialisation of the path in the database. We may just have computed it @@ -1134,7 +1141,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, writeFile(realPath, s); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); StringSink sink; dumpString(s, sink); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ff36cb00e..79b415875 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -311,9 +311,18 @@ typedef set InodesSeen; - the permissions are set of 444 or 555 (i.e., read-only with or without execute permission; setuid bits etc. are cleared) - the owner and group are set to the Nix user and group, if we're - running as root. */ -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); -void canonicalisePathMetaData(const Path & path, uid_t fromUid); + running as root. + If uidRange is not empty, this function will throw an error if it + encounters files owned by a user outside of the closed interval + [uidRange->first, uidRange->second]. +*/ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); void canonicaliseTimestampAndPermissions(const Path & path); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..b948380bb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -500,7 +500,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) - canonicalisePathMetaData(store->printStorePath(info->path), -1); + canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); info->narHash = hash.first; From f5fa3de759a2b4c1d0107a4304a0b3f9571c87b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 May 2020 00:11:59 +0200 Subject: [PATCH 03/87] Run builds in their own cgroup Also, run builds in a cgroup namespace (ensuring /proc/self/cgroup doesn't leak information about the outside world) and mount /sys. This enables running systemd-nspawn and thus NixOS containers in a Nix build. --- src/libstore/build.cc | 66 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4e654e8ad..816d695a5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2168,7 +2168,8 @@ void DerivationGoal::startBuilder() if (mkdir(chrootRootDir.c_str(), 0755) == -1) throw SysError("cannot create '%1%'", chrootRootDir); - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + // FIXME: only make root writable for user namespace builds. + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need @@ -2182,6 +2183,7 @@ void DerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); + chownToBuilder(chrootRootDir + "/etc"); writeFile(chrootRootDir + "/etc/passwd", fmt( "root:x:0:0:Nix build user:%3%:/noshell\n" @@ -2372,6 +2374,52 @@ void DerivationGoal::startBuilder() #if __linux__ if (useChroot) { + /* Create a cgroup. */ + // FIXME: do we want to use the parent cgroup? We should + // always use the same cgroup regardless of whether we're the + // daemon or run from a user session via sudo. + std::string msg; + std::vector cgroups; + for (auto & line : tokenizeString>(readFile("/proc/self/cgroup"), "\n")) { + static std::regex regex("([0-9]+):([^:]*):(.*)"); + std::smatch match; + if (!std::regex_match(line, match, regex)) + throw Error("invalid line '%s' in '/proc/self/cgroup'", line); + + /* We only create a systemd cgroup, since that's enough + for running systemd-nspawn. */ + std::string name; + if (match[2] == "name=systemd") + name = "systemd"; + //else if (match[2] == "") + // name = "unified"; + else continue; + + std::string cgroup = match[3]; + + auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup); + + if (!pathExists(hostCgroup)) + throw Error("expected unified cgroup directory '%s'", hostCgroup); + + auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); + + // FIXME: if the cgroup already exists, kill all processes + // in it and destroy it. + + if (mkdir(childCgroup.c_str(), 0755) == -1 && errno != EEXIST) + throw SysError("creating cgroup '%s'", childCgroup); + + chownToBuilder(childCgroup); + chownToBuilder(childCgroup + "/cgroup.procs"); + if (name == "unified") { + chownToBuilder(childCgroup + "/cgroup.threads"); + chownToBuilder(childCgroup + "/cgroup.subtree_control"); + } + + cgroups.push_back(childCgroup); + } + /* Set up private namespaces for the build: - The PID namespace causes the build to start as PID 1. @@ -2496,6 +2544,10 @@ void DerivationGoal::startBuilder() if (sandboxMountNamespace.get() == -1) throw SysError("getting sandbox mount namespace"); + /* Move the child into its own cgroup. */ + for (auto & childCgroup : cgroups) + writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; @@ -3279,6 +3331,12 @@ void DerivationGoal::runChild() if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); + /* Mount sysfs on /sys. FIXME: only in user namespace + builds. */ + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, @@ -3321,6 +3379,12 @@ void DerivationGoal::runChild() if (unshare(CLONE_NEWNS) == -1) throw SysError("unsharing mount namespace"); + /* Unshare the cgroup namespace. This means + /proc/self/cgroup will show the child's cgroup as '/' + rather than whatever it is in the parent. */ + if (unshare(CLONE_NEWCGROUP) == -1) + throw SysError("unsharing cgroup namespace"); + /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) throw SysError("cannot change directory to '%1%'", chrootRootDir); From ca2f64bcdaef5915f5147eac935ecb770511e438 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 May 2020 00:32:44 +0200 Subject: [PATCH 04/87] Reduce # of UIDs per build to 65536 2^18 was overkill. The idea was to enable multiple containers to run inside a build. However, those containers can use the same UID range - we don't really care about perfect isolation between containers inside a build. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7dc842bca..89db072b0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -149,7 +149,7 @@ public: "The Unix group that contains the build users."}; #if __linux__ - const uint32_t idsPerBuild = 1 << 18; + const uint32_t idsPerBuild = 1 << 16; Setting startId{this, 872415232, "start-id", "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"}; From 7bdcf43b401eba6aee29a359c5bce1f9cc01ce52 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 16 May 2020 21:09:48 +0200 Subject: [PATCH 05/87] Destroy the cgroup prior to building --- src/libstore/build.cc | 8 +++---- src/libstore/cgroup.cc | 49 ++++++++++++++++++++++++++++++++++++++++++ src/libstore/cgroup.hh | 13 +++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 src/libstore/cgroup.cc create mode 100644 src/libstore/cgroup.hh diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 816d695a5..1f1468d97 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -16,6 +16,7 @@ #include "machines.hh" #include "daemon.hh" #include "worker-protocol.hh" +#include "cgroup.hh" #include #include @@ -2400,14 +2401,13 @@ void DerivationGoal::startBuilder() auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup); if (!pathExists(hostCgroup)) - throw Error("expected unified cgroup directory '%s'", hostCgroup); + throw Error("expected cgroup directory '%s'", hostCgroup); auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); - // FIXME: if the cgroup already exists, kill all processes - // in it and destroy it. + destroyCgroup(childCgroup); - if (mkdir(childCgroup.c_str(), 0755) == -1 && errno != EEXIST) + if (mkdir(childCgroup.c_str(), 0755) == -1) throw SysError("creating cgroup '%s'", childCgroup); chownToBuilder(childCgroup); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc new file mode 100644 index 000000000..8cd682e68 --- /dev/null +++ b/src/libstore/cgroup.cc @@ -0,0 +1,49 @@ +#if __linux__ + +#include "cgroup.hh" +#include "util.hh" + +#include + +#include + +namespace nix { + +void destroyCgroup(const Path & cgroup) +{ + for (auto & entry : readDirectory(cgroup)) { + if (entry.type != DT_DIR) continue; + destroyCgroup(cgroup + "/" + entry.name); + } + + int round = 1; + + while (true) { + auto pids = tokenizeString>(readFile(cgroup + "/cgroup.procs")); + + if (pids.empty()) break; + + if (round > 20) + throw Error("cannot kill cgroup '%s'", cgroup); + + for (auto & pid_s : pids) { + pid_t pid; + if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid); + // FIXME: pid wraparound + if (kill(pid, SIGKILL) == -1 && errno != ESRCH) + throw SysError("killing member %d of cgroup '%s'", pid, cgroup); + } + + auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); + printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + std::this_thread::sleep_for(sleep); + round++; + } + + if (rmdir(cgroup.c_str()) == -1) + throw SysError("deleting cgroup '%s'", cgroup); +} + +} + +#endif diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh new file mode 100644 index 000000000..c7b09398e --- /dev/null +++ b/src/libstore/cgroup.hh @@ -0,0 +1,13 @@ +#pragma once + +#if __linux__ + +#include "types.hh" + +namespace nix { + +void destroyCgroup(const Path & cgroup); + +} + +#endif From 570c443f560e015cf02e4f96102eaaa0e6853562 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 16 May 2020 21:21:41 +0200 Subject: [PATCH 06/87] Simplify cgroup creation --- src/libstore/build.cc | 53 +++++++++++++----------------------------- src/libstore/cgroup.cc | 17 ++++++++++++++ src/libstore/cgroup.hh | 2 ++ 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1f1468d97..97554e9cf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2375,50 +2375,30 @@ void DerivationGoal::startBuilder() #if __linux__ if (useChroot) { - /* Create a cgroup. */ + /* Create a systemd cgroup since that's the minimum required + by systemd-nspawn. */ // FIXME: do we want to use the parent cgroup? We should // always use the same cgroup regardless of whether we're the // daemon or run from a user session via sudo. - std::string msg; - std::vector cgroups; - for (auto & line : tokenizeString>(readFile("/proc/self/cgroup"), "\n")) { - static std::regex regex("([0-9]+):([^:]*):(.*)"); - std::smatch match; - if (!std::regex_match(line, match, regex)) - throw Error("invalid line '%s' in '/proc/self/cgroup'", line); + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto systemdCgroup = ourCgroups["systemd"]; + if (systemdCgroup == "") + throw Error("'systemd' cgroup does not exist"); - /* We only create a systemd cgroup, since that's enough - for running systemd-nspawn. */ - std::string name; - if (match[2] == "name=systemd") - name = "systemd"; - //else if (match[2] == "") - // name = "unified"; - else continue; + auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); - std::string cgroup = match[3]; + if (!pathExists(hostCgroup)) + throw Error("expected cgroup directory '%s'", hostCgroup); - auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup); + auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); - if (!pathExists(hostCgroup)) - throw Error("expected cgroup directory '%s'", hostCgroup); + destroyCgroup(childCgroup); - auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); + if (mkdir(childCgroup.c_str(), 0755) == -1) + throw SysError("creating cgroup '%s'", childCgroup); - destroyCgroup(childCgroup); - - if (mkdir(childCgroup.c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", childCgroup); - - chownToBuilder(childCgroup); - chownToBuilder(childCgroup + "/cgroup.procs"); - if (name == "unified") { - chownToBuilder(childCgroup + "/cgroup.threads"); - chownToBuilder(childCgroup + "/cgroup.subtree_control"); - } - - cgroups.push_back(childCgroup); - } + chownToBuilder(childCgroup); + chownToBuilder(childCgroup + "/cgroup.procs"); /* Set up private namespaces for the build: @@ -2545,8 +2525,7 @@ void DerivationGoal::startBuilder() throw SysError("getting sandbox mount namespace"); /* Move the child into its own cgroup. */ - for (auto & childCgroup : cgroups) - writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 8cd682e68..887facdca 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -9,6 +9,23 @@ namespace nix { +std::map getCgroups(const Path & cgroupFile) +{ + std::map cgroups; + + for (auto & line : tokenizeString>(readFile(cgroupFile), "\n")) { + static std::regex regex("([0-9]+):([^:]*):(.*)"); + std::smatch match; + if (!std::regex_match(line, match, regex)) + throw Error("invalid line '%s' in '%s'", line, cgroupFile); + + std::string name = hasPrefix(match[2], "name=") ? std::string(match[2], 5) : match[2]; + cgroups.insert_or_assign(name, match[3]); + } + + return cgroups; +} + void destroyCgroup(const Path & cgroup) { for (auto & entry : readDirectory(cgroup)) { diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh index c7b09398e..dc6758957 100644 --- a/src/libstore/cgroup.hh +++ b/src/libstore/cgroup.hh @@ -6,6 +6,8 @@ namespace nix { +std::map getCgroups(const Path & cgroupFile); + void destroyCgroup(const Path & cgroup); } From ba50c3efa3b2394f5a8372939bc600008cd25e7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 May 2020 23:25:44 +0200 Subject: [PATCH 07/87] Add "uid-range" and "systemd-cgroup" system features "uid-range" provides 65536 UIDs to a build and runs the build as root in its user namespace. "systemd-cgroup" allows the build to mount the systemd cgroup controller (needed for running systemd-nspawn and NixOS containers). Also, add a configuration option "auto-allocate-uids" which is needed to enable these features, and some experimental feature gates. So to enable support for containers you need the following in nix.conf: experimental-features = auto-allocate-uids systemd-cgroup auto-allocate-uids = true system-features = uid-range systemd-cgroup --- src/libstore/build.cc | 270 ++++++++++---------------------------- src/libstore/cgroup.cc | 19 ++- src/libstore/globals.hh | 5 +- src/libstore/user-lock.cc | 212 ++++++++++++++++++++++++++++++ src/libstore/user-lock.hh | 39 ++++++ 5 files changed, 340 insertions(+), 205 deletions(-) create mode 100644 src/libstore/user-lock.cc create mode 100644 src/libstore/user-lock.hh diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 97554e9cf..1f79a8d2d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -16,7 +16,7 @@ #include "machines.hh" #include "daemon.hh" #include "worker-protocol.hh" -#include "cgroup.hh" +#include "user-lock.hh" #include #include @@ -504,154 +504,6 @@ void handleDiffHook( } } -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - uid_t getUID() { assert(uid); return uid; } - gid_t getGID() { assert(gid); return gid; } - uint32_t getIDCount() { return settings.idsPerBuild; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ -#if 0 - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -#endif -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - -#if 0 - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -#endif - - assert(settings.startId > 0); - assert(settings.startId % settings.idsPerBuild == 0); - assert(settings.uidCount % settings.idsPerBuild == 0); - assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); - - // FIXME: check whether the id range overlaps any known users - - size_t nrSlots = settings.uidCount / settings.idsPerBuild; - - for (size_t i = 0; i < nrSlots; i++) { - debug("trying user slot '%d'", i); - - createDirs(settings.nixStateDir + "/userpool"); - - fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - uid = settings.startId + i * settings.idsPerBuild; - gid = settings.startId + i * settings.idsPerBuild; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - // FIXME: use a cgroup to kill all processes in the build? -#if 0 - killUser(uid); -#endif -} - ////////////////////////////////////////////////////////////////////// @@ -840,6 +692,13 @@ private: Path chrootRootDir; + /* Whether to give the build more than 1 UID. */ + bool useUidRange = false; + + /* Whether to make the 'systemd' cgroup controller available to + the build. */ + bool useSystemdCgroup = false; + /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; @@ -896,8 +755,8 @@ private: result. */ std::map prevInfos; - const uid_t sandboxUid = 1000; - const gid_t sandboxGid = 100; + uid_t sandboxUid = -1; + gid_t sandboxGid = -1; const static Path homeDir; @@ -1445,6 +1304,7 @@ void DerivationGoal::inputsRealised() result = BuildResult(); } + void DerivationGoal::started() { auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : @@ -1459,6 +1319,7 @@ void DerivationGoal::started() { worker.updateProgress(); } + void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1556,25 +1417,28 @@ void DerivationGoal::tryToBuild() worker.wakeUp(shared_from_this()); } + void DerivationGoal::tryLocalBuild() { /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ - if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) { + static bool useBuildUsers = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; + if (useBuildUsers) { #if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); + if (!buildUser) + buildUser = acquireUserLock(); - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { + if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } + + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); #else /* Don't know how to block the creation of setuid/setgid binaries on this platform. */ @@ -2087,6 +1951,9 @@ void DerivationGoal::startBuilder() } } + useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range"); + useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("systemd-cgroup"); + if (useChroot) { /* Allow a user-configurable set of directories from the @@ -2166,7 +2033,7 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - if (mkdir(chrootRootDir.c_str(), 0755) == -1) + if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); // FIXME: only make root writable for user namespace builds. @@ -2186,6 +2053,12 @@ void DerivationGoal::startBuilder() createDirs(chrootRootDir + "/etc"); chownToBuilder(chrootRootDir + "/etc"); + if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name); + + sandboxUid = useUidRange ? 0 : 1000; + sandboxGid = useUidRange ? 0 : 100; + writeFile(chrootRootDir + "/etc/passwd", fmt( "root:x:0:0:Nix build user:%3%:/noshell\n" "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" @@ -2238,12 +2111,32 @@ void DerivationGoal::startBuilder() for (auto & i : drv->outputs) dirsInChroot.erase(worker.store.printStorePath(i.second.path)); -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ + if (useSystemdCgroup) { + settings.requireExperimentalFeature("systemd-cgroup"); + std::optional cgroup; + if (!buildUser || !(cgroup = buildUser->getCgroup())) + throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf"); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + } + #else - throw Error("sandboxing builds is not supported on this platform"); + if (useUidRange) + throw Error("feature 'uid-range' is not supported on this platform"); + if (useSystemdCgroup) + throw Error("feature 'systemd-cgroup' is not supported on this platform"); + #if __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ + #else + throw Error("sandboxing builds is not supported on this platform"); + #endif #endif + } else { + if (useUidRange) + throw Error("feature 'uid-range' is only supported in sandboxed builds"); + if (useSystemdCgroup) + throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); } if (needsHashRewrite()) { @@ -2375,31 +2268,6 @@ void DerivationGoal::startBuilder() #if __linux__ if (useChroot) { - /* Create a systemd cgroup since that's the minimum required - by systemd-nspawn. */ - // FIXME: do we want to use the parent cgroup? We should - // always use the same cgroup regardless of whether we're the - // daemon or run from a user session via sudo. - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto systemdCgroup = ourCgroups["systemd"]; - if (systemdCgroup == "") - throw Error("'systemd' cgroup does not exist"); - - auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); - - if (!pathExists(hostCgroup)) - throw Error("expected cgroup directory '%s'", hostCgroup); - - auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); - - destroyCgroup(childCgroup); - - if (mkdir(childCgroup.c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", childCgroup); - - chownToBuilder(childCgroup); - chownToBuilder(childCgroup + "/cgroup.procs"); - /* Set up private namespaces for the build: - The PID namespace causes the build to start as PID 1. @@ -2508,15 +2376,16 @@ void DerivationGoal::startBuilder() the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - uint32_t nrIds = settings.idsPerBuild; // FIXME + uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1; writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds)); + fmt("%d %d %d", sandboxUid, hostUid, nrIds)); - //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + if (!useUidRange) + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds)); + fmt("%d %d %d", sandboxGid, hostGid, nrIds)); /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ @@ -2525,7 +2394,10 @@ void DerivationGoal::startBuilder() throw SysError("getting sandbox mount namespace"); /* Move the child into its own cgroup. */ - writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + if (buildUser) { + if (auto cgroup = buildUser->getCgroup()) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + } /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); @@ -3361,7 +3233,7 @@ void DerivationGoal::runChild() /* Unshare the cgroup namespace. This means /proc/self/cgroup will show the child's cgroup as '/' rather than whatever it is in the parent. */ - if (unshare(CLONE_NEWCGROUP) == -1) + if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1) throw SysError("unsharing cgroup namespace"); /* Do the chroot(). */ @@ -3386,16 +3258,10 @@ void DerivationGoal::runChild() /* Switch to the sandbox uid/gid in the user namespace, which corresponds to the build user or calling user in the parent namespace. */ -#if 0 if (setgid(sandboxGid) == -1) throw SysError("setgid failed"); if (setuid(sandboxUid) == -1) throw SysError("setuid failed"); -#endif - if (setgid(0) == -1) - throw SysError("setgid failed"); - if (setuid(0) == -1) - throw SysError("setuid failed"); setUser = false; } @@ -3789,7 +3655,7 @@ void DerivationGoal::registerOutputs() something like that. */ canonicalisePathMetaData( actualPath, - buildUser ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) : std::nullopt, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); /* FIXME: this is in-memory. */ @@ -3866,7 +3732,7 @@ void DerivationGoal::registerOutputs() all files are owned by the build user, if applicable. */ canonicalisePathMetaData(actualPath, buildUser && !rewritten - ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) + ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 887facdca..9e5e937df 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -4,6 +4,7 @@ #include "util.hh" #include +#include #include @@ -19,7 +20,7 @@ std::map getCgroups(const Path & cgroupFile) if (!std::regex_match(line, match, regex)) throw Error("invalid line '%s' in '%s'", line, cgroupFile); - std::string name = hasPrefix(match[2], "name=") ? std::string(match[2], 5) : match[2]; + std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2]; cgroups.insert_or_assign(name, match[3]); } @@ -28,6 +29,8 @@ std::map getCgroups(const Path & cgroupFile) void destroyCgroup(const Path & cgroup) { + if (!pathExists(cgroup)) return; + for (auto & entry : readDirectory(cgroup)) { if (entry.type != DT_DIR) continue; destroyCgroup(cgroup + "/" + entry.name); @@ -35,6 +38,8 @@ void destroyCgroup(const Path & cgroup) int round = 1; + std::unordered_set pidsShown; + while (true) { auto pids = tokenizeString>(readFile(cgroup + "/cgroup.procs")); @@ -46,13 +51,23 @@ void destroyCgroup(const Path & cgroup) for (auto & pid_s : pids) { pid_t pid; if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid); + if (pidsShown.insert(pid).second) { + try { + auto cmdline = readFile(fmt("/proc/%d/cmdline", pid)); + using namespace std::string_literals; + warn("killing stray builder process %d (%s)...", + pid, trim(replaceStrings(cmdline, "\0"s, " "))); + } catch (SysError &) { + } + } // FIXME: pid wraparound if (kill(pid, SIGKILL) == -1 && errno != ESRCH) throw SysError("killing member %d of cgroup '%s'", pid, cgroup); } auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); - printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + if (sleep.count() > 100) + printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); std::this_thread::sleep_for(sleep); round++; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 89db072b0..5cf73c7b4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -149,10 +149,13 @@ public: "The Unix group that contains the build users."}; #if __linux__ + Setting autoAllocateUids{this, false, "auto-allocate-uids", + "Whether to allocate UIDs for builders automatically."}; + const uint32_t idsPerBuild = 1 << 16; Setting startId{this, 872415232, "start-id", - "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"}; + "The first UID and GID to use for dynamic ID allocation."}; Setting uidCount{this, idsPerBuild * 128, "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc new file mode 100644 index 000000000..8a09df4d1 --- /dev/null +++ b/src/libstore/user-lock.cc @@ -0,0 +1,212 @@ +#include "user-lock.hh" +#include "globals.hh" +#include "pathlocks.hh" +#include "cgroup.hh" + +namespace nix { + +struct SimpleUserLock : UserLock +{ + AutoCloseFD fdUserLock; + uid_t uid; + gid_t gid; + std::vector supplementaryGIDs; + + void kill() override + { + killUser(uid); + } + + std::pair getUIDRange() override + { + assert(uid); + return {uid, uid}; + } + + gid_t getGID() override { assert(gid); return gid; } + + std::vector getSupplementaryGIDs() override { return supplementaryGIDs; } + + static std::unique_ptr acquire() + { + assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); + + /* Get the members of the build-users-group. */ + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + + /* Copy the result of getgrnam. */ + Strings users; + for (char * * p = gr->gr_mem; *p; ++p) { + debug("found build user '%s'", *p); + users.push_back(*p); + } + + if (users.empty()) + throw Error("the build users group '%s' has no members", settings.buildUsersGroup); + + /* Find a user account that isn't currently in use for another + build. */ + for (auto & i : users) { + debug("trying user '%s'", i); + + struct passwd * pw = getpwnam(i.c_str()); + if (!pw) + throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup); + + auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir,pw->pw_uid); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%s'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + auto lock = std::make_unique(); + + lock->fdUserLock = std::move(fd); + lock->uid = pw->pw_uid; + lock->gid = gr->gr_gid; + + /* Sanity check... */ + if (lock->uid == getuid() || lock->uid == geteuid()) + throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); + + #if __linux__ + /* Get the list of supplementary groups of this build + user. This is usually either empty or contains a + group such as "kvm". */ + lock->supplementaryGIDs.resize(10); + int ngroups = lock->supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + lock->supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + + lock->supplementaryGIDs.resize(ngroups); + #endif + + return lock; + } + } + + return nullptr; + } +}; + +#if __linux__ +struct CgroupUserLock : UserLock +{ + AutoCloseFD fdUserLock; + uid_t uid; + + void kill() override + { + if (cgroup) { + destroyCgroup(*cgroup); + cgroup.reset(); + } + } + + std::pair getUIDRange() override + { + assert(uid); + return {uid, uid + settings.idsPerBuild - 1}; + } + + gid_t getGID() override + { + // We use the same GID ranges as for the UIDs. + assert(uid); + return uid; + } + + std::vector getSupplementaryGIDs() override { return {}; } // FIXME + + static std::unique_ptr acquire() + { + settings.requireExperimentalFeature("auto-allocate-uids"); + assert(settings.startId > 0); + assert(settings.startId % settings.idsPerBuild == 0); + assert(settings.uidCount % settings.idsPerBuild == 0); + assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + + // FIXME: check whether the id range overlaps any known users + + createDirs(settings.nixStateDir + "/userpool2"); + + size_t nrSlots = settings.uidCount / settings.idsPerBuild; + + for (size_t i = 0; i < nrSlots; i++) { + debug("trying user slot '%d'", i); + + createDirs(settings.nixStateDir + "/userpool2"); + + auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%s'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + auto lock = std::make_unique(); + lock->fdUserLock = std::move(fd); + lock->uid = settings.startId + i * settings.idsPerBuild; + auto s = drainFD(lock->fdUserLock.get()); + if (s != "") lock->cgroup = s; + return lock; + } + } + + return nullptr; + } + + std::optional cgroup; + + std::optional getCgroup() override + { + if (!cgroup) { + /* Create a systemd cgroup since that's the minimum + required by systemd-nspawn. */ + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto systemdCgroup = ourCgroups["systemd"]; + if (systemdCgroup == "") + throw Error("'systemd' cgroup does not exist"); + + auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); + + if (!pathExists(hostCgroup)) + throw Error("expected cgroup directory '%s'", hostCgroup); + + cgroup = fmt("%s/nix-%d", hostCgroup, uid); + + destroyCgroup(*cgroup); + + if (mkdir(cgroup->c_str(), 0755) == -1) + throw SysError("creating cgroup '%s'", *cgroup); + + /* Record the cgroup in the lock file. This ensures that + if we subsequently get executed under a different parent + cgroup, we kill the previous cgroup first. */ + if (ftruncate(fdUserLock.get(), 0) == -1) + throw Error("truncating user lock"); + writeFull(fdUserLock.get(), *cgroup); + } + + return cgroup; + }; +}; +#endif + +std::unique_ptr acquireUserLock() +{ + #if __linux__ + if (settings.autoAllocateUids) + return CgroupUserLock::acquire(); + else + #endif + return SimpleUserLock::acquire(); +} + +} diff --git a/src/libstore/user-lock.hh b/src/libstore/user-lock.hh new file mode 100644 index 000000000..88d068689 --- /dev/null +++ b/src/libstore/user-lock.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct UserLock +{ + virtual ~UserLock() { } + + /* Get the first and last UID. */ + virtual std::pair getUIDRange() = 0; + + /* Get the first UID. */ + uid_t getUID() + { + return getUIDRange().first; + } + + uid_t getUIDCount() + { + return getUIDRange().second - getUIDRange().first + 1; + } + + virtual gid_t getGID() = 0; + + virtual std::vector getSupplementaryGIDs() = 0; + + /* Kill any processes currently executing as this user. */ + virtual void kill() = 0; + + virtual std::optional getCgroup() { return {}; }; +}; + +/* Acquire a user lock. Note that this may return nullptr if no user + is available. */ +std::unique_ptr acquireUserLock(); + +} From 8c4cce553c16438f0ccbbaea6d77f2bd64306dfe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 May 2020 11:24:21 +0200 Subject: [PATCH 08/87] Fix macOS build --- src/libstore/build.cc | 11 +---------- src/libstore/user-lock.cc | 13 +++++++++++++ src/libstore/user-lock.hh | 2 ++ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1f79a8d2d..6c3f94a76 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1420,11 +1420,7 @@ void DerivationGoal::tryToBuild() void DerivationGoal::tryLocalBuild() { - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - static bool useBuildUsers = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; - if (useBuildUsers) { -#if defined(__linux__) || defined(__APPLE__) + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(); @@ -1439,11 +1435,6 @@ void DerivationGoal::tryLocalBuild() { /* Make sure that no other processes are executing under this uid. */ buildUser->kill(); -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif } actLock.reset(); diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc index 8a09df4d1..2254386da 100644 --- a/src/libstore/user-lock.cc +++ b/src/libstore/user-lock.cc @@ -209,4 +209,17 @@ std::unique_ptr acquireUserLock() return SimpleUserLock::acquire(); } +bool useBuildUsers() +{ + #if __linux__ + static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; + return b; + #elif __APPLE__ + static bool b = settings.buildUsersGroup != "" && getuid() == 0; + return b; + #else + return false; + #endif +} + } diff --git a/src/libstore/user-lock.hh b/src/libstore/user-lock.hh index 88d068689..bfb55b0d9 100644 --- a/src/libstore/user-lock.hh +++ b/src/libstore/user-lock.hh @@ -36,4 +36,6 @@ struct UserLock is available. */ std::unique_ptr acquireUserLock(); +bool useBuildUsers(); + } From 7349f257da8278af9aae35544b15c9a204e2a57b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 May 2020 11:57:33 +0200 Subject: [PATCH 09/87] Only mount /sys in uid-range builds Maybe this should be a separate system feature... /sys exposes a lot of impure info about the host system. --- src/libstore/build.cc | 11 ++++++----- src/libstore/user-lock.cc | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6c3f94a76..e927a65f0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3173,11 +3173,12 @@ void DerivationGoal::runChild() if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); - /* Mount sysfs on /sys. FIXME: only in user namespace - builds. */ - createDirs(chrootRootDir + "/sys"); - if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) - throw SysError("mounting /sys"); + /* Mount sysfs on /sys. */ + if (useUidRange) { + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + } /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc index 2254386da..fb2a45f48 100644 --- a/src/libstore/user-lock.cc +++ b/src/libstore/user-lock.cc @@ -122,7 +122,7 @@ struct CgroupUserLock : UserLock return uid; } - std::vector getSupplementaryGIDs() override { return {}; } // FIXME + std::vector getSupplementaryGIDs() override { return {}; } static std::unique_ptr acquire() { From c068cce107b94f23c5288521770b2c807729a052 Mon Sep 17 00:00:00 2001 From: Sagar Patel Date: Tue, 1 Nov 2022 00:09:00 -0400 Subject: [PATCH 10/87] docs: drop shell prompt character for consistency In addition to consistency, the fancy "Copy to clipboard" button on the website will copy the prompt character. Retaining the prompt character would mean having to edit each command after pasting in the terminal. --- .../src/installation/installing-binary.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 2d007ca1b..d6d7a61c8 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -3,7 +3,7 @@ The easiest way to install Nix is to run the following command: ```console -$ sh <(curl -L https://nixos.org/nix/install) +sh <(curl -L https://nixos.org/nix/install) ``` This will run the installer interactively (causing it to explain what @@ -27,7 +27,7 @@ you can authenticate with `sudo`. To explicitly select a single-user installation on your system: ```console -$ sh <(curl -L https://nixos.org/nix/install) --no-daemon +sh <(curl -L https://nixos.org/nix/install) --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` @@ -37,8 +37,8 @@ if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: ```console -$ mkdir /nix -$ chown alice /nix +mkdir /nix +chown alice /nix ``` The install script will modify the first writable file from amongst @@ -50,7 +50,7 @@ the install script to disable this behaviour. You can uninstall Nix simply by running: ```console -$ rm -rf /nix +rm -rf /nix ``` # Multi User Installation @@ -66,7 +66,7 @@ You can instruct the installer to perform a multi-user installation on your system: ```console -$ sh <(curl -L https://nixos.org/nix/install) --daemon +sh <(curl -L https://nixos.org/nix/install) --daemon ``` The multi-user installation of Nix will create build users between the @@ -255,7 +255,7 @@ These install scripts can be used the same as the main NixOS.org installation script: ```console -$ sh <(curl -L https://nixos.org/nix/install) +sh <(curl -L https://nixos.org/nix/install) ``` In the same directory of the install script are sha256 sums, and gpg @@ -270,10 +270,10 @@ it somewhere (e.g. in `/tmp`), and then run the script named `install` inside the binary tarball: ```console -$ cd /tmp -$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 -$ cd nix-1.8-x86_64-darwin -$ ./install +cd /tmp +tar xfj nix-1.8-x86_64-darwin.tar.bz2 +cd nix-1.8-x86_64-darwin +./install ``` If you need to edit the multi-user installation script to use different From 6259fd7ea6ac331d1037a150b07c01125e80ed8e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Nov 2022 13:29:39 +0100 Subject: [PATCH 11/87] Fix indentation --- src/nix-store/nix-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index eb65e7dde..b59a6d026 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -745,8 +745,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto current = sink.finish(); if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), - info->narHash.to_string(Base32, true), + store->printStorePath(path), + info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); status = 1; } From 40911d7dec75541a400fe8f556d4c70a7f845fac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Nov 2022 13:30:35 +0100 Subject: [PATCH 12/87] Remove stray tab --- src/libutil/experimental-features.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 30d071408..670079019 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -14,8 +14,8 @@ std::map stringifiedXpFeatures = { { Xp::NoUrlLiterals, "no-url-literals" }, { Xp::FetchClosure, "fetch-closure" }, { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::SystemdCgroup, "systemd-cgroup" }, + { Xp::AutoAllocateUids, "auto-allocate-uids" }, + { Xp::SystemdCgroup, "systemd-cgroup" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) From 2320a2f93e0aa61e8d45dcfb3fb536361a1fcb6f Mon Sep 17 00:00:00 2001 From: Artturin Date: Fri, 10 Jun 2022 17:46:57 +0300 Subject: [PATCH 13/87] systemd,launchd match nixos open files limit it was bumped in https://github.com/NixOS/nixpkgs/pull/176558 --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 +- misc/systemd/nix-daemon.service.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index da1970f69..5fa489b20 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -28,7 +28,7 @@ SoftResourceLimits NumberOfFiles - 4096 + 1048576 diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index e3ac42beb..f46413630 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -9,7 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process -LimitNOFILE=4096 +LimitNOFILE=1048576 [Install] WantedBy=multi-user.target From 2fde7e0108d70bcba64ebecc5e5c7ee2863e3446 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Nov 2022 16:03:42 +0100 Subject: [PATCH 14/87] Split auto UID allocation from cgroups Cgroups are now only used for derivations that require the uid-range range feature. This allows auto UID allocation even on systems that don't have cgroups (like macOS). Also, make things work on modern systems that use cgroups v2 (where there is a single hierarchy and no "systemd" controller). --- src/libstore/build/local-derivation-goal.cc | 19 +-- src/libstore/build/local-derivation-goal.hh | 7 +- src/libstore/cgroup.cc | 1 + src/libstore/globals.cc | 4 + src/libstore/globals.hh | 6 +- src/libstore/lock.cc | 149 +++++++++++--------- src/libstore/lock.hh | 24 ++-- src/libstore/parsed-derivations.cc | 6 + src/libstore/parsed-derivations.hh | 2 + 9 files changed, 122 insertions(+), 96 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64540d262..09da87476 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -160,7 +160,7 @@ void LocalDerivationGoal::tryLocalBuild() { if (useBuildUsers()) { if (!buildUser) - buildUser = acquireUserLock(); + buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1); if (!buildUser) { if (!actLock) @@ -495,8 +495,8 @@ void LocalDerivationGoal::startBuilder() } } - useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range"); useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("Systemd-cgroup"); + assert(!useSystemdCgroup); if (useChroot) { @@ -576,7 +576,8 @@ void LocalDerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1) + // FIXME: make this 0700 + if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); // FIXME: only make root writable for user namespace builds. @@ -596,8 +597,8 @@ void LocalDerivationGoal::startBuilder() createDirs(chrootRootDir + "/etc"); chownToBuilder(chrootRootDir + "/etc"); - if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536)) - throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name); + if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -670,7 +671,7 @@ void LocalDerivationGoal::startBuilder() #endif #endif } else { - if (useUidRange) + if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is only supported in sandboxed builds"); if (useSystemdCgroup) throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); @@ -934,12 +935,12 @@ void LocalDerivationGoal::startBuilder() the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1; + uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; writeFile("/proc/" + std::to_string(pid) + "/uid_map", fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); - if (!useUidRange) + if (!buildUser || buildUser->getUIDCount() == 1) writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", @@ -1793,7 +1794,7 @@ void LocalDerivationGoal::runChild() throw SysError("mounting /proc"); /* Mount sysfs on /sys. */ - if (useUidRange) { + if (buildUser && buildUser->getUIDCount() != 1) { createDirs(chrootRootDir + "/sys"); if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) throw SysError("mounting /sys"); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index e6700a383..61b0f9145 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -41,9 +41,6 @@ struct LocalDerivationGoal : public DerivationGoal Path chrootRootDir; - /* Whether to give the build more than 1 UID. */ - bool useUidRange = false; - /* Whether to make the 'systemd' cgroup controller available to the build. */ bool useSystemdCgroup = false; @@ -99,8 +96,8 @@ struct LocalDerivationGoal : public DerivationGoal result. */ std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100) : buildUser->getGID(); } + uid_t sandboxUid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } const static Path homeDir; diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 5d31609da..56e980be3 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -13,6 +13,7 @@ namespace nix { +// FIXME: obsolete, check for cgroup2 std::map getCgroups(const Path & cgroupFile) { std::map cgroups; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ff658c428..b7f55cae7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -130,6 +130,10 @@ StringSet Settings::getDefaultSystemFeatures() actually require anything special on the machines. */ StringSet features{"nixos-test", "benchmark", "big-parallel"}; + #if __linux__ + features.insert("uid-range"); + #endif + #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d3e86cc55..be741a830 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -46,6 +46,8 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +const uint32_t maxIdsPerBuild = 1 << 16; + class Settings : public Config { unsigned int getDefaultCores(); @@ -279,12 +281,10 @@ public: Setting autoAllocateUids{this, false, "auto-allocate-uids", "Whether to allocate UIDs for builders automatically."}; - const uint32_t idsPerBuild = 1 << 16; - Setting startId{this, 872415232, "start-id", "The first UID and GID to use for dynamic ID allocation."}; - Setting uidCount{this, idsPerBuild * 128, "id-count", + Setting uidCount{this, maxIdsPerBuild * 128, "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; #endif diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index cc3977496..ecc51cebe 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -20,12 +20,8 @@ struct SimpleUserLock : UserLock killUser(uid); } - std::pair getUIDRange() override - { - assert(uid); - return {uid, uid}; - } - + uid_t getUID() override { assert(uid); return uid; } + uid_t getUIDCount() override { return 1; } gid_t getGID() override { assert(gid); return gid; } std::vector getSupplementaryGIDs() override { return supplementaryGIDs; } @@ -115,48 +111,65 @@ struct SimpleUserLock : UserLock } }; -#if __linux__ -struct CgroupUserLock : UserLock +struct AutoUserLock : UserLock { AutoCloseFD fdUserLock; - uid_t uid; + uid_t firstUid = 0; + uid_t nrIds = 1; + #if __linux__ + std::optional cgroup; + #endif + + ~AutoUserLock() + { + // Get rid of our cgroup, ignoring errors. + if (cgroup) rmdir(cgroup->c_str()); + } void kill() override { + #if __linux__ if (cgroup) { + printError("KILL CGROUP %s", *cgroup); destroyCgroup(*cgroup); - cgroup.reset(); + if (mkdir(cgroup->c_str(), 0755) == -1) + throw SysError("creating cgroup '%s'", *cgroup); + } else + #endif + { + assert(firstUid); + printError("KILL USER %d", firstUid); + killUser(firstUid); } } - std::pair getUIDRange() override - { - assert(uid); - return {uid, uid + settings.idsPerBuild - 1}; - } + uid_t getUID() override { assert(firstUid); return firstUid; } + + gid_t getUIDCount() override { return nrIds; } gid_t getGID() override { // We use the same GID ranges as for the UIDs. - assert(uid); - return uid; + assert(firstUid); + return firstUid; } std::vector getSupplementaryGIDs() override { return {}; } - static std::unique_ptr acquire() + static std::unique_ptr acquire(uid_t nrIds) { settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); - assert(settings.startId % settings.idsPerBuild == 0); - assert(settings.uidCount % settings.idsPerBuild == 0); + assert(settings.startId % maxIdsPerBuild == 0); + assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + assert(nrIds <= maxIdsPerBuild); // FIXME: check whether the id range overlaps any known users createDirs(settings.nixStateDir + "/userpool2"); - size_t nrSlots = settings.uidCount / settings.idsPerBuild; + size_t nrSlots = settings.uidCount / maxIdsPerBuild; for (size_t i = 0; i < nrSlots; i++) { debug("trying user slot '%d'", i); @@ -170,11 +183,47 @@ struct CgroupUserLock : UserLock throw SysError("opening user lock '%s'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { - auto lock = std::make_unique(); + auto s = drainFD(fd.get()); + + #if __linux__ + if (s != "") { + /* Kill the old cgroup, to ensure there are no + processes left over from an interrupted build. */ + destroyCgroup(s); + } + #endif + + if (ftruncate(fd.get(), 0) == -1) + throw Error("truncating user lock"); + + auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); - lock->uid = settings.startId + i * settings.idsPerBuild; - auto s = drainFD(lock->fdUserLock.get()); - if (s != "") lock->cgroup = s; + lock->firstUid = settings.startId + i * maxIdsPerBuild; + lock->nrIds = nrIds; + + if (nrIds > 1) { + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto ourCgroup = ourCgroups[""]; + if (ourCgroup == "") + throw Error("cannot determine cgroup name from /proc/self/cgroup"); + + auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); + + printError("PARENT CGROUP = %s", ourCgroupPath); + + if (!pathExists(ourCgroupPath)) + throw Error("expected cgroup directory '%s'", ourCgroupPath); + + lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid); + + printError("CHILD CGROUP = %s", *lock->cgroup); + + /* Record the cgroup in the lock file. This ensures that + if we subsequently get executed under a different parent + cgroup, we kill the previous cgroup first. */ + writeFull(lock->fdUserLock.get(), *lock->cgroup); + } + return lock; } } @@ -182,50 +231,16 @@ struct CgroupUserLock : UserLock return nullptr; } - std::optional cgroup; - - std::optional getCgroup() override - { - if (!cgroup) { - /* Create a systemd cgroup since that's the minimum - required by systemd-nspawn. */ - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto systemdCgroup = ourCgroups["systemd"]; - if (systemdCgroup == "") - throw Error("'systemd' cgroup does not exist"); - - auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); - - if (!pathExists(hostCgroup)) - throw Error("expected cgroup directory '%s'", hostCgroup); - - cgroup = fmt("%s/nix-%d", hostCgroup, uid); - - destroyCgroup(*cgroup); - - if (mkdir(cgroup->c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", *cgroup); - - /* Record the cgroup in the lock file. This ensures that - if we subsequently get executed under a different parent - cgroup, we kill the previous cgroup first. */ - if (ftruncate(fdUserLock.get(), 0) == -1) - throw Error("truncating user lock"); - writeFull(fdUserLock.get(), *cgroup); - } - - return cgroup; - }; -}; -#endif - -std::unique_ptr acquireUserLock() -{ #if __linux__ - if (settings.autoAllocateUids) - return CgroupUserLock::acquire(); - else + std::optional getCgroup() override { return cgroup; } #endif +}; + +std::unique_ptr acquireUserLock(uid_t nrIds) +{ + if (settings.autoAllocateUids) + return AutoUserLock::acquire(nrIds); + else return SimpleUserLock::acquire(); } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 4b6d34069..62676a523 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -11,18 +11,16 @@ struct UserLock virtual ~UserLock() { } /* Get the first and last UID. */ - virtual std::pair getUIDRange() = 0; + std::pair getUIDRange() + { + auto first = getUID(); + return {first, first + getUIDCount() - 1}; + } /* Get the first UID. */ - uid_t getUID() - { - return getUIDRange().first; - } + virtual uid_t getUID() = 0; - uid_t getUIDCount() - { - return getUIDRange().second - getUIDRange().first + 1; - } + virtual uid_t getUIDCount() = 0; virtual gid_t getGID() = 0; @@ -31,12 +29,14 @@ struct UserLock /* Kill any processes currently executing as this user. */ virtual void kill() = 0; + #if __linux__ virtual std::optional getCgroup() { return {}; }; + #endif }; -/* Acquire a user lock. Note that this may return nullptr if no user - is available. */ -std::unique_ptr acquireUserLock(); +/* Acquire a user lock for a UID range of size `nrIds`. Note that this + may return nullptr if no user is available. */ +std::unique_ptr acquireUserLock(uid_t nrIds); bool useBuildUsers(); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index f2288a04e..487dbcfbb 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -90,6 +90,7 @@ std::optional ParsedDerivation::getStringsAttr(const std::string & name StringSet ParsedDerivation::getRequiredSystemFeatures() const { + // FIXME: cache this? StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); @@ -125,6 +126,11 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +bool ParsedDerivation::useUidRange() const +{ + return getRequiredSystemFeatures().count("uid-range"); +} + static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 95bec21e8..bfb3857c0 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -38,6 +38,8 @@ public: bool substitutesAllowed() const; + bool useUidRange() const; + std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); }; From 05d258667d12b2decda87024a59250c43343b509 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Nov 2022 08:00:29 -0800 Subject: [PATCH 15/87] Fix build on macOS --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/globals.hh | 26 +++++++++++++++++---- src/libstore/lock.cc | 10 ++++---- src/libstore/lock.hh | 2 ++ 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 09da87476..45ea9968f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -659,7 +659,7 @@ void LocalDerivationGoal::startBuilder() } #else - if (useUidRange) + if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is not supported on this platform"); if (useSystemdCgroup) throw Error("feature 'systemd-cgroup' is not supported on this platform"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index be741a830..88fe72202 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -46,7 +46,13 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; -const uint32_t maxIdsPerBuild = 1 << 16; +const uint32_t maxIdsPerBuild = + #if __linux__ + 1 << 16 + #else + 1 + #endif + ; class Settings : public Config { @@ -277,16 +283,26 @@ public: multi-user settings with untrusted users. )"}; - #if __linux__ Setting autoAllocateUids{this, false, "auto-allocate-uids", "Whether to allocate UIDs for builders automatically."}; - Setting startId{this, 872415232, "start-id", + Setting startId{this, + #if __linux__ + 872415232, + #else + 56930, + #endif + "start-id", "The first UID and GID to use for dynamic ID allocation."}; - Setting uidCount{this, maxIdsPerBuild * 128, "id-count", + Setting uidCount{this, + #if __linux__ + maxIdsPerBuild * 128, + #else + 128, + #endif + "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; - #endif Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index ecc51cebe..f9892bb91 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -122,15 +122,16 @@ struct AutoUserLock : UserLock ~AutoUserLock() { + #if __linux__ // Get rid of our cgroup, ignoring errors. if (cgroup) rmdir(cgroup->c_str()); + #endif } void kill() override { #if __linux__ if (cgroup) { - printError("KILL CGROUP %s", *cgroup); destroyCgroup(*cgroup); if (mkdir(cgroup->c_str(), 0755) == -1) throw SysError("creating cgroup '%s'", *cgroup); @@ -138,7 +139,6 @@ struct AutoUserLock : UserLock #endif { assert(firstUid); - printError("KILL USER %d", firstUid); killUser(firstUid); } } @@ -201,6 +201,7 @@ struct AutoUserLock : UserLock lock->firstUid = settings.startId + i * maxIdsPerBuild; lock->nrIds = nrIds; + #if __linux__ if (nrIds > 1) { auto ourCgroups = getCgroups("/proc/self/cgroup"); auto ourCgroup = ourCgroups[""]; @@ -209,20 +210,17 @@ struct AutoUserLock : UserLock auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); - printError("PARENT CGROUP = %s", ourCgroupPath); - if (!pathExists(ourCgroupPath)) throw Error("expected cgroup directory '%s'", ourCgroupPath); lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid); - printError("CHILD CGROUP = %s", *lock->cgroup); - /* Record the cgroup in the lock file. This ensures that if we subsequently get executed under a different parent cgroup, we kill the previous cgroup first. */ writeFull(lock->fdUserLock.get(), *lock->cgroup); } + #endif return lock; } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 62676a523..b5536408c 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -4,6 +4,8 @@ #include +#include + namespace nix { struct UserLock From 97a9a8925abb6889fc368f1108c682879abadcdb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 8 Nov 2022 23:34:57 +0100 Subject: [PATCH 16/87] add CODEOWNERS --- .github/CODEOWNERS | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..bba083834 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,15 @@ +# Pull requests concerning the listed files will automatically invite the respective maintainers as reviewers. +# This file is not used for denoting any kind of ownership, but is merely a tool for handling notifications. +# +# Merge permissions are required for maintaining an entry in this file. +# For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ + +# Notified if nothing else matches +* @edolstra @thufschmitt + +# This file +.github/CODEOWNERS @edolstra + +# Public documentation +/doc @fricklerhandwerk +*.md @fricklerhandwerk From 6c6eff8ac40e2f5d7b6ff8e772feebb1aa484039 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Nov 2022 17:24:12 +0100 Subject: [PATCH 17/87] Remove the SystemdCgroup feature --- src/libstore/build/local-derivation-goal.cc | 23 +++++++-------------- src/libstore/build/local-derivation-goal.hh | 4 ---- src/libutil/experimental-features.cc | 1 - src/libutil/experimental-features.hh | 1 - 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 45ea9968f..e652c425c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -495,9 +495,6 @@ void LocalDerivationGoal::startBuilder() } } - useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("Systemd-cgroup"); - assert(!useSystemdCgroup); - if (useChroot) { /* Allow a user-configurable set of directories from the @@ -649,20 +646,18 @@ void LocalDerivationGoal::startBuilder() dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); } - if (useSystemdCgroup) { - settings.requireExperimentalFeature(Xp::SystemdCgroup); - std::optional cgroup; - if (!buildUser || !(cgroup = buildUser->getCgroup())) - throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf"); - chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); + if (buildUser) { + if (auto cgroup = buildUser->getCgroup()) { + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); + } } #else if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is not supported on this platform"); - if (useSystemdCgroup) - throw Error("feature 'systemd-cgroup' is not supported on this platform"); #if __APPLE__ /* We don't really have any parent prep work to do (yet?) All work happens in the child, instead. */ @@ -673,8 +668,6 @@ void LocalDerivationGoal::startBuilder() } else { if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is only supported in sandboxed builds"); - if (useSystemdCgroup) - throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); } if (needsHashRewrite() && pathExists(homeDir)) @@ -1845,7 +1838,7 @@ void LocalDerivationGoal::runChild() /* Unshare the cgroup namespace. This means /proc/self/cgroup will show the child's cgroup as '/' rather than whatever it is in the parent. */ - if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1) + if (buildUser && buildUser->getUIDCount() != 1 && unshare(CLONE_NEWCGROUP) == -1) throw SysError("unsharing cgroup namespace"); /* Do the chroot(). */ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 61b0f9145..070ae53f3 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -41,10 +41,6 @@ struct LocalDerivationGoal : public DerivationGoal Path chrootRootDir; - /* Whether to make the 'systemd' cgroup controller available to - the build. */ - bool useSystemdCgroup = false; - /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 670079019..0f05f3752 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -15,7 +15,6 @@ std::map stringifiedXpFeatures = { { Xp::FetchClosure, "fetch-closure" }, { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::SystemdCgroup, "systemd-cgroup" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index c749d4767..cf0c06eac 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -24,7 +24,6 @@ enum struct ExperimentalFeature FetchClosure, ReplFlake, AutoAllocateUids, - SystemdCgroup, }; /** From 09f00dd4d01aa1b6866978d162022133e521614f Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 16 Nov 2022 16:49:49 +0100 Subject: [PATCH 18/87] Replace src/libutil/json.cc with nlohmann json generation --- src/libexpr/eval.cc | 144 +++++++------- src/libexpr/primops.cc | 13 +- src/libexpr/value-to-json.cc | 50 +++-- src/libexpr/value-to-json.hh | 7 +- src/libexpr/value.hh | 6 +- src/libstore/binary-cache-store.cc | 18 +- src/libstore/build/derivation-goal.cc | 1 - src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/nar-accessor.cc | 30 +-- src/libstore/nar-accessor.hh | 6 +- src/libstore/parsed-derivations.cc | 16 +- src/libstore/remote-fs-accessor.cc | 8 +- src/libstore/store-api.cc | 48 ++--- src/libstore/store-api.hh | 4 +- src/libutil/json.cc | 203 -------------------- src/libutil/json.hh | 185 ------------------ src/libutil/tests/json.cc | 193 ------------------- src/nix-env/nix-env.cc | 35 ++-- src/nix/eval.cc | 7 +- src/nix/flake.cc | 37 ++-- src/nix/ls.cc | 5 +- src/nix/make-content-addressed.cc | 12 +- src/nix/path-info.cc | 8 +- src/nix/search.cc | 19 +- src/nix/show-derivation.cc | 67 +++---- 25 files changed, 266 insertions(+), 858 deletions(-) delete mode 100644 src/libutil/json.cc delete mode 100644 src/libutil/json.hh delete mode 100644 src/libutil/tests/json.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 563f24e48..e78d28b97 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,7 +7,6 @@ #include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" -#include "json.hh" #include "function-trace.hh" #include @@ -21,6 +20,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -35,6 +35,8 @@ #endif +using json = nlohmann::json; + namespace nix { static char * allocString(size_t size) @@ -2441,97 +2443,97 @@ void EvalState::printStats() std::fstream fs; if (outPath != "-") fs.open(outPath, std::fstream::out); - JSONObject topObj(outPath == "-" ? std::cerr : fs, true); - topObj.attr("cpuTime",cpuTime); - { - auto envs = topObj.object("envs"); - envs.attr("number", nrEnvs); - envs.attr("elements", nrValuesInEnvs); - envs.attr("bytes", bEnvs); - } - { - auto lists = topObj.object("list"); - lists.attr("elements", nrListElems); - lists.attr("bytes", bLists); - lists.attr("concats", nrListConcats); - } - { - auto values = topObj.object("values"); - values.attr("number", nrValues); - values.attr("bytes", bValues); - } - { - auto syms = topObj.object("symbols"); - syms.attr("number", symbols.size()); - syms.attr("bytes", symbols.totalSize()); - } - { - auto sets = topObj.object("sets"); - sets.attr("number", nrAttrsets); - sets.attr("bytes", bAttrsets); - sets.attr("elements", nrAttrsInAttrsets); - } - { - auto sizes = topObj.object("sizes"); - sizes.attr("Env", sizeof(Env)); - sizes.attr("Value", sizeof(Value)); - sizes.attr("Bindings", sizeof(Bindings)); - sizes.attr("Attr", sizeof(Attr)); - } - topObj.attr("nrOpUpdates", nrOpUpdates); - topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); - topObj.attr("nrThunks", nrThunks); - topObj.attr("nrAvoided", nrAvoided); - topObj.attr("nrLookups", nrLookups); - topObj.attr("nrPrimOpCalls", nrPrimOpCalls); - topObj.attr("nrFunctionCalls", nrFunctionCalls); + json topObj = json::object(); + topObj["cpuTime"] = cpuTime; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - { - auto gc = topObj.object("gc"); - gc.attr("heapSize", heapSize); - gc.attr("totalBytes", totalBytes); - } + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + }; #endif if (countCalls) { + topObj["primops"] = primOpCalls; { - auto obj = topObj.object("primops"); - for (auto & i : primOpCalls) - obj.attr(i.first, i.second); - } - { - auto list = topObj.list("functions"); + auto& list = topObj["functions"]; + list = json::array(); for (auto & [fun, count] : functionCalls) { - auto obj = list.object(); + json obj = json::object(); if (fun->name) - obj.attr("name", (std::string_view) symbols[fun->name]); + obj["name"] = (std::string_view) symbols[fun->name]; else - obj.attr("name", nullptr); + obj["name"] = nullptr; if (auto pos = positions[fun->pos]) { - obj.attr("file", (std::string_view) pos.file); - obj.attr("line", pos.line); - obj.attr("column", pos.column); + obj["file"] = (std::string_view) pos.file; + obj["line"] = pos.line; + obj["column"] = pos.column; } - obj.attr("count", count); + obj["count"] = count; + list.push_back(obj); } } { - auto list = topObj.list("attributes"); + auto list = topObj["attributes"]; + list = json::array(); for (auto & i : attrSelects) { - auto obj = list.object(); + json obj = json::object(); if (auto pos = positions[i.first]) { - obj.attr("file", (const std::string &) pos.file); - obj.attr("line", pos.line); - obj.attr("column", pos.column); + obj["file"] = (const std::string &) pos.file; + obj["line"] = pos.line; + obj["column"] = pos.column; } - obj.attr("count", i.second); + obj["count"] = i.second; + list.push_back(obj); } } } if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - auto list = topObj.list("symbols"); - symbols.dump([&](const std::string & s) { list.elem(s); }); + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 22f6ad3cc..05265411c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -8,12 +8,12 @@ #include "references.hh" #include "store-api.hh" #include "util.hh" -#include "json.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" #include +#include #include #include @@ -1011,6 +1011,7 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val derivation. */ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { + using nlohmann::json; state.forceAttrs(*args[0], pos); /* Figure out the name first (for stack backtraces). */ @@ -1032,11 +1033,10 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } /* Check whether attributes should be passed as a JSON file. */ - std::ostringstream jsonBuf; - std::unique_ptr jsonObject; + std::optional jsonObject; attr = args[0]->attrs->find(state.sStructuredAttrs); if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) - jsonObject = std::make_unique(jsonBuf); + jsonObject = json::object(); /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; @@ -1138,8 +1138,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (i->name == state.sStructuredAttrs) continue; - auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, pos, placeholder, context); + (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, posDrvName); @@ -1183,8 +1182,8 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } if (jsonObject) { + drv.env.emplace("__json", jsonObject->dump()); jsonObject.reset(); - drv.env.emplace("__json", jsonBuf.str()); } /* Everything in the context of the strings in the derivation diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4d63d8b49..5dc453b2e 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,84 +1,82 @@ #include "value-to-json.hh" -#include "json.hh" #include "eval-inline.hh" #include "util.hh" #include #include +#include namespace nix { - -void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore) +using json = nlohmann::json; +json printValueAsJSON(EvalState & state, bool strict, + Value & v, const PosIdx pos, PathSet & context, bool copyToStore) { checkInterrupt(); if (strict) state.forceValue(v, pos); + json out; + switch (v.type()) { case nInt: - out.write(v.integer); + out = v.integer; break; case nBool: - out.write(v.boolean); + out = v.boolean; break; case nString: copyContext(v, context); - out.write(v.string.s); + out = v.string.s; break; case nPath: if (copyToStore) - out.write(state.copyPathToStore(context, v.path)); + out = state.copyPathToStore(context, v.path); else - out.write(v.path); + out = v.path; break; case nNull: - out.write(nullptr); break; case nAttrs: { auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { - out.write(*maybeString); + out = *maybeString; break; } auto i = v.attrs->find(state.sOutPath); if (i == v.attrs->end()) { - auto obj(out.object()); + out = json::object(); StringSet names; for (auto & j : *v.attrs) names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore); + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); } } else - printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore); + return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); break; } case nList: { - auto list(out.list()); - for (auto elem : v.listItems()) { - auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore); - } + out = json::array(); + for (auto elem : v.listItems()) + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); break; } case nExternal: - v.external->printValueAsJSON(state, strict, out, context, copyToStore); + return v.external->printValueAsJSON(state, strict, context, copyToStore); break; case nFloat: - out.write(v.fpoint); + out = v.fpoint; break; case nThunk: @@ -91,17 +89,17 @@ void printValueAsJSON(EvalState & state, bool strict, state.debugThrowLastTrace(e); throw e; } + return out; } void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) { - JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, pos, out, context, copyToStore); + str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } -void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context, bool copyToStore) const +json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, + PathSet & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 7ddc8a5b1..22f26b790 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -5,13 +5,12 @@ #include #include +#include namespace nix { -class JSONPlaceholder; - -void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true); +nlohmann::json printValueAsJSON(EvalState & state, bool strict, + Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 590ba7783..5adac72f8 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -7,6 +7,7 @@ #if HAVE_BOEHMGC #include #endif +#include namespace nix { @@ -62,7 +63,6 @@ class StorePath; class Store; class EvalState; class XMLWriter; -class JSONPlaceholder; typedef int64_t NixInt; @@ -98,8 +98,8 @@ class ExternalValueBase virtual bool operator ==(const ExternalValueBase & b) const; /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ - virtual void printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const; + virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, + PathSet & context, bool copyToStore = true) const; /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a26770c79..12d0c32fb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -9,7 +9,6 @@ #include "remote-fs-accessor.hh" #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" -#include "json.hh" #include "thread-pool.hh" #include "callback.hh" @@ -194,19 +193,12 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { - std::ostringstream jsonOut; + nlohmann::json j = { + {"version", 1}, + {"root", listNar(ref(narAccessor), "", true)}, + }; - { - JSONObject jsonRoot(jsonOut); - jsonRoot.attr("version", 1); - - { - auto res = jsonRoot.placeholder("root"); - listNar(res, ref(narAccessor), "", true); - } - } - - upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); } /* Optionally maintain an index of DWARF debug info files diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 00e375fe9..1938f4bcb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -7,7 +7,6 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" -#include "json.hh" #include "compression.hh" #include "worker-protocol.hh" #include "topo-sort.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 5cea3b590..c786e8613 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -8,7 +8,6 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" -#include "json.hh" #include "compression.hh" #include "daemon.hh" #include "worker-protocol.hh" @@ -56,6 +55,7 @@ #include #include +#include namespace nix { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 398147fc3..9a0003588 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -1,6 +1,5 @@ #include "nar-accessor.hh" #include "archive.hh" -#include "json.hh" #include #include @@ -243,42 +242,43 @@ ref makeLazyNarAccessor(const std::string & listing, return make_ref(listing, getNarBytes); } -void listNar(JSONPlaceholder & res, ref accessor, - const Path & path, bool recurse) +using nlohmann::json; +json listNar(ref accessor, const Path & path, bool recurse) { auto st = accessor->stat(path); - auto obj = res.object(); + json obj = json::object(); switch (st.type) { case FSAccessor::Type::tRegular: - obj.attr("type", "regular"); - obj.attr("size", st.fileSize); + obj["type"] = "regular"; + obj["size"] = st.fileSize; if (st.isExecutable) - obj.attr("executable", true); + obj["executable"] = true; if (st.narOffset) - obj.attr("narOffset", st.narOffset); + obj["narOffset"] = st.narOffset; break; case FSAccessor::Type::tDirectory: - obj.attr("type", "directory"); + obj["type"] = "directory"; { - auto res2 = obj.object("entries"); + obj["entries"] = json::object(); + json &res2 = obj["entries"]; for (auto & name : accessor->readDirectory(path)) { if (recurse) { - auto res3 = res2.placeholder(name); - listNar(res3, accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + "/" + name, true); } else - res2.object(name); + res2[name] = json::object(); } } break; case FSAccessor::Type::tSymlink: - obj.attr("type", "symlink"); - obj.attr("target", accessor->readLink(path)); + obj["type"] = "symlink"; + obj["target"] = accessor->readLink(path); break; default: throw Error("path '%s' does not exist in NAR", path); } + return obj; } } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index c2241a04c..7d998ae0b 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -2,6 +2,7 @@ #include +#include #include "fs-accessor.hh" namespace nix { @@ -24,11 +25,8 @@ ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -class JSONPlaceholder; - /* Write a JSON representation of the contents of a NAR (except file contents). */ -void listNar(JSONPlaceholder & res, ref accessor, - const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const Path & path, bool recurse); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index f2288a04e..59a30db10 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -2,7 +2,6 @@ #include #include -#include "json.hh" namespace nix { @@ -144,16 +143,11 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s auto e = json.find("exportReferencesGraph"); if (e != json.end() && e->is_object()) { for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(store.parseStorePath(p.get())); - store.pathInfoToJSON(jsonRoot, - store.exportReferences(storePaths, inputPaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(store.parseStorePath(p.get())); + json[i.key()] = store.pathInfoToJSON( + store.exportReferences(storePaths, inputPaths), false, true); } } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0ce335646..fcfb527f5 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,6 +1,6 @@ +#include #include "remote-fs-accessor.hh" #include "nar-accessor.hh" -#include "json.hh" #include #include @@ -38,10 +38,8 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - std::ostringstream str; - JSONPlaceholder jsonRoot(str); - listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(hashPart, "ls"), str.str()); + nlohmann::json j = listNar(narAccessor, "", true); + writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 06a9758fc..8811ab578 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,14 +6,16 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "json.hh" #include "url.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include #include +using json = nlohmann::json; + namespace nix { @@ -838,56 +840,53 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } - -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, +json Store::pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase, AllowInvalidFlag allowInvalid) { - auto jsonList = jsonOut.list(); + json::array_t jsonList = json::array(); for (auto & storePath : storePaths) { - auto jsonPath = jsonList.object(); + auto& jsonPath = jsonList.emplace_back(json::object()); try { auto info = queryPathInfo(storePath); - jsonPath.attr("path", printStorePath(info->path)); - jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) - .attr("narSize", info->narSize); + jsonPath["path"] = printStorePath(info->path); + jsonPath["narHash"] = info->narHash.to_string(hashBase, true); + jsonPath["narSize"] = info->narSize; { - auto jsonRefs = jsonPath.list("references"); + auto& jsonRefs = (jsonPath["references"] = json::array()); for (auto & ref : info->references) - jsonRefs.elem(printStorePath(ref)); + jsonRefs.emplace_back(printStorePath(ref)); } if (info->ca) - jsonPath.attr("ca", renderContentAddress(info->ca)); + jsonPath["ca"] = renderContentAddress(info->ca); std::pair closureSizes; if (showClosureSize) { closureSizes = getClosureSize(info->path); - jsonPath.attr("closureSize", closureSizes.first); + jsonPath["closureSize"] = closureSizes.first; } if (includeImpureInfo) { if (info->deriver) - jsonPath.attr("deriver", printStorePath(*info->deriver)); + jsonPath["deriver"] = printStorePath(*info->deriver); if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + jsonPath["registrationTime"] = info->registrationTime; if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + jsonPath["ultimate"] = info->ultimate; if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); for (auto & sig : info->sigs) - jsonSigs.elem(sig); + jsonPath["signatures"].push_back(sig); } auto narInfo = std::dynamic_pointer_cast( @@ -895,21 +894,22 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (narInfo) { if (!narInfo->url.empty()) - jsonPath.attr("url", narInfo->url); + jsonPath["url"] = narInfo->url; if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); + jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); if (narInfo->fileSize) - jsonPath.attr("downloadSize", narInfo->fileSize); + jsonPath["downloadSize"] = narInfo->fileSize; if (showClosureSize) - jsonPath.attr("closureDownloadSize", closureSizes.second); + jsonPath["closureDownloadSize"] = closureSizes.second; } } } catch (InvalidPath &) { - jsonPath.attr("path", printStorePath(storePath)); - jsonPath.attr("valid", false); + jsonPath["path"] = printStorePath(storePath); + jsonPath["valid"] = false; } } + return jsonList; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c8a667c6d..151ec10d6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -14,6 +14,7 @@ #include "path-info.hh" #include "repair-flag.hh" +#include #include #include #include @@ -68,7 +69,6 @@ struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; -class JSONPlaceholder; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; @@ -512,7 +512,7 @@ public: variable elements such as the registration time are included. If ‘showClosureSize’ is true, the closure size of each path is included. */ - void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, + nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); diff --git a/src/libutil/json.cc b/src/libutil/json.cc deleted file mode 100644 index 2f9e97ff5..000000000 --- a/src/libutil/json.cc +++ /dev/null @@ -1,203 +0,0 @@ -#include "json.hh" - -#include -#include -#include - -namespace nix { - -template<> -void toJSON(std::ostream & str, const std::string_view & s) -{ - constexpr size_t BUF_SIZE = 4096; - char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts - size_t bufPos = 0; - - const auto flush = [&] { - str.write(buf, bufPos); - bufPos = 0; - }; - const auto put = [&] (char c) { - buf[bufPos++] = c; - }; - - put('"'); - for (auto i = s.begin(); i != s.end(); i++) { - if (bufPos >= BUF_SIZE) flush(); - if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } - else if (*i == '\n') { put('\\'); put('n'); } - else if (*i == '\r') { put('\\'); put('r'); } - else if (*i == '\t') { put('\\'); put('t'); } - else if (*i >= 0 && *i < 32) { - const char hex[17] = "0123456789abcdef"; - put('\\'); - put('u'); - put(hex[(uint16_t(*i) >> 12) & 0xf]); - put(hex[(uint16_t(*i) >> 8) & 0xf]); - put(hex[(uint16_t(*i) >> 4) & 0xf]); - put(hex[(uint16_t(*i) >> 0) & 0xf]); - } - else put(*i); - } - put('"'); - flush(); -} - -void toJSON(std::ostream & str, const char * s) -{ - if (!s) str << "null"; else toJSON(str, std::string_view(s)); -} - -template<> void toJSON(std::ostream & str, const int & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned int & n) { str << n; } -template<> void toJSON(std::ostream & str, const long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } -template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const float & n) { str << n; } -template<> void toJSON(std::ostream & str, const double & n) { str << n; } -template<> void toJSON(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); } - -template<> void toJSON(std::ostream & str, const bool & b) -{ - str << (b ? "true" : "false"); -} - -template<> void toJSON(std::ostream & str, const std::nullptr_t & b) -{ - str << "null"; -} - -JSONWriter::JSONWriter(std::ostream & str, bool indent) - : state(new JSONState(str, indent)) -{ - state->stack++; -} - -JSONWriter::JSONWriter(JSONState * state) - : state(state) -{ - state->stack++; -} - -JSONWriter::~JSONWriter() -{ - if (state) { - assertActive(); - state->stack--; - if (state->stack == 0) delete state; - } -} - -void JSONWriter::comma() -{ - assertActive(); - if (first) { - first = false; - } else { - state->str << ','; - } - if (state->indent) indent(); -} - -void JSONWriter::indent() -{ - state->str << '\n' << std::string(state->depth * 2, ' '); -} - -void JSONList::open() -{ - state->depth++; - state->str << '['; -} - -JSONList::~JSONList() -{ - state->depth--; - if (state->indent && !first) indent(); - state->str << "]"; -} - -JSONList JSONList::list() -{ - comma(); - return JSONList(state); -} - -JSONObject JSONList::object() -{ - comma(); - return JSONObject(state); -} - -JSONPlaceholder JSONList::placeholder() -{ - comma(); - return JSONPlaceholder(state); -} - -void JSONObject::open() -{ - state->depth++; - state->str << '{'; -} - -JSONObject::~JSONObject() -{ - if (state) { - state->depth--; - if (state->indent && !first) indent(); - state->str << "}"; - } -} - -void JSONObject::attr(std::string_view s) -{ - comma(); - toJSON(state->str, s); - state->str << ':'; - if (state->indent) state->str << ' '; -} - -JSONList JSONObject::list(std::string_view name) -{ - attr(name); - return JSONList(state); -} - -JSONObject JSONObject::object(std::string_view name) -{ - attr(name); - return JSONObject(state); -} - -JSONPlaceholder JSONObject::placeholder(std::string_view name) -{ - attr(name); - return JSONPlaceholder(state); -} - -JSONList JSONPlaceholder::list() -{ - assertValid(); - first = false; - return JSONList(state); -} - -JSONObject JSONPlaceholder::object() -{ - assertValid(); - first = false; - return JSONObject(state); -} - -JSONPlaceholder::~JSONPlaceholder() -{ - if (first) { - assert(std::uncaught_exceptions()); - if (state->stack != 0) - write(nullptr); - } -} - -} diff --git a/src/libutil/json.hh b/src/libutil/json.hh deleted file mode 100644 index 3790b1a2e..000000000 --- a/src/libutil/json.hh +++ /dev/null @@ -1,185 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace nix { - -void toJSON(std::ostream & str, const char * s); - -template -void toJSON(std::ostream & str, const T & n); - -class JSONWriter -{ -protected: - - struct JSONState - { - std::ostream & str; - bool indent; - size_t depth = 0; - size_t stack = 0; - JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } - ~JSONState() - { - assert(stack == 0); - } - }; - - JSONState * state; - - bool first = true; - - JSONWriter(std::ostream & str, bool indent); - - JSONWriter(JSONState * state); - - ~JSONWriter(); - - void assertActive() - { - assert(state->stack != 0); - } - - void comma(); - - void indent(); -}; - -class JSONObject; -class JSONPlaceholder; - -class JSONList : JSONWriter -{ -private: - - friend class JSONObject; - friend class JSONPlaceholder; - - void open(); - - JSONList(JSONState * state) - : JSONWriter(state) - { - open(); - } - -public: - - JSONList(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } - - ~JSONList(); - - template - JSONList & elem(const T & v) - { - comma(); - toJSON(state->str, v); - return *this; - } - - JSONList list(); - - JSONObject object(); - - JSONPlaceholder placeholder(); -}; - -class JSONObject : JSONWriter -{ -private: - - friend class JSONList; - friend class JSONPlaceholder; - - void open(); - - JSONObject(JSONState * state) - : JSONWriter(state) - { - open(); - } - - void attr(std::string_view s); - -public: - - JSONObject(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } - - JSONObject(const JSONObject & obj) = delete; - - JSONObject(JSONObject && obj) - : JSONWriter(obj.state) - { - obj.state = 0; - } - - ~JSONObject(); - - template - JSONObject & attr(std::string_view name, const T & v) - { - attr(name); - toJSON(state->str, v); - return *this; - } - - JSONList list(std::string_view name); - - JSONObject object(std::string_view name); - - JSONPlaceholder placeholder(std::string_view name); -}; - -class JSONPlaceholder : JSONWriter -{ - -private: - - friend class JSONList; - friend class JSONObject; - - JSONPlaceholder(JSONState * state) - : JSONWriter(state) - { - } - - void assertValid() - { - assertActive(); - assert(first); - } - -public: - - JSONPlaceholder(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - } - - ~JSONPlaceholder(); - - template - void write(const T & v) - { - assertValid(); - first = false; - toJSON(state->str, v); - } - - JSONList list(); - - JSONObject object(); -}; - -} diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc deleted file mode 100644 index 156286999..000000000 --- a/src/libutil/tests/json.cc +++ /dev/null @@ -1,193 +0,0 @@ -#include "json.hh" -#include -#include - -namespace nix { - - /* ---------------------------------------------------------------------------- - * toJSON - * --------------------------------------------------------------------------*/ - - TEST(toJSON, quotesCharPtr) { - const char* input = "test"; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "\"test\""); - } - - TEST(toJSON, quotesStdString) { - std::string input = "test"; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "\"test\""); - } - - TEST(toJSON, convertsNullptrtoNull) { - auto input = nullptr; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "null"); - } - - TEST(toJSON, convertsNullToNull) { - const char* input = 0; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "null"); - } - - - TEST(toJSON, convertsFloat) { - auto input = 1.024f; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "1.024"); - } - - TEST(toJSON, convertsDouble) { - const double input = 1.024; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "1.024"); - } - - TEST(toJSON, convertsBool) { - auto input = false; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "false"); - } - - TEST(toJSON, quotesTab) { - std::stringstream out; - toJSON(out, "\t"); - - ASSERT_EQ(out.str(), "\"\\t\""); - } - - TEST(toJSON, quotesNewline) { - std::stringstream out; - toJSON(out, "\n"); - - ASSERT_EQ(out.str(), "\"\\n\""); - } - - TEST(toJSON, quotesCreturn) { - std::stringstream out; - toJSON(out, "\r"); - - ASSERT_EQ(out.str(), "\"\\r\""); - } - - TEST(toJSON, quotesCreturnNewLine) { - std::stringstream out; - toJSON(out, "\r\n"); - - ASSERT_EQ(out.str(), "\"\\r\\n\""); - } - - TEST(toJSON, quotesDoublequotes) { - std::stringstream out; - toJSON(out, "\""); - - ASSERT_EQ(out.str(), "\"\\\"\""); - } - - TEST(toJSON, substringEscape) { - std::stringstream out; - std::string_view s = "foo\t"; - toJSON(out, s.substr(3)); - - ASSERT_EQ(out.str(), "\"\\t\""); - } - - /* ---------------------------------------------------------------------------- - * JSONObject - * --------------------------------------------------------------------------*/ - - TEST(JSONObject, emptyObject) { - std::stringstream out; - { - JSONObject t(out); - } - ASSERT_EQ(out.str(), "{}"); - } - - TEST(JSONObject, objectWithList) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.list("list"); - l.elem("element"); - } - ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); - } - - TEST(JSONObject, objectWithListIndent) { - std::stringstream out; - { - JSONObject t(out, true); - auto l = t.list("list"); - l.elem("element"); - } - ASSERT_EQ(out.str(), -R"#({ - "list": [ - "element" - ] -})#"); - } - - TEST(JSONObject, objectWithPlaceholderAndList) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.placeholder("list"); - l.list().elem("element"); - } - - ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); - } - - TEST(JSONObject, objectWithPlaceholderAndObject) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.placeholder("object"); - l.object().attr("key", "value"); - } - - ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); - } - - /* ---------------------------------------------------------------------------- - * JSONList - * --------------------------------------------------------------------------*/ - - TEST(JSONList, empty) { - std::stringstream out; - { - JSONList l(out); - } - ASSERT_EQ(out.str(), R"#([])#"); - } - - TEST(JSONList, withElements) { - std::stringstream out; - { - JSONList l(out); - l.elem("one"); - l.object(); - l.placeholder().write("three"); - } - ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); - } -} - diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index fdd66220a..776c5f6db 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -12,7 +12,6 @@ #include "local-fs-store.hh" #include "user-env.hh" #include "util.hh" -#include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" @@ -26,6 +25,7 @@ #include #include #include +#include using namespace nix; using std::cout; @@ -911,43 +911,47 @@ static VersionDiff compareVersionAgainstSet( static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printMeta) { - JSONObject topObj(cout, true); + using nlohmann::json; + json topObj = json::object(); for (auto & i : elems) { try { if (i.hasFailed()) continue; - JSONObject pkgObj = topObj.object(i.attrPath); auto drvName = DrvName(i.queryName()); - pkgObj.attr("name", drvName.fullName); - pkgObj.attr("pname", drvName.name); - pkgObj.attr("version", drvName.version); - pkgObj.attr("system", i.querySystem()); - pkgObj.attr("outputName", i.queryOutputName()); + json &pkgObj = topObj[i.attrPath]; + pkgObj = { + {"name", drvName.fullName}, + {"pname", drvName.name}, + {"version", drvName.version}, + {"system", i.querySystem()}, + {"outputName", i.queryOutputName()}, + }; { DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); - JSONObject outputObj = pkgObj.object("outputs"); + json &outputObj = pkgObj["outputs"]; + outputObj = json::object(); for (auto & j : outputs) { if (j.second) - outputObj.attr(j.first, globals.state->store->printStorePath(*j.second)); + outputObj[j.first] = globals.state->store->printStorePath(*j.second); else - outputObj.attr(j.first, nullptr); + outputObj[j.first] = nullptr; } } if (printMeta) { - JSONObject metaObj = pkgObj.object("meta"); + json &metaObj = pkgObj["meta"]; + metaObj = json::object(); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - metaObj.attr(j, nullptr); + metaObj[j] = nullptr; } else { - auto placeholder = metaObj.placeholder(j); PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); + metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context); } } } @@ -958,6 +962,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin throw; } } + std::cout << topObj.dump(2); } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ddd2790c6..ba82b5772 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -4,10 +4,11 @@ #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" -#include "json.hh" #include "value-to-json.hh" #include "progress-bar.hh" +#include + using namespace nix; struct CmdEval : MixJSON, InstallableCommand @@ -115,9 +116,7 @@ struct CmdEval : MixJSON, InstallableCommand } else if (json) { - JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); - std::cout << std::endl; + std::cout << printValueAsJSON(*state, true, *v, pos, context, false).dump() << std::endl; } else { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3967f1102..336f6723a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -11,7 +11,6 @@ #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" -#include "json.hh" #include "eval-cache.hh" #include "markdown.hh" @@ -21,6 +20,7 @@ using namespace nix; using namespace nix::flake; +using json = nlohmann::json; class FlakeCommand : virtual Args, public MixFlakeOptions { @@ -917,35 +917,44 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun { auto flake = lockFlake(); - auto jsonRoot = json ? std::optional(std::cout) : std::nullopt; - StorePathSet sources; sources.insert(flake.flake.sourceInfo->storePath); - if (jsonRoot) - jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); // FIXME: use graph output, handle cycles. - std::function & jsonObj)> traverse; - traverse = [&](const Node & node, std::optional & jsonObj) + std::function traverse; + traverse = [&](const Node & node) { - auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional(); + nlohmann::json jsonObj2 = json ? json::object() : nlohmann::json(nullptr); for (auto & [inputName, input] : node.inputs) { if (auto inputNode = std::get_if<0>(&input)) { - auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional(); auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) : (*inputNode)->lockedRef.input.fetch(store).first.storePath; - if (jsonObj3) - jsonObj3->attr("path", store->printStorePath(storePath)); - sources.insert(std::move(storePath)); - traverse(**inputNode, jsonObj3); + if (json) { + auto& jsonObj3 = jsonObj2[inputName]; + jsonObj3["path"] = store->printStorePath(storePath); + sources.insert(std::move(storePath)); + jsonObj3["inputs"] = traverse(**inputNode); + } else { + sources.insert(std::move(storePath)); + traverse(**inputNode); + } } } + return jsonObj2; }; - traverse(*flake.lockFile.root, jsonRoot); + if (json) { + nlohmann::json jsonRoot = { + {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"inputs", traverse(*flake.lockFile.root)}, + }; + std::cout << jsonRoot.dump() << std::endl; + } else { + traverse(*flake.lockFile.root); + } if (!dryRun && !dstUri.empty()) { ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 07554994b..e964b01b3 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -3,7 +3,7 @@ #include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" -#include "json.hh" +#include using namespace nix; @@ -91,10 +91,9 @@ struct MixLs : virtual Args, MixJSON if (path == "/") path = ""; if (json) { - JSONPlaceholder jsonRoot(std::cout); if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - listNar(jsonRoot, accessor, path, recursive); + std::cout << listNar(accessor, path, recursive); } else listText(accessor); } diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index 34860c38f..f2e4cefbe 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -2,10 +2,13 @@ #include "store-api.hh" #include "make-content-addressed.hh" #include "common-args.hh" -#include "json.hh" + +#include using namespace nix; +using nlohmann::json; + struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON { CmdMakeContentAddressed() @@ -25,6 +28,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, ; } + using StorePathsCommand::run; void run(ref srcStore, StorePaths && storePaths) override { auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); @@ -33,13 +37,13 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, StorePathSet(storePaths.begin(), storePaths.end())); if (json) { - JSONObject jsonRoot(std::cout); - JSONObject jsonRewrites(jsonRoot.object("rewrites")); + nlohmann::json jsonRewrites = json::object(); for (auto & path : storePaths) { auto i = remappings.find(path); assert(i != remappings.end()); - jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second)); + jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second); } + std::cout << json::object({"rewrites", jsonRewrites}).dump(); } else { for (auto & path : storePaths) { auto i = remappings.find(path); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index d690fe594..613c5b191 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -1,12 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "json.hh" #include "common-args.hh" #include #include +#include + using namespace nix; struct CmdPathInfo : StorePathsCommand, MixJSON @@ -86,11 +87,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - JSONPlaceholder jsonRoot(std::cout); - store->pathInfoToJSON(jsonRoot, + std::cout << store->pathInfoToJSON( // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid); + true, showClosureSize, SRI, AllowInvalid).dump(); } else { diff --git a/src/nix/search.cc b/src/nix/search.cc index bdd45cbed..d2a31607d 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -5,7 +5,6 @@ #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" -#include "json.hh" #include "shared.hh" #include "eval-cache.hh" #include "attr-path.hh" @@ -13,8 +12,10 @@ #include #include +#include using namespace nix; +using json = nlohmann::json; std::string wrap(std::string prefix, std::string s) { @@ -84,7 +85,8 @@ struct CmdSearch : InstallableCommand, MixJSON auto state = getEvalState(); - auto jsonOut = json ? std::make_unique(std::cout) : nullptr; + std::optional jsonOut; + if (json) jsonOut = json::object(); uint64_t results = 0; @@ -151,10 +153,11 @@ struct CmdSearch : InstallableCommand, MixJSON { results++; if (json) { - auto jsonElem = jsonOut->object(attrPath2); - jsonElem.attr("pname", name.name); - jsonElem.attr("version", name.version); - jsonElem.attr("description", description); + (*jsonOut)[attrPath2] = { + {"pname", name.name}, + {"version", name.version}, + {"description", description}, + }; } else { auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); @@ -193,6 +196,10 @@ struct CmdSearch : InstallableCommand, MixJSON for (auto & cursor : installable->getCursors(*state)) visit(*cursor, cursor->getAttrPath(), true); + if (json) { + std::cout << jsonOut->dump() << std::endl; + } + if (!json && !results) throw Error("no results for the given search term(s)!"); } diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index fb46b4dbf..af2e676a4 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -5,10 +5,11 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" -#include "json.hh" #include "derivations.hh" +#include using namespace nix; +using json = nlohmann::json; struct CmdShowDerivation : InstallablesCommand { @@ -48,77 +49,63 @@ struct CmdShowDerivation : InstallablesCommand drvPaths = std::move(closure); } - { - - JSONObject jsonRoot(std::cout, true); + json jsonRoot = json::object(); for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); + json& drvObj = jsonRoot[store->printStorePath(drvPath)]; auto drv = store->readDerivation(drvPath); { - auto outputsObj(drvObj.object("outputs")); + json& outputsObj = drvObj["outputs"]; + outputsObj = json::object(); for (auto & [_outputName, output] : drv.outputs) { auto & outputName = _outputName; // work around clang bug - auto outputObj { outputsObj.object(outputName) }; + auto& outputObj = outputsObj[outputName]; + outputObj = json::object(); std::visit(overloaded { [&](const DerivationOutput::InputAddressed & doi) { - outputObj.attr("path", store->printStorePath(doi.path)); + outputObj["path"] = store->printStorePath(doi.path); }, [&](const DerivationOutput::CAFixed & dof) { - outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); - outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); - outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); + outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName)); + outputObj["hashAlgo"] = dof.hash.printMethodAlgo(); + outputObj["hash"] = dof.hash.hash.to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { - outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + outputObj["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); - outputObj.attr("impure", true); + outputObj["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); + outputObj["impure"] = true; }, }, output.raw()); } } { - auto inputsList(drvObj.list("inputSrcs")); + auto& inputsList = drvObj["inputSrcs"]; + inputsList = json::array(); for (auto & input : drv.inputSrcs) - inputsList.elem(store->printStorePath(input)); + inputsList.emplace_back(store->printStorePath(input)); } { - auto inputDrvsObj(drvObj.object("inputDrvs")); - for (auto & input : drv.inputDrvs) { - auto inputList(inputDrvsObj.list(store->printStorePath(input.first))); - for (auto & outputId : input.second) - inputList.elem(outputId); - } + auto& inputDrvsObj = drvObj["inputDrvs"]; + inputDrvsObj = json::object(); + for (auto & input : drv.inputDrvs) + inputDrvsObj[store->printStorePath(input.first)] = input.second; } - drvObj.attr("system", drv.platform); - drvObj.attr("builder", drv.builder); - - { - auto argsList(drvObj.list("args")); - for (auto & arg : drv.args) - argsList.elem(arg); - } - - { - auto envObj(drvObj.object("env")); - for (auto & var : drv.env) - envObj.attr(var.first, var.second); - } + drvObj["system"] = drv.platform; + drvObj["builder"] = drv.builder; + drvObj["args"] = drv.args; + drvObj["env"] = drv.env; } - - } - - std::cout << "\n"; + std::cout << jsonRoot.dump(2) << std::endl; } }; From f423d4425f6573206045e9626812002906d9493d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Nov 2022 11:56:45 +0100 Subject: [PATCH 19/87] Fix segfault in unprivileged mode --- src/libstore/build/local-derivation-goal.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 070ae53f3..f92280aa1 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -92,8 +92,8 @@ struct LocalDerivationGoal : public DerivationGoal result. */ std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } const static Path homeDir; From f1ab082ac4f589a36a9eb0cd98d1cc235eedc419 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 09:37:11 +0100 Subject: [PATCH 20/87] createTempDir(): Use std::atomic --- src/libutil/filesystem.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 403389e60..3a732cff8 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,5 +1,6 @@ #include #include +#include #include "finally.hh" #include "util.hh" @@ -10,7 +11,7 @@ namespace fs = std::filesystem; namespace nix { static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) + std::atomic & counter) { tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); if (includePid) @@ -22,9 +23,9 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, Path createTempDir(const Path & tmpRoot, const Path & prefix, bool includePid, bool useGlobalCounter, mode_t mode) { - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); while (1) { checkInterrupt(); From 128910ba23f586ba1765a137ecff23cfd22cff89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 10:39:28 +0100 Subject: [PATCH 21/87] Separate cgroup support from auto-uid-allocation The new experimental feature 'cgroups' enables the use of cgroups for all builds. This allows better containment and enables setting resource limits and getting some build stats. --- src/libstore/build/local-derivation-goal.cc | 114 ++++++++++++++------ src/libstore/build/local-derivation-goal.hh | 7 ++ src/libstore/lock.cc | 71 ------------ src/libstore/lock.hh | 7 -- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + 6 files changed, 93 insertions(+), 108 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e652c425c..2d1e093ca 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,6 +15,7 @@ #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" +#include "cgroup.hh" #include #include @@ -129,26 +130,36 @@ void LocalDerivationGoal::killChild() if (pid != -1) { worker.childTerminated(this); - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ - assert(pid == -1); + killSandbox(); + + pid.wait(); } DerivationGoal::killChild(); } +void LocalDerivationGoal::killSandbox() +{ + if (cgroup) { + destroyCgroup(*cgroup); + } + + else if (buildUser) { + auto uid = buildUser->getUID(); + assert(uid != 0); + killUser(uid); + } +} + + void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { @@ -169,10 +180,6 @@ void LocalDerivationGoal::tryLocalBuild() { worker.waitForAWhile(shared_from_this()); return; } - - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); } actLock.reset(); @@ -263,7 +270,7 @@ void LocalDerivationGoal::cleanupPostChildKill() malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - if (buildUser) buildUser->kill(); + killSandbox(); /* Terminate the recursive Nix daemon. */ stopDaemon(); @@ -356,6 +363,55 @@ static void linkOrCopy(const Path & from, const Path & to) void LocalDerivationGoal::startBuilder() { + if ((buildUser && buildUser->getUIDCount() != 1) + || settings.isExperimentalFeatureEnabled(Xp::Cgroups)) + { + #if __linux__ + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto ourCgroup = ourCgroups[""]; + if (ourCgroup == "") + throw Error("cannot determine cgroup name from /proc/self/cgroup"); + + auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); + + if (!pathExists(ourCgroupPath)) + throw Error("expected cgroup directory '%s'", ourCgroupPath); + + static std::atomic counter{0}; + + cgroup = buildUser + ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID()) + : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++); + + debug("using cgroup '%s'", *cgroup); + + /* When using a build user, record the cgroup we used for that + user so that if we got interrupted previously, we can kill + any left-over cgroup first. */ + if (buildUser) { + auto cgroupsDir = settings.nixStateDir + "/cgroups"; + createDirs(cgroupsDir); + + auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); + + if (pathExists(cgroupFile)) { + auto prevCgroup = readFile(cgroupFile); + destroyCgroup(prevCgroup); + } + + writeFile(cgroupFile, *cgroup); + } + + #else + throw Error("cgroups are not supported on this platform"); + #endif + } + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + killSandbox(); + /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", @@ -646,13 +702,13 @@ void LocalDerivationGoal::startBuilder() dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); } - if (buildUser) { - if (auto cgroup = buildUser->getCgroup()) { - chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); - chownToBuilder(*cgroup + "/cgroup.threads"); - //chownToBuilder(*cgroup + "/cgroup.subtree_control"); - } + if (cgroup) { + if (mkdir(cgroup->c_str(), 0755) != 0) + throw SysError("creating cgroup '%s'", *cgroup); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); } #else @@ -965,10 +1021,8 @@ void LocalDerivationGoal::startBuilder() } /* Move the child into its own cgroup. */ - if (buildUser) { - if (auto cgroup = buildUser->getCgroup()) - writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); - } + if (cgroup) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); @@ -1838,7 +1892,7 @@ void LocalDerivationGoal::runChild() /* Unshare the cgroup namespace. This means /proc/self/cgroup will show the child's cgroup as '/' rather than whatever it is in the parent. */ - if (buildUser && buildUser->getUIDCount() != 1 && unshare(CLONE_NEWCGROUP) == -1) + if (cgroup && unshare(CLONE_NEWCGROUP) == -1) throw SysError("unsharing cgroup namespace"); /* Do the chroot(). */ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index f92280aa1..1ec6b3649 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -15,6 +15,9 @@ struct LocalDerivationGoal : public DerivationGoal /* The process ID of the builder. */ Pid pid; + /* The cgroup of the builder, if any. */ + std::optional cgroup; + /* The temporary directory. */ Path tmpDir; @@ -197,6 +200,10 @@ struct LocalDerivationGoal : public DerivationGoal /* Forcibly kill the child process, if any. */ void killChild() override; + /* Kill any processes running under the build user UID or in the + cgroup of the build. */ + void killSandbox(); + /* Create alternative path calculated from but distinct from the input, so we can avoid overwriting outputs (or other store paths) that already exist. */ diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index f9892bb91..4fad3bfd2 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,7 +1,6 @@ #include "lock.hh" #include "globals.hh" #include "pathlocks.hh" -#include "cgroup.hh" #include #include @@ -15,11 +14,6 @@ struct SimpleUserLock : UserLock gid_t gid; std::vector supplementaryGIDs; - void kill() override - { - killUser(uid); - } - uid_t getUID() override { assert(uid); return uid; } uid_t getUIDCount() override { return 1; } gid_t getGID() override { assert(gid); return gid; } @@ -116,32 +110,6 @@ struct AutoUserLock : UserLock AutoCloseFD fdUserLock; uid_t firstUid = 0; uid_t nrIds = 1; - #if __linux__ - std::optional cgroup; - #endif - - ~AutoUserLock() - { - #if __linux__ - // Get rid of our cgroup, ignoring errors. - if (cgroup) rmdir(cgroup->c_str()); - #endif - } - - void kill() override - { - #if __linux__ - if (cgroup) { - destroyCgroup(*cgroup); - if (mkdir(cgroup->c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", *cgroup); - } else - #endif - { - assert(firstUid); - killUser(firstUid); - } - } uid_t getUID() override { assert(firstUid); return firstUid; } @@ -183,55 +151,16 @@ struct AutoUserLock : UserLock throw SysError("opening user lock '%s'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { - auto s = drainFD(fd.get()); - - #if __linux__ - if (s != "") { - /* Kill the old cgroup, to ensure there are no - processes left over from an interrupted build. */ - destroyCgroup(s); - } - #endif - - if (ftruncate(fd.get(), 0) == -1) - throw Error("truncating user lock"); - auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); lock->firstUid = settings.startId + i * maxIdsPerBuild; lock->nrIds = nrIds; - - #if __linux__ - if (nrIds > 1) { - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto ourCgroup = ourCgroups[""]; - if (ourCgroup == "") - throw Error("cannot determine cgroup name from /proc/self/cgroup"); - - auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); - - if (!pathExists(ourCgroupPath)) - throw Error("expected cgroup directory '%s'", ourCgroupPath); - - lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid); - - /* Record the cgroup in the lock file. This ensures that - if we subsequently get executed under a different parent - cgroup, we kill the previous cgroup first. */ - writeFull(lock->fdUserLock.get(), *lock->cgroup); - } - #endif - return lock; } } return nullptr; } - - #if __linux__ - std::optional getCgroup() override { return cgroup; } - #endif }; std::unique_ptr acquireUserLock(uid_t nrIds) diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index b5536408c..e7ceefab8 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -27,13 +27,6 @@ struct UserLock virtual gid_t getGID() = 0; virtual std::vector getSupplementaryGIDs() = 0; - - /* Kill any processes currently executing as this user. */ - virtual void kill() = 0; - - #if __linux__ - virtual std::optional getCgroup() { return {}; }; - #endif }; /* Acquire a user lock for a UID range of size `nrIds`. Note that this diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 0f05f3752..e0902971e 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -15,6 +15,7 @@ std::map stringifiedXpFeatures = { { Xp::FetchClosure, "fetch-closure" }, { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, + { Xp::Cgroups, "cgroups" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index cf0c06eac..af775feb0 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -24,6 +24,7 @@ enum struct ExperimentalFeature FetchClosure, ReplFlake, AutoAllocateUids, + Cgroups, }; /** From 20f66c6889aa9d907feee4946702d655b6bd796f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 13:40:48 +0100 Subject: [PATCH 22/87] Indentation --- src/libcmd/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index e097f23b3..5945a2578 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -927,7 +927,7 @@ std::vector, BuiltPath>> Installable::bui case Realise::Outputs: { if (settings.printMissing) - printMissing(store, pathsToBuild, lvlInfo); + printMissing(store, pathsToBuild, lvlInfo); for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { if (!buildResult.success()) From fa68eb367e79297bb1c0451cd92ad18a06edce96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 13:40:59 +0100 Subject: [PATCH 23/87] Get CPU stats from the cgroup --- src/libstore/build-result.hh | 5 ++- src/libstore/build/derivation-goal.cc | 8 +++++ src/libstore/build/local-derivation-goal.cc | 14 +++++--- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/cgroup.cc | 39 +++++++++++++++++++-- src/libstore/cgroup.hh | 14 +++++++- 6 files changed, 71 insertions(+), 11 deletions(-) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 24fb1f763..a5749cf33 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -5,7 +5,7 @@ #include #include - +#include namespace nix { @@ -78,6 +78,9 @@ struct BuildResult was repeated). */ time_t startTime = 0, stopTime = 0; + /* User and system CPU time the build took. */ + std::optional cpuUser, cpuSystem; + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 41d2e2a1c..9bc3dc742 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -869,6 +869,14 @@ void DerivationGoal::buildDone() cleanupPostChildKill(); + if (buildResult.cpuUser && buildResult.cpuSystem) { + debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", + worker.store.printStorePath(drvPath), + status, + ((double) buildResult.cpuUser->count()) / 1000000, + ((double) buildResult.cpuSystem->count()) / 1000000); + } + bool diskFull = false; try { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2d1e093ca..f273ebe8a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -137,7 +137,7 @@ void LocalDerivationGoal::killChild() also send a conventional kill to the child. */ ::kill(-pid, SIGKILL); /* ignore the result */ - killSandbox(); + killSandbox(true); pid.wait(); } @@ -146,10 +146,14 @@ void LocalDerivationGoal::killChild() } -void LocalDerivationGoal::killSandbox() +void LocalDerivationGoal::killSandbox(bool getStats) { if (cgroup) { - destroyCgroup(*cgroup); + auto stats = destroyCgroup(*cgroup); + if (getStats) { + buildResult.cpuUser = stats.cpuUser; + buildResult.cpuSystem = stats.cpuSystem; + } } else if (buildUser) { @@ -270,7 +274,7 @@ void LocalDerivationGoal::cleanupPostChildKill() malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - killSandbox(); + killSandbox(true); /* Terminate the recursive Nix daemon. */ stopDaemon(); @@ -410,7 +414,7 @@ void LocalDerivationGoal::startBuilder() /* Make sure that no other processes are executing under the sandbox uids. This must be done before any chownToBuilder() calls. */ - killSandbox(); + killSandbox(false); /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1ec6b3649..34c4e9187 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -202,7 +202,7 @@ struct LocalDerivationGoal : public DerivationGoal /* Kill any processes running under the build user UID or in the cgroup of the build. */ - void killSandbox(); + void killSandbox(bool getStats); /* Create alternative path calculated from but distinct from the input, so we can avoid overwriting outputs (or other store paths) diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 56e980be3..2a485f0f9 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -31,13 +31,16 @@ std::map getCgroups(const Path & cgroupFile) return cgroups; } -void destroyCgroup(const Path & cgroup) +static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) { - if (!pathExists(cgroup)) return; + if (!pathExists(cgroup)) return {}; + + if (!pathExists(cgroup + "/cgroup.procs")) + throw Error("'%s' is not a cgroup", cgroup); for (auto & entry : readDirectory(cgroup)) { if (entry.type != DT_DIR) continue; - destroyCgroup(cgroup + "/" + entry.name); + destroyCgroup(cgroup + "/" + entry.name, false); } int round = 1; @@ -79,8 +82,38 @@ void destroyCgroup(const Path & cgroup) round++; } + CgroupStats stats; + + if (returnStats) { + auto cpustatPath = cgroup + "/cpu.stat"; + + if (pathExists(cpustatPath)) { + for (auto & line : tokenizeString>(readFile(cpustatPath), "\n")) { + std::string_view userPrefix = "user_usec "; + if (hasPrefix(line, userPrefix)) { + auto n = string2Int(line.substr(userPrefix.size())); + if (n) stats.cpuUser = std::chrono::microseconds(*n); + } + + std::string_view systemPrefix = "system_usec "; + if (hasPrefix(line, systemPrefix)) { + auto n = string2Int(line.substr(systemPrefix.size())); + if (n) stats.cpuSystem = std::chrono::microseconds(*n); + } + } + } + + } + if (rmdir(cgroup.c_str()) == -1) throw SysError("deleting cgroup '%s'", cgroup); + + return stats; +} + +CgroupStats destroyCgroup(const Path & cgroup) +{ + return destroyCgroup(cgroup, true); } } diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh index dc6758957..3ead4735f 100644 --- a/src/libstore/cgroup.hh +++ b/src/libstore/cgroup.hh @@ -2,13 +2,25 @@ #if __linux__ +#include +#include + #include "types.hh" namespace nix { std::map getCgroups(const Path & cgroupFile); -void destroyCgroup(const Path & cgroup); +struct CgroupStats +{ + std::optional cpuUser, cpuSystem; +}; + +/* Destroy the cgroup denoted by 'path'. The postcondition is that + 'path' does not exist, and thus any processes in the cgroup have + been killed. Also return statistics from the cgroup just before + destruction. */ +CgroupStats destroyCgroup(const Path & cgroup); } From e6b71f84a0a766429fdceaf188ea0167e36a20d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 16:59:36 +0100 Subject: [PATCH 24/87] Use cgroup.kill to quickly kill cgroups --- src/libstore/cgroup.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 2a485f0f9..f693d77be 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -35,9 +35,19 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) { if (!pathExists(cgroup)) return {}; - if (!pathExists(cgroup + "/cgroup.procs")) + auto procsFile = cgroup + "/cgroup.procs"; + + if (!pathExists(procsFile)) throw Error("'%s' is not a cgroup", cgroup); + /* Use the fast way to kill every process in a cgroup, if + available. */ + auto killFile = cgroup + "/cgroup.kill"; + if (pathExists(killFile)) + writeFile(killFile, "1"); + + /* Otherwise, manually kill every process in the subcgroups and + this cgroup. */ for (auto & entry : readDirectory(cgroup)) { if (entry.type != DT_DIR) continue; destroyCgroup(cgroup + "/" + entry.name, false); @@ -48,7 +58,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) std::unordered_set pidsShown; while (true) { - auto pids = tokenizeString>(readFile(cgroup + "/cgroup.procs")); + auto pids = tokenizeString>(readFile(procsFile)); if (pids.empty()) break; From cc620d961f1fa68038ef44f5947fc053708ab3ad Mon Sep 17 00:00:00 2001 From: Liu Xiaoyi Date: Sun, 20 Nov 2022 20:04:22 +0800 Subject: [PATCH 25/87] Updated uninstall guide involving systemd Co-authored-by: sequencer --- .../src/installation/installing-binary.md | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 31faeadc2..1b7db59cd 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -88,6 +88,28 @@ extension. The installer will also create `/etc/profile.d/nix.sh`. ### Linux +To begin, if you are on Linux with systemd, remove the Nix daemon service: + +```console +sudo systemctl stop nix-daemon.socket +sudo systemctl stop nix-daemon.service +sudo systemctl disable nix-daemon.socket +sudo systemctl disable nix-daemon.service +sudo systemctl daemon-reload +``` + +Then you can remove systemd service files: + +```console +sudo rm -f /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket +``` + +Also, the installer script uses systemd-tmpfiles (if presents) to create the socket directory. You may also want to remove the configuration for that: + +```console +sudo rm -f /etc/tmpfiles.d/nix-daemon.conf +``` + Remove files created by Nix: ```console @@ -103,16 +125,6 @@ done sudo groupdel 30000 ``` -If you are on Linux with systemd, remove the Nix daemon service: - -```console -sudo systemctl stop nix-daemon.socket -sudo systemctl stop nix-daemon.service -sudo systemctl disable nix-daemon.socket -sudo systemctl disable nix-daemon.service -sudo systemctl daemon-reload -``` - There may also be references to Nix in - `/etc/profile` From 4d55acf5154639cd1a54435ace28cd497b5a9eb8 Mon Sep 17 00:00:00 2001 From: Liu Xiaoyi Date: Mon, 21 Nov 2022 13:46:22 +0800 Subject: [PATCH 26/87] Apply suggestions from code review Co-authored-by: Sandro --- doc/manual/src/installation/installing-binary.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 1b7db59cd..8cdd64df6 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -91,23 +91,21 @@ extension. The installer will also create `/etc/profile.d/nix.sh`. To begin, if you are on Linux with systemd, remove the Nix daemon service: ```console -sudo systemctl stop nix-daemon.socket sudo systemctl stop nix-daemon.service -sudo systemctl disable nix-daemon.socket -sudo systemctl disable nix-daemon.service +sudo systemctl disable nix-daemon.socket nix-daemon.service sudo systemctl daemon-reload ``` Then you can remove systemd service files: ```console -sudo rm -f /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket +sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket ``` Also, the installer script uses systemd-tmpfiles (if presents) to create the socket directory. You may also want to remove the configuration for that: ```console -sudo rm -f /etc/tmpfiles.d/nix-daemon.conf +sudo rm /etc/tmpfiles.d/nix-daemon.conf ``` Remove files created by Nix: From f538ee434285304cb61cf10bf13127f13bfced1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 09:38:08 +0100 Subject: [PATCH 27/87] Rename derivedPathsWithHintsToJSON -> builtPathsToJSON --- src/libstore/derived-path.cc | 2 +- src/libstore/derived-path.hh | 2 +- src/nix/build.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 44587ae78..7fe9b9648 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -64,7 +64,7 @@ nlohmann::json stuffToJSON(const std::vector & ts, ref store) { return res; } -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) +nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store) { return stuffToJSON(buildables, store); } nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) { return stuffToJSON(paths, store); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 24a0ae773..b2d0956b8 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -125,7 +125,7 @@ struct BuiltPath : _BuiltPathRaw { typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); +nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store); nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); } diff --git a/src/nix/build.cc b/src/nix/build.cc index 9c648d28e..2b91f8e0a 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,7 +78,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile Realise::Outputs, installables, buildMode); - if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsToJSON(buildables, store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) From 300753d594fd7cd818d08f9c7a18a9ebc305bd95 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 10:49:01 +0100 Subject: [PATCH 28/87] nix build --json: Include build statistics Example: # nix build -L --extra-experimental-features cgroups --impure --expr 'with import {}; runCommand "foo" {} "dd if=/dev/urandom bs=1M count=1024 | md5sum; mkdir $out"' --json [ { "cpuSystem": 1.911431, "cpuUser": 1.214249, "drvPath": "/nix/store/xzdqz67xba18hljhycp0hwfigzrs2z69-foo.drv", "outputs": { "out": "/nix/store/rh9mc9l2gkpq8kn2sgzndr6ll7ffjh6l-foo" }, "startTime": 1669024076, "stopTime": 1669024079 } ] --- src/libcmd/installables.cc | 29 ++++++++++++++----------- src/libcmd/installables.hh | 11 ++++++++-- src/libstore/derived-path.cc | 23 ++++---------------- src/libstore/derived-path.hh | 3 --- src/nix/app.cc | 11 +++++++--- src/nix/build.cc | 42 ++++++++++++++++++++++++++++++++---- src/nix/profile.cc | 4 ++-- 7 files changed, 77 insertions(+), 46 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5945a2578..e036b8836 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -840,20 +840,20 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -BuiltPaths Installable::build( +std::vector Installable::build( ref evalStore, ref store, Realise mode, const std::vector> & installables, BuildMode bMode) { - BuiltPaths res; - for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode)) - res.push_back(builtPath); + std::vector res; + for (auto & [_, builtPathWithResult] : build2(evalStore, store, mode, installables, bMode)) + res.push_back(builtPathWithResult); return res; } -std::vector, BuiltPath>> Installable::build2( +std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, Realise mode, @@ -873,7 +873,7 @@ std::vector, BuiltPath>> Installable::bui } } - std::vector, BuiltPath>> res; + std::vector, BuiltPathWithResult>> res; switch (mode) { @@ -914,10 +914,10 @@ std::vector, BuiltPath>> Installable::bui output, *drvOutput->second); } } - res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, BuiltPath::Opaque { bo.path }}); + res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); }, }, path.raw()); } @@ -939,10 +939,10 @@ std::vector, BuiltPath>> Installable::bui std::map outputs; for (auto & path : buildResult.builtOutputs) outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, BuiltPath::Opaque { bo.path }}); + res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -965,9 +965,12 @@ BuiltPaths Installable::toBuiltPaths( OperateOn operateOn, const std::vector> & installables) { - if (operateOn == OperateOn::Output) - return Installable::build(evalStore, store, mode, installables); - else { + if (operateOn == OperateOn::Output) { + BuiltPaths res; + for (auto & p : Installable::build(evalStore, store, mode, installables)) + res.push_back(p.path); + return res; + } else { if (mode == Realise::Nothing) settings.readOnlyMode = true; diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 948f78919..02ea351d3 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -7,6 +7,7 @@ #include "eval.hh" #include "store-api.hh" #include "flake/flake.hh" +#include "build-result.hh" #include @@ -51,6 +52,12 @@ enum class OperateOn { Derivation }; +struct BuiltPathWithResult +{ + BuiltPath path; + std::optional result; +}; + struct Installable { virtual ~Installable() { } @@ -91,14 +98,14 @@ struct Installable return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); } - static BuiltPaths build( + static std::vector build( ref evalStore, ref store, Realise mode, const std::vector> & installables, BuildMode bMode = bmNormal); - static std::vector, BuiltPath>> build2( + static std::vector, BuiltPathWithResult>> build2( ref evalStore, ref store, Realise mode, diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 7fe9b9648..88b59f615 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -53,28 +53,13 @@ StorePathSet BuiltPath::outPaths() const ); } -template -nlohmann::json stuffToJSON(const std::vector & ts, ref store) { - auto res = nlohmann::json::array(); - for (const T & t : ts) { - std::visit([&res, store](const auto & t) { - res.push_back(t.toJSON(store)); - }, t.raw()); - } - return res; -} - -nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store) -{ return stuffToJSON(buildables, store); } -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) -{ return stuffToJSON(paths, store); } - - -std::string DerivedPath::Opaque::to_string(const Store & store) const { +std::string DerivedPath::Opaque::to_string(const Store & store) const +{ return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string(const Store & store) const +{ return store.printStorePath(drvPath) + "!" + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b2d0956b8..878696136 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -125,7 +125,4 @@ struct BuiltPath : _BuiltPathRaw { typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store); -nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); - } diff --git a/src/nix/app.cc b/src/nix/app.cc index 48de8fb82..5658f2a52 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -37,11 +37,13 @@ struct InstallableDerivedPath : Installable * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. */ -StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) +StringPairs resolveRewrites( + Store & store, + const std::vector & dependencies) { StringPairs res; for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep)) + if (auto drvDep = std::get_if(&dep.path)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( downstreamPlaceholder(store, drvDep->drvPath, outputName), @@ -53,7 +55,10 @@ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) /** * Resolve the given string assuming the given context. */ -std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) +std::string resolveString( + Store & store, + const std::string & toResolve, + const std::vector & dependencies) { auto rewrites = resolveRewrites(store, dependencies); return rewriteStrings(toResolve, rewrites); diff --git a/src/nix/build.cc b/src/nix/build.cc index 2b91f8e0a..94b169167 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -10,6 +10,37 @@ using namespace nix; +nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +{ + auto res = nlohmann::json::array(); + for (auto & t : paths) { + std::visit([&res, store](const auto & t) { + res.push_back(t.toJSON(store)); + }, t.raw()); + } + return res; +} + +nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +{ + auto res = nlohmann::json::array(); + for (auto & b : buildables) { + std::visit([&](const auto & t) { + auto j = t.toJSON(store); + if (b.result) { + j["startTime"] = b.result->startTime; + j["stopTime"] = b.result->stopTime; + if (b.result->cpuUser) + j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000; + if (b.result->cpuSystem) + j["cpuSystem"] = ((double) b.result->cpuSystem->count()) / 1000000; + } + res.push_back(j); + }, b.path.raw()); + } + return res; +} + struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; @@ -78,7 +109,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile Realise::Outputs, installables, buildMode); - if (json) logger->cout("%s", builtPathsToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) @@ -98,7 +129,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildable.raw()); + }, buildable.path.raw()); } if (printOutputPaths) { @@ -113,11 +144,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile std::cout << store->printStorePath(output.second) << std::endl; } }, - }, buildable.raw()); + }, buildable.path.raw()); } } - updateProfile(buildables); + BuiltPaths buildables2; + for (auto & b : buildables) + buildables2.push_back(b.path); + updateProfile(buildables2); } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 3814e7d5a..11910523d 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -253,11 +253,11 @@ struct ProfileManifest static std::map builtPathsPerInstallable( - const std::vector, BuiltPath>> & builtPaths) + const std::vector, BuiltPathWithResult>> & builtPaths) { std::map res; for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath); + res[installable.get()].push_back(builtPath.path); return res; } From ec45f4b82eaef8da04f4b828b2b06a77aa3f986f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 11:12:45 +0100 Subject: [PATCH 29/87] Fix indentation --- src/libstore/local-store.cc | 50 ++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 04223d860..b67668e52 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -897,48 +897,48 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, std::shared_ptr LocalStore::queryPathInfoInternal(State & state, const StorePath & path) { - /* Get the path info. */ + /* Get the path info. */ auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); - if (!useQueryPathInfo.next()) - return std::shared_ptr(); + if (!useQueryPathInfo.next()) + return std::shared_ptr(); - auto id = useQueryPathInfo.getInt(0); + auto id = useQueryPathInfo.getInt(0); - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } - auto info = std::make_shared(path, narHash); + auto info = std::make_shared(path, narHash); - info->id = id; + info->id = id; - info->registrationTime = useQueryPathInfo.getInt(2); + info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); + if (s) info->deriver = parseStorePath(s); - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); - info->ultimate = useQueryPathInfo.getInt(5) == 1; + info->ultimate = useQueryPathInfo.getInt(5) == 1; s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); + if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = parseContentAddressOpt(s); - /* Get the references. */ + /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); - return info; + return info; } @@ -1041,9 +1041,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) auto path = path_; auto outputs = retrySQLite>>([&]() { auto state(_state.lock()); - std::map> outputs; + std::map> outputs; uint64_t drvId; - drvId = queryValidPathId(*state, path); + drvId = queryValidPathId(*state, path); auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); while (use.next()) outputs.insert_or_assign( From 82d5cf2a76ec009fd94a925c22a5e099a0b7321b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 11:45:41 +0100 Subject: [PATCH 30/87] Fix macOS build --- src/libstore/build/local-derivation-goal.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f273ebe8a..34f8ab5f1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -149,11 +149,15 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::killSandbox(bool getStats) { if (cgroup) { + #if __linux__ auto stats = destroyCgroup(*cgroup); if (getStats) { buildResult.cpuUser = stats.cpuUser; buildResult.cpuSystem = stats.cpuSystem; } + #else + abort(); + #endif } else if (buildUser) { From 9d17ce07e872e88057480744414e0d1ef4fd5fa8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 12:55:49 +0100 Subject: [PATCH 31/87] AutoUserLock: If sandboxing is disabled, use the build users group We have to use a gid that has write access to the Nix store. --- src/libstore/build/local-derivation-goal.cc | 60 ++++++++++----------- src/libstore/lock.cc | 22 ++++---- src/libstore/lock.hh | 2 +- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 34f8ab5f1..b7084384a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -177,9 +177,38 @@ void LocalDerivationGoal::tryLocalBuild() { return; } + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = derivationType.isSandboxed() && !noChroot; + } + + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir.get()) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + if (useBuildUsers()) { if (!buildUser) - buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1); + buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); if (!buildUser) { if (!actLock) @@ -433,35 +462,6 @@ void LocalDerivationGoal::startBuilder() additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType.isSandboxed() && !noChroot; - } - - auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 4fad3bfd2..3b93979a8 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -109,22 +109,18 @@ struct AutoUserLock : UserLock { AutoCloseFD fdUserLock; uid_t firstUid = 0; + gid_t firstGid = 0; uid_t nrIds = 1; uid_t getUID() override { assert(firstUid); return firstUid; } gid_t getUIDCount() override { return nrIds; } - gid_t getGID() override - { - // We use the same GID ranges as for the UIDs. - assert(firstUid); - return firstUid; - } + gid_t getGID() override { assert(firstGid); return firstGid; } std::vector getSupplementaryGIDs() override { return {}; } - static std::unique_ptr acquire(uid_t nrIds) + static std::unique_ptr acquire(uid_t nrIds, bool useChroot) { settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); @@ -154,6 +150,14 @@ struct AutoUserLock : UserLock auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); lock->firstUid = settings.startId + i * maxIdsPerBuild; + if (useChroot) + lock->firstGid = lock->firstUid; + else { + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + lock->firstGid = gr->gr_gid; + } lock->nrIds = nrIds; return lock; } @@ -163,10 +167,10 @@ struct AutoUserLock : UserLock } }; -std::unique_ptr acquireUserLock(uid_t nrIds) +std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot) { if (settings.autoAllocateUids) - return AutoUserLock::acquire(nrIds); + return AutoUserLock::acquire(nrIds, useChroot); else return SimpleUserLock::acquire(); } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index e7ceefab8..49ad86de7 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -31,7 +31,7 @@ struct UserLock /* Acquire a user lock for a UID range of size `nrIds`. Note that this may return nullptr if no user is available. */ -std::unique_ptr acquireUserLock(uid_t nrIds); +std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot); bool useBuildUsers(); From a4af966d5df96d8bb90cb7096985819e99c384c1 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 21 Nov 2022 14:32:38 +0100 Subject: [PATCH 32/87] add maintainers' handbook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write down the process we have been developing and following so far. Co-Authored-By: Théophane Hufschmitt Co-Authored-By: John Ericson --- maintainers/README.md | 79 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 maintainers/README.md diff --git a/maintainers/README.md b/maintainers/README.md new file mode 100644 index 000000000..cfb0e26a8 --- /dev/null +++ b/maintainers/README.md @@ -0,0 +1,79 @@ +# Nix maintainers team + +## Motivation + +The goal of the team to help other people to contribute to Nix. + +## Members + +- Eelco Dolstra (@edolstra) – Team lead +- Théophane Hufschmitt (@thufschmitt) +- Valentin Gagarin (@fricklerhandwerk) +- Thomas Bereknyei (@tomberek) +- Robert Hensing (@roberth) + +## Meeting protocol + +The team meets twice a week: + +- Discussion meeting: [Fridays 12:00-13:00 UTC](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) + + 1. Triage issues and pull requests from the _No Status_ column (30 min) + 2. Discuss issues and pull requests from the _To discuss_ column (30 min) + +- Work meeting: [Mondays 12:00-14:00 UTC](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) + + 1. Code review on pull requests from _In review_. + 2. Other chores and tasks. + +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). + +## Project board protocol + +The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. + +Issues on the board progress through the following states: + +- No Status + + Team members can add pull requests or issues to discuss or review together. + + During the discussion meeting, the team triages new items. + If there is disagreement on the general idea behind the issue or pull request, items are moved to _To discuss_, otherwise to _In review_. + +- To discuss + + Pull requests and issues that are important and controverisal enough to warrant a discussion by the whole team. + + This may be where the merit of the change itself or the implementation strategy is contested by a team member. + +- In review + + Pull requests in this column are reviewed together during work meetings. + This is both for spreading implementation knowledge and for establishing common values in code reviews. + + When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. + +- Assigned for merging + + One team member is assigned to each of these pull requests. + They will communicate with the authors, and make the final approval once all remaining issues are addressed. + + If more substantive issues arise, the assignee can move the pull request back to _To discuss_ to involve the team again. + +The process is illustrated in the following diagram: + +```mermaid +flowchart TD + discuss[To discuss] + + review[To review] + + New --> |Disagreement on idea| discuss + New & discuss --> |Consensus on idea| review + + review --> |Consensus on implementation| Assigned + + Assigned --> |Implementation issues arise| review + Assigned --> |Remaining issues fixed| Merged +``` From 44dc5c6c136b5440b2bdce7c537dc6a4f857c349 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 21 Nov 2022 15:13:19 +0100 Subject: [PATCH 33/87] reword --- maintainers/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/maintainers/README.md b/maintainers/README.md index cfb0e26a8..5a744fb91 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,7 +2,7 @@ ## Motivation -The goal of the team to help other people to contribute to Nix. +The goal of the team is to help other people to contribute to Nix. ## Members @@ -39,11 +39,11 @@ Issues on the board progress through the following states: Team members can add pull requests or issues to discuss or review together. During the discussion meeting, the team triages new items. - If there is disagreement on the general idea behind the issue or pull request, items are moved to _To discuss_, otherwise to _In review_. + If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. - To discuss - Pull requests and issues that are important and controverisal enough to warrant a discussion by the whole team. + Pull requests and issues that are important and controverisal are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. From c776dfbb35e961ac3f011ab8665dfc85ab067ef8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 18:46:55 +0100 Subject: [PATCH 34/87] Use hex for startId Co-authored-by: Linus Heckemann --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 88fe72202..653d108aa 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -288,7 +288,7 @@ public: Setting startId{this, #if __linux__ - 872415232, + 0x34000000, #else 56930, #endif From b37c2d84b67635fc928ed174166f04d6f4d30c6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Nov 2022 09:02:17 +0100 Subject: [PATCH 35/87] Always call setgroups() We shouldn't skip this if the supplementary group list is empty, because then the sandbox won't drop the supplementary groups of the parent (like "root"). --- src/libstore/build/local-derivation-goal.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7084384a..232440f74 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1988,9 +1988,8 @@ void LocalDerivationGoal::runChild() if (setUser && buildUser) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) + auto gids = buildUser->getSupplementaryGIDs(); + if (setgroups(gids.size(), gids.data()) == -1) throw SysError("cannot set supplementary groups of build user"); if (setgid(buildUser->getGID()) == -1 || From 3d23b9d0324ff415af9e5f35568aca98c04a90cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Nov 2022 09:03:30 +0100 Subject: [PATCH 36/87] SimpleUserLock::getSupplementaryGIDs(): Filter out main gid This avoids having the user's gid in the supplementary group list as well. --- src/libstore/lock.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 3b93979a8..7459d837d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -71,21 +71,22 @@ struct SimpleUserLock : UserLock user. This is usually either empty or contains a group such as "kvm". */ int ngroups = 32; // arbitrary initial guess - lock->supplementaryGIDs.resize(ngroups); + std::vector gids; + gids.resize(ngroups); int err = getgrouplist( pw->pw_name, pw->pw_gid, - lock->supplementaryGIDs.data(), + gids.data(), &ngroups); /* Our initial size of 32 wasn't sufficient, the correct size has been stored in ngroups, so we try again. */ if (err == -1) { - lock->supplementaryGIDs.resize(ngroups); + gids.resize(ngroups); err = getgrouplist( pw->pw_name, pw->pw_gid, - lock->supplementaryGIDs.data(), + gids.data(), &ngroups); } @@ -94,7 +95,9 @@ struct SimpleUserLock : UserLock throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); // Finally, trim back the GID list to its real size. - lock->supplementaryGIDs.resize(ngroups); + for (auto i = 0; i < ngroups; i++) + if (gids[i] != lock->gid) + lock->supplementaryGIDs.push_back(gids[i]); #endif return lock; From 989fc8a8b9cf98addbef85bf909be7b00b0462db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Nov 2022 15:24:50 +0100 Subject: [PATCH 37/87] Add release notes --- doc/manual/src/release-notes/rl-next.md | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2069e4578..47181fd39 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -10,3 +10,43 @@ This avoids a lot of spurious errors where some benign strings end-up having a context just because they are read from a store path ([#7260](https://github.com/NixOS/nix/pull/7260)). + +* Nix can now automatically pick UIDs for builds, removing the need to + create `nixbld*` user accounts. these UIDs are allocated starting at + 872415232 on Linux and 56930 on macOS. + + This is an experimental feature. To enable it, add the following to + `nix.conf`: + + ``` + extra-experimental-features = auto-allocate-uids + auto-allocate-uids = true + ``` + +* On Linux, Nix can now run builds in a user namespace where the build + runs as root (UID 0) and has 65,536 UIDs available. This is + primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. + + A build can enable this by requiring the `uid-range` system feature, + i.e. by setting the derivation attribute + + ``` + requiredSystemFeatures = [ "uid-range" ]; + ``` + + The `uid-range` system feature requires the `auto-allocate-uids` + setting to be enabled (see above). + +* On Linux, Nix has experimental support for running builds inside a + cgroup. It can be enabled by adding + + ``` + extra-experimental-features = cgroups + ``` + + to `nix.conf`. It is also automatically enabled for builds that + require the `uid-range` system feature. + +* `nix build --json` now prints some statistics about top-level + derivations, such as CPU statistics when cgroups are enabled. From 2aa3f2e81020c1c780be6329e1133068779c8f08 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Nov 2022 17:07:59 +0100 Subject: [PATCH 38/87] Include UID in hex --- doc/manual/src/release-notes/rl-next.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 47181fd39..4c91002fb 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -12,8 +12,8 @@ ([#7260](https://github.com/NixOS/nix/pull/7260)). * Nix can now automatically pick UIDs for builds, removing the need to - create `nixbld*` user accounts. these UIDs are allocated starting at - 872415232 on Linux and 56930 on macOS. + create `nixbld*` user accounts. These UIDs are allocated starting at + 872415232 (0x34000000) on Linux and 56930 on macOS. This is an experimental feature. To enable it, add the following to `nix.conf`: From 341a807444a23cb12ed1cf3ad80b0c99f7dc5873 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Nov 2022 11:33:59 +0100 Subject: [PATCH 39/87] the point is setting a default reviewer, not notifications --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bba083834..d58577551 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,7 @@ # Merge permissions are required for maintaining an entry in this file. # For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ -# Notified if nothing else matches +# Default reviewers if nothing else matches * @edolstra @thufschmitt # This file From d6318e1638ed17fef701ada19ec7fde4b54d3bcf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Nov 2022 13:47:08 +0100 Subject: [PATCH 40/87] refactor rendering documentation of builtins as in [1], make the document structure visible, like in a template [1]: 4655563470b59e0ef50a33af003058c2b54db778 --- doc/manual/generate-builtins.nix | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 6c8b88da2..115bb3f94 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,16 +1,20 @@ -with builtins; -with import ./utils.nix; +builtinsDump: +let + showBuiltin = name: + let + inherit (builtinsDump.${name}) doc args; + in + '' +
+ ${name} ${listArgs args} +
+
-builtins: + ${doc} + +
+ ''; + listArgs = args: builtins.concatStringsSep " " (map (s: "${s}") args); +in +with builtins; concatStringsSep "\n" (map showBuiltin (attrNames builtinsDump)) -concatStrings (map - (name: - let builtin = builtins.${name}; in - "
${name} " - + concatStringsSep " " (map (s: "${s}") builtin.args) - + "
" - + "
\n\n" - + builtin.doc - + "\n\n
" - ) - (attrNames builtins)) From 0b4c4d74344613a15791e61179a441d80803d67a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Nov 2022 22:30:56 +0100 Subject: [PATCH 41/87] Don't use GC_STRNDUP It calls strlen() on the input (rather than simply copying at most `size` bytes), which can fail if the input is not zero-terminated and is inefficient in any case. Fixes #7347. --- src/libexpr/eval.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 563f24e48..6ba44cc1d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -69,15 +69,11 @@ static char * dupString(const char * s) // empty string. static const char * makeImmutableStringWithLen(const char * s, size_t size) { - char * t; if (size == 0) return ""; -#if HAVE_BOEHMGC - t = GC_STRNDUP(s, size); -#else - t = strndup(s, size); -#endif - if (!t) throw std::bad_alloc(); + auto t = allocString(size + 1); + memcpy(t, s, size); + t[size] = 0; return t; } From c4ce89f772d85e2a39727927903ca105a8c6a168 Mon Sep 17 00:00:00 2001 From: Liu Xiaoyi Date: Sat, 26 Nov 2022 22:01:51 +0800 Subject: [PATCH 42/87] Clarify uninstallation steps on Linux Co-authored-by: Valentin Gagarin --- .../src/installation/installing-binary.md | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 8cdd64df6..a9378681d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -88,25 +88,28 @@ extension. The installer will also create `/etc/profile.d/nix.sh`. ### Linux -To begin, if you are on Linux with systemd, remove the Nix daemon service: +If you are on Linux with systemd: -```console -sudo systemctl stop nix-daemon.service -sudo systemctl disable nix-daemon.socket nix-daemon.service -sudo systemctl daemon-reload -``` +1. Remove the Nix daemon service: -Then you can remove systemd service files: + ```console + sudo systemctl stop nix-daemon.service + sudo systemctl disable nix-daemon.socket nix-daemon.service + sudo systemctl daemon-reload + ``` -```console -sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket -``` +1. Remove systemd service files: -Also, the installer script uses systemd-tmpfiles (if presents) to create the socket directory. You may also want to remove the configuration for that: + ```console + sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket + ``` -```console -sudo rm /etc/tmpfiles.d/nix-daemon.conf -``` +1. The installer script uses systemd-tmpfiles to create the socket directory. + You may also want to remove the configuration for that: + + ```console + sudo rm /etc/tmpfiles.d/nix-daemon.conf + ``` Remove files created by Nix: From 5b798f6caeab64854394e1250e30aab91241cf26 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Nov 2022 12:57:18 +0100 Subject: [PATCH 43/87] Fix random client failures during GC server shutdown We need to close the GC server socket before shutting down the active GC client connections, otherwise a client may (re)connect and get ECONNRESET. But also handle ECONNRESET for resilience. Fixes random failures like GC socket disconnected connecting to '/tmp/nix-shell.y07M0H/nix-test/default/var/nix/gc-socket/socket' sending GC root '/tmp/nix-shell.y07M0H/nix-test/default/store/kb5yzija0f1x5xkqkgclrdzldxj6nnc6-non-blocking' reading GC root from client: error: unexpected EOF reading a line 1 store paths deleted, 0.00 MiB freed error: reading from file: Connection reset by peer in gc-non-blocking.sh. --- src/libstore/gc.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 9ef8972f3..5d91829f1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -147,7 +147,7 @@ void LocalStore::addTempRoot(const StorePath & path) } catch (SysError & e) { /* The garbage collector may have exited, so we need to restart. */ - if (e.errNo == EPIPE) { + if (e.errNo == EPIPE || e.errNo == ECONNRESET) { debug("GC socket disconnected"); state->fdRootsSocket.close(); goto restart; @@ -506,6 +506,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Finally cleanup([&]() { debug("GC roots server shutting down"); + fdServer.close(); while (true) { auto item = remove_begin(*connections.lock()); if (!item) break; From f1b5c6876bc570ff9ac79410d8e47aadcb9aed52 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Nov 2022 16:38:34 +0100 Subject: [PATCH 44/87] Add tests for auto-uid-allocation, uid-range and cgroups --- flake.nix | 6 ++++ tests/containers.nix | 68 ++++++++++++++++++++++++++++++++++++ tests/id-test.nix | 8 +++++ tests/systemd-nspawn.nix | 75 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+) create mode 100644 tests/containers.nix create mode 100644 tests/id-test.nix create mode 100644 tests/systemd-nspawn.nix diff --git a/flake.nix b/flake.nix index cc2a48d9c..d9d01da10 100644 --- a/flake.nix +++ b/flake.nix @@ -506,6 +506,12 @@ overlay = self.overlays.default; }); + tests.containers = (import ./tests/containers.nix rec { + system = "x86_64-linux"; + inherit nixpkgs; + overlay = self.overlays.default; + }); + tests.setuid = nixpkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] (system: diff --git a/tests/containers.nix b/tests/containers.nix new file mode 100644 index 000000000..d1e791b8c --- /dev/null +++ b/tests/containers.nix @@ -0,0 +1,68 @@ +# Test whether we can run a NixOS container inside a Nix build using systemd-nspawn. +{ nixpkgs, system, overlay }: + +with import (nixpkgs + "/nixos/lib/testing-python.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; + +makeTest ({ + name = "containers"; + + nodes = + { + host = + { config, lib, pkgs, nodes, ... }: + { virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = + [ pkgs.stdenv + (import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel + ]; + virtualisation.memorySize = 4096; + nix.binaryCaches = lib.mkForce [ ]; + nix.extraOptions = + '' + extra-experimental-features = nix-command auto-allocate-uids + extra-system-features = uid-range + ''; + nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; + }; + }; + + testScript = { nodes }: '' + start_all() + + host.succeed("nix --version >&2") + + # Test that 'id' gives the expected result in various configurations. + + # Existing UIDs, sandbox. + host.succeed("nix build --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1") + host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") + + # Existing UIDs, no sandbox. + host.succeed("nix build --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2") + host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]") + + # Auto-allocated UIDs, sandbox. + host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3") + host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") + + # Auto-allocated UIDs, no sandbox. + host.succeed("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4") + host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]") + + # Auto-allocated UIDs, UID range, sandbox. + host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true") + host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]") + + # Auto-allocated UIDs, UID range, no sandbox. + host.fail("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") + + # Run systemd-nspawn in a Nix build. + host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + ''; + +}) diff --git a/tests/id-test.nix b/tests/id-test.nix new file mode 100644 index 000000000..8eb9d38f9 --- /dev/null +++ b/tests/id-test.nix @@ -0,0 +1,8 @@ +{ name, uidRange ? false }: + +with import {}; + +runCommand name + { requiredSystemFeatures = if uidRange then ["uid-range"] else []; + } + "id; id > $out" diff --git a/tests/systemd-nspawn.nix b/tests/systemd-nspawn.nix new file mode 100644 index 000000000..49944eba3 --- /dev/null +++ b/tests/systemd-nspawn.nix @@ -0,0 +1,75 @@ +{ nixpkgs }: + +let + + machine = { config, pkgs, ... }: + { + system.stateVersion = "22.05"; + boot.isContainer = true; + systemd.services.console-getty.enable = false; + networking.dhcpcd.enable = false; + + services.httpd = { + enable = true; + adminAddr = "nixos@example.org"; + }; + + systemd.services.test = { + wantedBy = [ "multi-user.target" ]; + after = [ "httpd.service" ]; + script = '' + source /.env + echo "Hello World" > $out/msg + ls -lR /dev > $out/dev + ${pkgs.curl}/bin/curl -sS --fail http://localhost/ > $out/page.html + ''; + unitConfig = { + FailureAction = "exit-force"; + FailureActionExitStatus = 42; + SuccessAction = "exit-force"; + }; + }; + }; + + config = (import (nixpkgs + "/nixos/lib/eval-config.nix") { + modules = [ machine ]; + }).config; + +in + +with import nixpkgs {}; + +runCommand "test" + { buildInputs = [ config.system.path ]; + requiredSystemFeatures = [ "uid-range" ]; + toplevel = config.system.build.toplevel; + } + '' + root=$(pwd)/root + mkdir -p $root $root/etc + + export > $root/.env + + # Make /run a tmpfs to shut up a systemd warning. + mkdir /run + mount -t tmpfs none /run + chmod 0700 /run + + mount -t cgroup2 none /sys/fs/cgroup + + mkdir -p $out + + touch /etc/os-release + echo a5ea3f98dedc0278b6f3cc8c37eeaeac > /etc/machine-id + + SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 \ + ${config.systemd.package}/bin/systemd-nspawn \ + --keep-unit \ + -M ${config.networking.hostName} -D "$root" \ + --register=no \ + --resolv-conf=off \ + --bind-ro=/nix/store \ + --bind=$out \ + --private-network \ + $toplevel/init + '' From fc1458561086a6cf2c1311294c9089785288aea3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Nov 2022 18:58:21 +0100 Subject: [PATCH 45/87] Fix evaluation --- tests/systemd-nspawn.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/systemd-nspawn.nix b/tests/systemd-nspawn.nix index 49944eba3..424436b3f 100644 --- a/tests/systemd-nspawn.nix +++ b/tests/systemd-nspawn.nix @@ -31,13 +31,16 @@ let }; }; - config = (import (nixpkgs + "/nixos/lib/eval-config.nix") { + cfg = (import (nixpkgs + "/nixos/lib/eval-config.nix") { modules = [ machine ]; - }).config; + system = "x86_64-linux"; + }); + + config = cfg.config; in -with import nixpkgs {}; +with cfg._module.args.pkgs; runCommand "test" { buildInputs = [ config.system.path ]; From 04ec1575177709dd3c1bd147cb8d0966237663d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 28 Nov 2022 10:38:23 +0100 Subject: [PATCH 46/87] repl: print a newline on ctrl-D --- src/libcmd/repl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index bb254ff8d..ddf0d9a0a 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -270,6 +270,7 @@ void NixRepl::mainLoop() // ctrl-D should exit the debugger. state->debugStop = false; state->debugQuit = true; + std::cout << std::endl; break; } try { From 9b35cc716b392dfb96a0e994349137f653b9c532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 28 Nov 2022 14:59:06 +0100 Subject: [PATCH 47/87] use logger->cout in order to avoid potential problems with the progress bar Co-authored-by: Eelco Dolstra --- src/libcmd/repl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index ddf0d9a0a..557952277 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -270,7 +270,7 @@ void NixRepl::mainLoop() // ctrl-D should exit the debugger. state->debugStop = false; state->debugQuit = true; - std::cout << std::endl; + logger->cout(""); break; } try { From ff12d1c1a1bb0dcea5a9ac6b8a5036d7e5dc11ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Nov 2022 20:49:17 +0100 Subject: [PATCH 48/87] Check that auto-allocated UIDs don't clash with existing accounts --- src/libstore/lock.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 7459d837d..2858137d6 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -127,13 +127,10 @@ struct AutoUserLock : UserLock { settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); - assert(settings.startId % maxIdsPerBuild == 0); assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); assert(nrIds <= maxIdsPerBuild); - // FIXME: check whether the id range overlaps any known users - createDirs(settings.nixStateDir + "/userpool2"); size_t nrSlots = settings.uidCount / maxIdsPerBuild; @@ -150,11 +147,18 @@ struct AutoUserLock : UserLock throw SysError("opening user lock '%s'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { + + auto firstUid = settings.startId + i * maxIdsPerBuild; + + auto pw = getpwuid(firstUid); + if (pw) + throw Error("auto-allocated UID %d clashes with existing user account '%s'", firstUid, pw->pw_name); + auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); - lock->firstUid = settings.startId + i * maxIdsPerBuild; + lock->firstUid = firstUid; if (useChroot) - lock->firstGid = lock->firstUid; + lock->firstGid = firstUid; else { struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) From 67bcb99700a0da1395fa063d7c6586740b304598 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Nov 2022 21:54:02 +0100 Subject: [PATCH 49/87] Add a setting for enabling cgroups --- doc/manual/src/release-notes/rl-next.md | 5 +++-- src/libstore/build/local-derivation-goal.cc | 7 ++++++- src/libstore/globals.hh | 23 +++++++++++++++------ tests/containers.nix | 2 +- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 4c91002fb..db2bd7419 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -43,10 +43,11 @@ ``` extra-experimental-features = cgroups + use-cgroups = true ``` - to `nix.conf`. It is also automatically enabled for builds that - require the `uid-range` system feature. + to `nix.conf`. Cgroups are required for derivations that require the + `uid-range` system feature. * `nix build --json` now prints some statistics about top-level derivations, such as CPU statistics when cgroups are enabled. diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d44694890..69a7df411 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -401,9 +401,14 @@ static void linkOrCopy(const Path & from, const Path & to) void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) - || settings.isExperimentalFeatureEnabled(Xp::Cgroups)) + #if __linux__ + || settings.useCgroups + #endif + ) { #if __linux__ + settings.requireExperimentalFeature(Xp::Cgroups); + auto ourCgroups = getCgroups("/proc/self/cgroup"); auto ourCgroup = ourCgroups[""]; if (ourCgroup == "") diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 653d108aa..b40dcfa77 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -304,6 +304,17 @@ public: "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; + #if __linux__ + Setting useCgroups{ + this, false, "use-cgroups", + R"( + Whether to execute builds inside cgroups. Cgroups are + enabled automatically for derivations that require the + `uid-range` system feature. + )" + }; + #endif + Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; @@ -592,10 +603,10 @@ public: cache) must have a signature by a trusted key. A trusted key is one listed in `trusted-public-keys`, or a public key counterpart to a private key stored in a file listed in `secret-key-files`. - + Set to `false` to disable signature checking and trust all non-content-addressed paths unconditionally. - + (Content-addressed paths are inherently trustworthy and thus unaffected by this configuration option.) )"}; @@ -681,7 +692,7 @@ public: is `root`. > **Warning** - > + > > Adding a user to `trusted-users` is essentially equivalent to > giving that user root access to the system. For example, the user > can set `sandbox-paths` and thereby obtain read access to @@ -771,13 +782,13 @@ public: The program executes with no arguments. The program's environment contains the following environment variables: - - `DRV_PATH` + - `DRV_PATH` The derivation for the built paths. Example: `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` - - `OUT_PATHS` + - `OUT_PATHS` Output paths of the built derivation, separated by a space character. @@ -815,7 +826,7 @@ public: documentation](https://ec.haxx.se/usingcurl-netrc.html). > **Note** - > + > > This must be an absolute path, and `~` is not resolved. For > example, `~/.netrc` won't resolve to your home directory's > `.netrc`. diff --git a/tests/containers.nix b/tests/containers.nix index d1e791b8c..59e953c3b 100644 --- a/tests/containers.nix +++ b/tests/containers.nix @@ -23,7 +23,7 @@ makeTest ({ nix.binaryCaches = lib.mkForce [ ]; nix.extraOptions = '' - extra-experimental-features = nix-command auto-allocate-uids + extra-experimental-features = nix-command auto-allocate-uids cgroups extra-system-features = uid-range ''; nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; From 7dd3e1fec47b9dd6aa6a0b9a58962078a8499453 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Nov 2022 22:04:51 +0100 Subject: [PATCH 50/87] Add example --- doc/manual/src/release-notes/rl-next.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index db2bd7419..8b314b5f6 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -26,7 +26,8 @@ * On Linux, Nix can now run builds in a user namespace where the build runs as root (UID 0) and has 65,536 UIDs available. This is primarily useful for running containers such as `systemd-nspawn` - inside a Nix build. + inside a Nix build. For an example, see + https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. A build can enable this by requiring the `uid-range` system feature, i.e. by setting the derivation attribute From dc61e1028d73db406d47c0fb979219f193b74777 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 22:00:44 +0000 Subject: [PATCH 51/87] Bump zeebe-io/backport-action from 0.0.8 to 0.0.9 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 0.0.8 to 0.0.9. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v0.0.8...v0.0.9) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 75be788ef..7568145b6 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.8 + uses: zeebe-io/backport-action@v0.0.9 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From 46a6be28bef45640de5344a09d56add7068a9aa4 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 29 Nov 2022 10:01:46 +0000 Subject: [PATCH 52/87] Add nobody user/group to Nix docker image --- docker.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker.nix b/docker.nix index bb2b4e7ff..203a06b53 100644 --- a/docker.nix +++ b/docker.nix @@ -36,6 +36,17 @@ let shell = "${pkgs.bashInteractive}/bin/bash"; home = "/root"; gid = 0; + groups = [ "root" ]; + description = "System administrator"; + }; + + nobody = { + uid = 65534; + shell = "${pkgs.shadow}/bin/nologin"; + home = "/var/empty"; + gid = 65534; + groups = [ "nobody" ]; + description = "Unprivileged account (don't use!)"; }; } // lib.listToAttrs ( @@ -57,6 +68,7 @@ let groups = { root.gid = 0; nixbld.gid = 30000; + nobody.gid = 65534; }; userToPasswd = ( From 4f762e2b023fd451fdbab0de8d6394dd7201640d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Nov 2022 13:10:53 +0100 Subject: [PATCH 53/87] Restore ownership of / for non-uid-range builds --- src/libstore/build/local-derivation-goal.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 69a7df411..359966288 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -646,8 +646,7 @@ void LocalDerivationGoal::startBuilder() if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); - // FIXME: only make root writable for user namespace builds. - if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need From 0b092bd87f35e463fea66ddd40639e7b260680a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 30 Nov 2022 13:46:33 +0100 Subject: [PATCH 54/87] nix store make-content-addressed: Fix JSON construction Fixes error: [json.exception.type_error.301] cannot create object from initializer list in tests/fetchClosure.sh. --- src/nix/make-content-addressed.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index f2e4cefbe..f236bebd6 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -43,7 +43,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, assert(i != remappings.end()); jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second); } - std::cout << json::object({"rewrites", jsonRewrites}).dump(); + std::cout << nlohmann::json{"rewrites", jsonRewrites}.dump(); } else { for (auto & path : storePaths) { auto i = remappings.find(path); From 3f881e3378743039b32f3d3c650255b448732803 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 30 Nov 2022 23:40:53 +0100 Subject: [PATCH 55/87] add missing newline, for consistent formatting --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 057719e34..8c7c4d358 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -99,6 +99,7 @@ let in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); parsedToplevel = builtins.fromJSON toplevel; + manpages = processCommand { command = "nix"; details = parsedToplevel; From 0ea62670eda39ebeaff5335fd8225707ec27e7a3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 1 Dec 2022 04:40:02 +0100 Subject: [PATCH 56/87] move documentation on `auto-allocate-uids` to options docs this is where it belongs and can be found together with the other options. --- doc/manual/src/release-notes/rl-next.md | 42 +++++++++++-------------- src/libstore/globals.hh | 36 +++++++++++++++++---- 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8b314b5f6..bf51aa1f7 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -12,43 +12,37 @@ ([#7260](https://github.com/NixOS/nix/pull/7260)). * Nix can now automatically pick UIDs for builds, removing the need to - create `nixbld*` user accounts. These UIDs are allocated starting at - 872415232 (0x34000000) on Linux and 56930 on macOS. + create `nixbld*` user accounts. - This is an experimental feature. To enable it, add the following to - `nix.conf`: + See [`auto-allocate-uids`]. - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` + [`auto-allocate-uids`]: (../command-ref/conf-file.md#conf-auto-allocate-uids) * On Linux, Nix can now run builds in a user namespace where the build - runs as root (UID 0) and has 65,536 UIDs available. This is - primarily useful for running containers such as `systemd-nspawn` - inside a Nix build. For an example, see - https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + runs as root (UID 0) and has 65,536 UIDs available. - A build can enable this by requiring the `uid-range` system feature, - i.e. by setting the derivation attribute + + + This is primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + A build can enable this by by setting the derivation attribute: ``` requiredSystemFeatures = [ "uid-range" ]; ``` - The `uid-range` system feature requires the `auto-allocate-uids` - setting to be enabled (see above). + The `uid-range` [system feature] requires the [`auto-allocate-uids`] + setting to be enabled. + + [system feature]: (../command-ref/conf-file.md#conf-system-features), * On Linux, Nix has experimental support for running builds inside a - cgroup. It can be enabled by adding + cgroup. - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` - - to `nix.conf`. Cgroups are required for derivations that require the - `uid-range` system feature. + See [`use-cgroups`](../command-ref/conf-file.md#conf-use-cgroups). * `nix build --json` now prints some statistics about top-level derivations, such as CPU statistics when cgroups are enabled. diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b40dcfa77..b61a34461 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -284,7 +284,21 @@ public: )"}; Setting autoAllocateUids{this, false, "auto-allocate-uids", - "Whether to allocate UIDs for builders automatically."}; + R"( + Whether to allocate UIDs for builders automatically. + + These UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. + + > **Warning** + > This is an experimental feature. + + To enable it, add the following to [`nix.conf`](#): + + ``` + extra-experimental-features = auto-allocate-uids + auto-allocate-uids = true + ``` + )"}; Setting startId{this, #if __linux__ @@ -308,11 +322,21 @@ public: Setting useCgroups{ this, false, "use-cgroups", R"( - Whether to execute builds inside cgroups. Cgroups are - enabled automatically for derivations that require the - `uid-range` system feature. - )" - }; + Whether to execute builds inside cgroups. + Only on Linux with systemd. + + cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. + + > **Warning** + > This is an experimental feature. + + To enable it, add the following to [`nix.conf`](#): + + ``` + extra-experimental-features = cgroups + use-cgroups = true + ``` + )"}; #endif Setting impersonateLinux26{this, false, "impersonate-linux-26", From ad467265466dbccc816f550def9455ee720c5d3d Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 30 Nov 2022 22:53:41 -0600 Subject: [PATCH 57/87] doc: listToAttrs: document repeated keys --- src/libexpr/primops.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 05265411c..3572ca181 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2420,12 +2420,15 @@ static RegisterPrimOp primop_listToAttrs({ Construct a set from a list specifying the names and values of each attribute. Each element of the list should be a set consisting of a string-valued attribute `name` specifying the name of the attribute, - and an attribute `value` specifying its value. Example: + and an attribute `value` specifying its value. + In case of duplicate occurrences of the same name, the first + takes precedence. Example: ```nix builtins.listToAttrs [ { name = "foo"; value = 123; } { name = "bar"; value = 456; } + { name = "bar"; value = 420; } ] ``` From dfa27e6b2feb082b0a276338868b069458ec00db Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 1 Dec 2022 03:37:14 +0100 Subject: [PATCH 58/87] refactor rendering documentation of options this makes more obvious what the code produces, and the structure of the output easier to change --- doc/manual/generate-options.nix | 66 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 680b709c8..814144c20 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -1,29 +1,41 @@ -with builtins; -with import ./utils.nix; +let + inherit (builtins) attrNames concatStringsSep isAttrs isBool; + inherit (import ./utils.nix) concatStrings squash splitLines; +in -options: +optionsInfo: +let + showOption = name: + let + inherit (optionsInfo.${name}) description documentDefault defaultValue aliases; + result = squash '' + - [`${name}`]{#conf-${name}} -concatStrings (map - (name: - let option = options.${name}; in - " - [`${name}`](#conf-${name})" - + "

\n\n" - + concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n" - + (if option.documentDefault - then " **Default:** " + ( - if option.defaultValue == "" || option.defaultValue == [] - then "*empty*" - else if isBool option.defaultValue - then (if option.defaultValue then "`true`" else "`false`") - else - # n.b. a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is "null" in - # JSON, but that converts to "{ }" here. - (if isAttrs option.defaultValue then "`\"\"`" - else "`" + toString option.defaultValue + "`")) + "\n\n" - else " **Default:** *machine-specific*\n") - + (if option.aliases != [] - then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" - else "") - ) - (attrNames options)) + ${indent " " body} + ''; + # separate body to cleanly handle indentation + body = '' + ${description} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + showAliases = aliases: + if aliases == [] then "" else + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + in result; +in concatStrings (map showOption (attrNames optionsInfo)) From ec18b7d09b04f3bc4f4de81136fcb79a315bc829 Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 30 Nov 2022 23:21:09 -0600 Subject: [PATCH 59/87] doc: listToAttrs: fix line wrapping --- src/libexpr/primops.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3572ca181..3e7fdf9d6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2420,9 +2420,8 @@ static RegisterPrimOp primop_listToAttrs({ Construct a set from a list specifying the names and values of each attribute. Each element of the list should be a set consisting of a string-valued attribute `name` specifying the name of the attribute, - and an attribute `value` specifying its value. - In case of duplicate occurrences of the same name, the first - takes precedence. Example: + and an attribute `value` specifying its value. In case of duplicate + occurrences of the same name, the first takes precedence. Example: ```nix builtins.listToAttrs From f1e1ba9fe094a774f0fd05e537228e628d0bc8cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Dec 2022 16:29:09 +0100 Subject: [PATCH 60/87] Really fix 'nix store make-content-addressed --json' https://hydra.nixos.org/log/mcgypcf9vj4n8vdmw7lj3l05c899v73w-nix-2.12.0pre20221201_16b03f0-x86_64-unknown-linux-musl.drv --- src/nix/make-content-addressed.cc | 6 ++++-- tests/fetchClosure.sh | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index f236bebd6..d86b90fc7 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -37,13 +37,15 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, StorePathSet(storePaths.begin(), storePaths.end())); if (json) { - nlohmann::json jsonRewrites = json::object(); + auto jsonRewrites = json::object(); for (auto & path : storePaths) { auto i = remappings.find(path); assert(i != remappings.end()); jsonRewrites[srcStore->printStorePath(path)] = srcStore->printStorePath(i->second); } - std::cout << nlohmann::json{"rewrites", jsonRewrites}.dump(); + auto json = json::object(); + json["rewrites"] = jsonRewrites; + std::cout << json.dump(); } else { for (auto & path : storePaths) { auto i = remappings.find(path); diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 44050c878..d88c55c3c 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -1,7 +1,6 @@ source common.sh enableFeatures "fetch-closure" -needLocalStore "'--no-require-sigs' can’t be used with the daemon" clearStore clearCacheCache @@ -28,15 +27,19 @@ clearStore [ ! -e $nonCaPath ] [ -e $caPath ] -# In impure mode, we can use non-CA paths. -[[ $(nix eval --raw --no-require-sigs --impure --expr " - builtins.fetchClosure { - fromStore = \"file://$cacheDir\"; - fromPath = $nonCaPath; - } -") = $nonCaPath ]] +if [[ "$NIX_REMOTE" != "daemon" ]]; then -[ -e $nonCaPath ] + # In impure mode, we can use non-CA paths. + [[ $(nix eval --raw --no-require-sigs --impure --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + ") = $nonCaPath ]] + + [ -e $nonCaPath ] + +fi # 'toPath' set to empty string should fail but print the expected path. nix eval -v --json --expr " From ef524013aad45aec846d27a7309cbfb1afdf54c4 Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Thu, 1 Dec 2022 10:32:45 -0600 Subject: [PATCH 61/87] doc: listToAttrs: add extra whitespace --- src/libexpr/primops.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3e7fdf9d6..8a4c19f7c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2420,8 +2420,12 @@ static RegisterPrimOp primop_listToAttrs({ Construct a set from a list specifying the names and values of each attribute. Each element of the list should be a set consisting of a string-valued attribute `name` specifying the name of the attribute, - and an attribute `value` specifying its value. In case of duplicate - occurrences of the same name, the first takes precedence. Example: + and an attribute `value` specifying its value. + + In case of duplicate occurrences of the same name, the first + takes precedence. + + Example: ```nix builtins.listToAttrs From 5b954123282f7d795f4265b02c65b8bc79d28cf3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 1 Dec 2022 19:04:04 +0100 Subject: [PATCH 62/87] =?UTF-8?q?encourage=20adding=20=F0=9F=91=8D=20to=20?= =?UTF-8?q?express=20interest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit decided on the @NixOS/documentation-team, see NixOS/nix.dev#359 for more information --- .github/ISSUE_TEMPLATE/bug_report.md | 4 ++++ .github/ISSUE_TEMPLATE/feature_request.md | 4 ++++ .github/ISSUE_TEMPLATE/missing_documentation.md | 3 +++ .github/PULL_REQUEST_TEMPLATE/pull_request_template.md | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index e6d346bc1..984f9a9ea 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -30,3 +30,7 @@ A clear and concise description of what you expected to happen. **Additional context** Add any other context about the problem here. + +**Priorities** + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 4fe86d5ec..42c658b52 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -18,3 +18,7 @@ A clear and concise description of any alternative solutions or features you've **Additional context** Add any other context or screenshots about the feature request here. + +**Priorities** + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index fbabd868e..942d7a971 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -26,3 +26,6 @@ assignees: '' +## Priorities + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md index 537aa0909..5311be01f 100644 --- a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -5,3 +5,7 @@ Please include relevant [release notes](https://github.com/NixOS/nix/blob/master **Testing** If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master). + +**Priorities** + +Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). From ac06c8d2488aeffa79ebff832db0d267139b8b4c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 11:56:06 +0100 Subject: [PATCH 63/87] Revert "docs: drop shell prompt character for consistency" This reverts commit c068cce107b94f23c5288521770b2c807729a052 because it makes the docs *less* consistent. --- .../src/installation/installing-binary.md | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index eea11a428..31faeadc2 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -3,7 +3,7 @@ The easiest way to install Nix is to run the following command: ```console -sh <(curl -L https://nixos.org/nix/install) +$ sh <(curl -L https://nixos.org/nix/install) ``` This will run the installer interactively (causing it to explain what @@ -27,7 +27,7 @@ you can authenticate with `sudo`. To explicitly select a single-user installation on your system: ```console -sh <(curl -L https://nixos.org/nix/install) --no-daemon +$ sh <(curl -L https://nixos.org/nix/install) --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` @@ -37,8 +37,8 @@ if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: ```console -mkdir /nix -chown alice /nix +$ mkdir /nix +$ chown alice /nix ``` The install script will modify the first writable file from amongst @@ -50,7 +50,7 @@ the install script to disable this behaviour. You can uninstall Nix simply by running: ```console -rm -rf /nix +$ rm -rf /nix ``` # Multi User Installation @@ -66,7 +66,7 @@ You can instruct the installer to perform a multi-user installation on your system: ```console -sh <(curl -L https://nixos.org/nix/install) --daemon +$ sh <(curl -L https://nixos.org/nix/install) --daemon ``` The multi-user installation of Nix will create build users between the @@ -274,7 +274,7 @@ These install scripts can be used the same as the main NixOS.org installation script: ```console -sh <(curl -L https://nixos.org/nix/install) +$ sh <(curl -L https://nixos.org/nix/install) ``` In the same directory of the install script are sha256 sums, and gpg @@ -289,10 +289,10 @@ it somewhere (e.g. in `/tmp`), and then run the script named `install` inside the binary tarball: ```console -cd /tmp -tar xfj nix-1.8-x86_64-darwin.tar.bz2 -cd nix-1.8-x86_64-darwin -./install +$ cd /tmp +$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +$ cd nix-1.8-x86_64-darwin +$ ./install ``` If you need to edit the multi-user installation script to use different From 1211e59a038379026496bbee4b203bbd66833b01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 12:38:03 +0100 Subject: [PATCH 64/87] Move cgroup.{cc,hh} to libutil --- src/{libstore => libutil}/cgroup.cc | 0 src/{libstore => libutil}/cgroup.hh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/{libstore => libutil}/cgroup.cc (100%) rename src/{libstore => libutil}/cgroup.hh (100%) diff --git a/src/libstore/cgroup.cc b/src/libutil/cgroup.cc similarity index 100% rename from src/libstore/cgroup.cc rename to src/libutil/cgroup.cc diff --git a/src/libstore/cgroup.hh b/src/libutil/cgroup.hh similarity index 100% rename from src/libstore/cgroup.hh rename to src/libutil/cgroup.hh From 1e6a5d1ff6e8ef5bf340502f74c4d5039cedc67a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 12:57:41 +0100 Subject: [PATCH 65/87] Clean up cgroup handling in getMaxCPU() Also, don't assume in LocalDerivationGoal that cgroups are mounted on /sys/fs/cgroup. --- src/libstore/build/local-derivation-goal.cc | 6 ++- src/libutil/cgroup.cc | 17 +++++++ src/libutil/cgroup.hh | 2 + src/libutil/util.cc | 49 +++++++-------------- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c9b7b24f3..d2798888b 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -409,12 +409,16 @@ void LocalDerivationGoal::startBuilder() #if __linux__ settings.requireExperimentalFeature(Xp::Cgroups); + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + auto ourCgroups = getCgroups("/proc/self/cgroup"); auto ourCgroup = ourCgroups[""]; if (ourCgroup == "") throw Error("cannot determine cgroup name from /proc/self/cgroup"); - auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); + auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup); if (!pathExists(ourCgroupPath)) throw Error("expected cgroup directory '%s'", ourCgroupPath); diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index f693d77be..a008481ca 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "finally.hh" #include #include @@ -10,9 +11,25 @@ #include #include +#include namespace nix { +std::optional getCgroupFS() +{ + static auto res = [&]() -> std::optional { + auto fp = fopen("/proc/mounts", "r"); + if (!fp) return std::nullopt; + Finally delFP = [&]() { fclose(fp); }; + while (auto ent = getmntent(fp)) + if (std::string_view(ent->mnt_type) == "cgroup2") + return ent->mnt_dir; + + return std::nullopt; + }(); + return res; +} + // FIXME: obsolete, check for cgroup2 std::map getCgroups(const Path & cgroupFile) { diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh index 3ead4735f..d08c8ad29 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/cgroup.hh @@ -9,6 +9,8 @@ namespace nix { +std::optional getCgroupFS(); + std::map getCgroups(const Path & cgroupFile); struct CgroupStats diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 623b74bdd..2c2aae82e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -2,6 +2,7 @@ #include "sync.hh" #include "finally.hh" #include "serialise.hh" +#include "cgroup.hh" #include #include @@ -36,7 +37,6 @@ #include #include -#include #include #endif @@ -727,43 +727,24 @@ unsigned int getMaxCPU() { #if __linux__ try { - FILE *fp = fopen("/proc/mounts", "r"); - if (!fp) - return 0; + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; - Strings cgPathParts; + if (!pathExists("/proc/self/cgroup")) return 0; - struct mntent *ent; - while ((ent = getmntent(fp))) { - std::string mountType, mountPath; + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; - mountType = ent->mnt_type; - mountPath = ent->mnt_dir; + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - if (mountType == "cgroup2") { - cgPathParts.push_back(mountPath); - break; - } - } - - fclose(fp); - - if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { - std::string currentCgroup = readFile("/proc/self/cgroup"); - Strings cgValues = tokenizeString(currentCgroup, ":"); - cgPathParts.push_back(trim(cgValues.back(), "\n")); - cgPathParts.push_back("cpu.max"); - std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); - - if (pathExists(fullCgPath)) { - std::string cpuMax = readFile(fullCgPath); - std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); - std::string quota = cpuMaxParts[0]; - std::string period = trim(cpuMaxParts[1], "\n"); - - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } + if (pathExists(cpuFile)) { + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); } } catch (Error &) { ignoreException(); } #endif From fa99ef6a879e77024d60e73901a4773c6756c1bb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 15:03:40 +0100 Subject: [PATCH 66/87] getMaxCPU(): Lower verbosity level for ignored exceptions Fixes #7268. --- src/libutil/util.cc | 22 +++++++++------------- src/libutil/util.hh | 2 +- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2c2aae82e..a93ef1901 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -730,23 +730,19 @@ unsigned int getMaxCPU() auto cgroupFS = getCgroupFS(); if (!cgroupFS) return 0; - if (!pathExists("/proc/self/cgroup")) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroups = getCgroups("/proc/self/cgroupp"); auto cgroup = cgroups[""]; if (cgroup == "") return 0; auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - if (pathExists(cpuFile)) { - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") return std::ceil(std::stoi(quota) / std::stof(period)); - } - } catch (Error &) { ignoreException(); } + } catch (Error &) { ignoreException(lvlDebug); } #endif return 0; @@ -1408,7 +1404,7 @@ std::string shellEscape(const std::string_view s) } -void ignoreException() +void ignoreException(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -1416,7 +1412,7 @@ void ignoreException() try { throw; } catch (std::exception & e) { - printError("error (ignored): %1%", e.what()); + printMsg(lvl, "error (ignored): %1%", e.what()); } } catch (...) { } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e5c678682..94d8cc555 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -528,7 +528,7 @@ std::string shellEscape(const std::string_view s); /* Exception handling in destructors: print an error message, then ignore the exception. */ -void ignoreException(); +void ignoreException(Verbosity lvl = lvlError); From 19c53949714ffc6bf0e5d78aa6dd94cae2febc22 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 2 Dec 2022 09:00:03 -0500 Subject: [PATCH 67/87] Change "while evaluating " to "while *calling*" in trace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old way was not correct. Here is an example: ``` $ nix-instantiate --eval --expr 'let x = a: throw "asdf"; in x 1' --show-trace error: asdf … while evaluating 'x' at «string»:1:9: 1| let x = a: throw "asdf"; in x 1 | ^ … from call site at «string»:1:29: 1| let x = a: throw "asdf"; in x 1 | ^ ``` and yet also: ``` $ nix-instantiate --eval --expr 'let x = a: throw "asdf"; in x' --show-trace ``` Here is the thing: in both cases we are evaluating `x`! Nix is a higher-order languages, and functions are a sort of value. When we write `x = a: ...`, `a: ...` is the expression that `x` is being defined to be, and that is already a value. Therefore, we should *never* get an trace that says "while evaluating `x`", because evaluating `a: ...` is *trival* and nothing happens during it! What is actually happening here is we are applying `x` and evaluating its *body* with arguments substituted for parameters. I think the simplest way to say is just "while *calling* `x`", and so that is what I changed it to. --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c759acc10..76a10b9f8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1646,7 +1646,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto dts = debugRepl ? makeDebugTraceStacker( *this, *lambda.body, env2, positions[lambda.pos], - "while evaluating %s", + "while calling %s", lambda.name ? concatStrings("'", symbols[lambda.name], "'") : "anonymous lambda") @@ -1655,7 +1655,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & lambda.body->eval(*this, env2, vCur); } catch (Error & e) { if (loggerSettings.showTrace.get()) { - addErrorTrace(e, lambda.pos, "while evaluating %s", + addErrorTrace(e, lambda.pos, "while calling %s", (lambda.name ? concatStrings("'", symbols[lambda.name], "'") : "anonymous lambda")); From cccd57c022753c3ad727847c9c83c9d2c9c639e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 4 Dec 2022 18:22:11 +0100 Subject: [PATCH 68/87] getMaxCPU: fix cgroup path Given this typo I am not sure if it has been tested. --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a93ef1901..4f2caaa40 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -730,7 +730,7 @@ unsigned int getMaxCPU() auto cgroupFS = getCgroupFS(); if (!cgroupFS) return 0; - auto cgroups = getCgroups("/proc/self/cgroupp"); + auto cgroups = getCgroups("/proc/self/cgroup"); auto cgroup = cgroups[""]; if (cgroup == "") return 0; From 67d76cb8df0b484dca5ce907a182f578650e8356 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Dec 2022 16:34:39 +0100 Subject: [PATCH 69/87] Fix typo --- maintainers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 5a744fb91..b46d08fe3 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -43,7 +43,7 @@ Issues on the board progress through the following states: - To discuss - Pull requests and issues that are important and controverisal are discussed by the team during discussion meetings. + Pull requests and issues that are important and controversial are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. From 647e60b35674b6d44f5260fd27ea37705c298ecf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Dec 2022 16:35:25 +0100 Subject: [PATCH 70/87] maintainers/README.md: Use CET --- maintainers/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintainers/README.md b/maintainers/README.md index b46d08fe3..60768db0a 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -16,12 +16,12 @@ The goal of the team is to help other people to contribute to Nix. The team meets twice a week: -- Discussion meeting: [Fridays 12:00-13:00 UTC](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Triage issues and pull requests from the _No Status_ column (30 min) 2. Discuss issues and pull requests from the _To discuss_ column (30 min) -- Work meeting: [Mondays 12:00-14:00 UTC](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from _In review_. 2. Other chores and tasks. From e4f9f3bf246d66c57b07b45583469b98ba0db367 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 5 Dec 2022 11:27:47 -0500 Subject: [PATCH 71/87] check the store for input before failing (hopefully fix #6700) --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 67cfc38af..5aed51bcd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -501,6 +501,14 @@ void DerivationGoal::inputsRealised() now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal. */ std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); + if (!attempt) { + /* TODO (impure derivations-induced tech debt) (see below): + The above attempt should have found it, but because we manage + inputDrvOutputs statefully, sometimes it gets out of sync with + the real source of truth (store). So we query the store + directly if there's a problem. */ + attempt = fullDrv.tryResolve(worker.store); + } assert(attempt); Derivation drvResolved { *std::move(attempt) }; From 8c7661da0963a6a578605374cd9b309177563b3d Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 5 Dec 2022 23:21:58 -0500 Subject: [PATCH 72/87] check the store for input before failing (hopefully fix #6383) --- src/libstore/build/derivation-goal.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..7dd39051b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1016,11 +1016,20 @@ void DerivationGoal::resolvedFinished() throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", worker.store.printStorePath(drvPath), wantedOutput); - auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); - if (!realisation) + + const Realisation * realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + if (!realisation) { + /* The above `get` should work. But sateful tracking of + outputs in resolvedResult, this can get out of sync with the + store, which is our actual source of truth. For now we just + check the store directly if it fails. */ + realisation = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }).get(); + if (!realisation) { throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + } + } if (drv->type().isPure()) { auto newRealisation = *realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; From 484578d3f9b15c34c0e3a42cc0d1f87f1c15d7f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 10:30:36 +0100 Subject: [PATCH 73/87] Tweak option descriptions --- src/libstore/globals.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b61a34461..ca72ad31e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -285,9 +285,10 @@ public: Setting autoAllocateUids{this, false, "auto-allocate-uids", R"( - Whether to allocate UIDs for builders automatically. + Whether to select UIDs for builds automatically, instead of using the + users in `build-users-group`. - These UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. + UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. > **Warning** > This is an experimental feature. @@ -323,9 +324,10 @@ public: this, false, "use-cgroups", R"( Whether to execute builds inside cgroups. - Only on Linux with systemd. + This is only supported on Linux. - cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. + Cgroups are required and enabled automatically for derivations + that require the `uid-range` system feature. > **Warning** > This is an experimental feature. From 5b4b2eefa1e0f59f02844cacf06077cc21336e17 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 13:55:09 +0100 Subject: [PATCH 74/87] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.12.md | 43 +++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 46 ------------------------- 3 files changed, 44 insertions(+), 46 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.12.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 908e7e3d9..6a514fa2c 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -65,6 +65,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md new file mode 100644 index 000000000..82de22cb4 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -0,0 +1,43 @@ +# Release 2.12 (2022-12-06) + +* On Linux, Nix can now run builds in a user namespace where they run + as root (UID 0) and have 65,536 UIDs available. + + This is primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + A build can enable this by setting the derivation attribute: + + ``` + requiredSystemFeatures = [ "uid-range" ]; + ``` + + The `uid-range` [system feature] requires the [`auto-allocate-uids`] + setting to be enabled. + + [system feature]: (../command-ref/conf-file.md#conf-system-features) + +* Nix can now automatically pick UIDs for builds, removing the need to + create `nixbld*` user accounts. See [`auto-allocate-uids`]. + + [`auto-allocate-uids`]: (../command-ref/conf-file.md#conf-auto-allocate-uids) + +* On Linux, Nix has experimental support for running builds inside a + cgroup. See + [`use-cgroups`](../command-ref/conf-file.md#conf-use-cgroups). + +* `` now accepts an additional argument `impure` which + defaults to `false`. If it is set to `true`, the `hash` and `sha256` + arguments will be ignored and the resulting derivation will have + `__impure` set to `true`, making it an impure derivation. + +* If `builtins.readFile` is called on a file with context, then only + the parts of the context that appear in the content of the file are + retained. This avoids a lot of spurious errors where strings end up + having a context just because they are read from a store path + ([#7260](https://github.com/NixOS/nix/pull/7260)). + +* `nix build --json` now prints some statistics about top-level + derivations, such as CPU statistics when cgroups are enabled. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bf51aa1f7..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,48 +1,2 @@ # Release X.Y (202?-??-??) -* `` now accepts an additional argument `impure` which - defaults to `false`. If it is set to `true`, the `hash` and `sha256` - arguments will be ignored and the resulting derivation will have - `__impure` set to `true`, making it an impure derivation. - -* If `builtins.readFile` is called on a file with context, then only the parts - of that context that appear in the content of the file are retained. - This avoids a lot of spurious errors where some benign strings end-up having - a context just because they are read from a store path - ([#7260](https://github.com/NixOS/nix/pull/7260)). - -* Nix can now automatically pick UIDs for builds, removing the need to - create `nixbld*` user accounts. - - See [`auto-allocate-uids`]. - - [`auto-allocate-uids`]: (../command-ref/conf-file.md#conf-auto-allocate-uids) - -* On Linux, Nix can now run builds in a user namespace where the build - runs as root (UID 0) and has 65,536 UIDs available. - - - - This is primarily useful for running containers such as `systemd-nspawn` - inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - - [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - - A build can enable this by by setting the derivation attribute: - - ``` - requiredSystemFeatures = [ "uid-range" ]; - ``` - - The `uid-range` [system feature] requires the [`auto-allocate-uids`] - setting to be enabled. - - [system feature]: (../command-ref/conf-file.md#conf-system-features), - -* On Linux, Nix has experimental support for running builds inside a - cgroup. - - See [`use-cgroups`](../command-ref/conf-file.md#conf-use-cgroups). - -* `nix build --json` now prints some statistics about top-level - derivations, such as CPU statistics when cgroups are enabled. From 08dcd22582d65e73f29df79b3765e76cea8f3314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 6 Dec 2022 16:36:42 +0100 Subject: [PATCH 75/87] tests: don't refer to TMPDIR --- tests/check.nix | 2 +- tests/check.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/check.nix b/tests/check.nix index ed91ff845..ddab8eea9 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -44,7 +44,7 @@ with import ./config.nix; }; hashmismatch = import { - url = "file://" + builtins.getEnv "TMPDIR" + "/dummy"; + url = "file://" + builtins.getEnv "TEST_ROOT" + "/dummy"; sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; }; diff --git a/tests/check.sh b/tests/check.sh index 495202781..fbd784fc5 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -91,13 +91,13 @@ nix-build check.nix -A fetchurl --no-out-link --check nix-build check.nix -A fetchurl --no-out-link --repair [[ $(cat $path) != foo ]] -echo 'Hello World' > $TMPDIR/dummy +echo 'Hello World' > $TEST_ROOT/dummy nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] -echo -n > $TMPDIR/dummy +echo -n > $TEST_ROOT/dummy nix-build check.nix -A hashmismatch --no-out-link -echo 'Hello World' > $TMPDIR/dummy +echo 'Hello World' > $TEST_ROOT/dummy nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] From 1c8de7d3d03d7a6ba259387b0698874fa879428c Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Tue, 6 Dec 2022 11:25:38 -0500 Subject: [PATCH 76/87] improve style --- src/libstore/build/derivation-goal.cc | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7dd39051b..1f9bfaae0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1017,30 +1017,33 @@ void DerivationGoal::resolvedFinished() "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", worker.store.printStorePath(drvPath), wantedOutput); - const Realisation * realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); - if (!realisation) { + auto realisation = [&]{ + auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + if (take1) return *take1; + /* The above `get` should work. But sateful tracking of outputs in resolvedResult, this can get out of sync with the store, which is our actual source of truth. For now we just check the store directly if it fails. */ - realisation = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }).get(); - if (!realisation) { - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", - worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); - } - } + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }); + if (take2) return *take2; + + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + }(); + if (drv->type().isPure()) { - auto newRealisation = *realisation; + auto newRealisation = realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; newRealisation.signatures.clear(); if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } - outputPaths.insert(realisation->outPath); - builtOutputs.emplace(realisation->id, *realisation); + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(realisation.id, realisation); } runPostBuildHook( From 18431a453e5dbdb4e317ec683ab9b9e4a257358b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 17:26:49 +0100 Subject: [PATCH 77/87] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 3ca2c9b2c..a3ebb9f51 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.12.0 \ No newline at end of file +2.13.0 \ No newline at end of file From e5a2af2832d285f221ea021db3a55257db6e8dfe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 7 Dec 2022 10:00:27 +0100 Subject: [PATCH 78/87] add template for installer issues since the installer prompts users to file issues, labelling them automatically should reduce triaging effort significantly. --- .github/ISSUE_TEMPLATE/installer.md | 36 +++++++++++++++++++++++++++++ scripts/install-multi-user.sh | 3 ++- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/installer.md diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md new file mode 100644 index 000000000..3768a49c9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -0,0 +1,36 @@ +--- +name: Installer issue +about: Report problems with installation +title: '' +labels: installer +assignees: '' + +--- + +## Platform + + + +- [ ] Linux: +- [ ] macOS +- [ ] WSL + +## Additional information + + + +## Output + +
Output + +```log + + + +``` + +
+ +## Priorities + +Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 96c0f302b..194a263fb 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -97,7 +97,8 @@ is_os_darwin() { } contact_us() { - echo "You can open an issue at https://github.com/nixos/nix/issues" + echo "You can open an issue at" + echo "https://github.com/NixOS/nix/issues/new?labels=installer&template=installer.md" echo "" echo "Or feel free to contact the team:" echo " - Matrix: #nix:nixos.org" From 6833ded76441a43b94ed0cf1827b9714862fdedc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 7 Dec 2022 10:03:09 +0100 Subject: [PATCH 79/87] let installer blurb point to community page being too specific about it requires more maintenance (or otherwise produced more confusion and churn), since these points of contact change over time. --- scripts/install-multi-user.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 96c0f302b..af8a9b278 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -99,11 +99,7 @@ is_os_darwin() { contact_us() { echo "You can open an issue at https://github.com/nixos/nix/issues" echo "" - echo "Or feel free to contact the team:" - echo " - Matrix: #nix:nixos.org" - echo " - IRC: in #nixos on irc.libera.chat" - echo " - twitter: @nixos_org" - echo " - forum: https://discourse.nixos.org" + echo "Or get in touch with the community: https://nixos.org/community" } get_help() { echo "We'd love to help if you need it." From 8e0946e8df968391d1430af8377bdb51204e4666 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 26 Sep 2022 20:55:56 +0200 Subject: [PATCH 80/87] Remove repeat and enforce-determinism options These only functioned if a very narrow combination of conditions held: - The result path does not yet exist (--check did not result in repeated builds), AND - The result path is not available from any configured substituters, AND - No remote builders that can build the path are available. If any of these do not hold, a derivation would be built 0 or 1 times regardless of the repeat option. Thus, remove it to avoid confusion. --- doc/manual/redirects.js | 3 - doc/manual/src/advanced-topics/diff-hook.md | 34 ----------- doc/manual/src/command-ref/nix-store.md | 4 -- doc/manual/src/release-notes/rl-next.md | 3 + src/libmain/progress-bar.cc | 10 ++-- src/libstore/build/derivation-goal.cc | 17 +----- src/libstore/build/derivation-goal.hh | 5 -- src/libstore/build/local-derivation-goal.cc | 62 +-------------------- src/libstore/daemon.cc | 1 - src/libstore/globals.hh | 22 -------- src/libstore/legacy-ssh-store.cc | 4 +- src/nix-store/nix-store.cc | 9 ++- 12 files changed, 21 insertions(+), 153 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 2e77edd0f..69f75d3a0 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -35,7 +35,6 @@ const redirects = { "conf-build-max-jobs": "command-ref/conf-file.html#conf-build-max-jobs", "conf-build-max-log-size": "command-ref/conf-file.html#conf-build-max-log-size", "conf-build-max-silent-time": "command-ref/conf-file.html#conf-build-max-silent-time", - "conf-build-repeat": "command-ref/conf-file.html#conf-build-repeat", "conf-build-timeout": "command-ref/conf-file.html#conf-build-timeout", "conf-build-use-chroot": "command-ref/conf-file.html#conf-build-use-chroot", "conf-build-use-sandbox": "command-ref/conf-file.html#conf-build-use-sandbox", @@ -47,7 +46,6 @@ const redirects = { "conf-connect-timeout": "command-ref/conf-file.html#conf-connect-timeout", "conf-cores": "command-ref/conf-file.html#conf-cores", "conf-diff-hook": "command-ref/conf-file.html#conf-diff-hook", - "conf-enforce-determinism": "command-ref/conf-file.html#conf-enforce-determinism", "conf-env-keep-derivations": "command-ref/conf-file.html#conf-env-keep-derivations", "conf-extra-binary-caches": "command-ref/conf-file.html#conf-extra-binary-caches", "conf-extra-platforms": "command-ref/conf-file.html#conf-extra-platforms", @@ -74,7 +72,6 @@ const redirects = { "conf-plugin-files": "command-ref/conf-file.html#conf-plugin-files", "conf-post-build-hook": "command-ref/conf-file.html#conf-post-build-hook", "conf-pre-build-hook": "command-ref/conf-file.html#conf-pre-build-hook", - "conf-repeat": "command-ref/conf-file.html#conf-repeat", "conf-require-sigs": "command-ref/conf-file.html#conf-require-sigs", "conf-restrict-eval": "command-ref/conf-file.html#conf-restrict-eval", "conf-run-diff-hook": "command-ref/conf-file.html#conf-run-diff-hook", diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md index 161e64b2a..4a742c160 100644 --- a/doc/manual/src/advanced-topics/diff-hook.md +++ b/doc/manual/src/advanced-topics/diff-hook.md @@ -121,37 +121,3 @@ error: are not valid, so checking is not possible Run the build without `--check`, and then try with `--check` again. - -# Automatic and Optionally Enforced Determinism Verification - -Automatically verify every build at build time by executing the build -multiple times. - -Setting `repeat` and `enforce-determinism` in your `nix.conf` permits -the automated verification of every build Nix performs. - -The following configuration will run each build three times, and will -require the build to be deterministic: - - enforce-determinism = true - repeat = 2 - -Setting `enforce-determinism` to false as in the following -configuration will run the build multiple times, execute the build -hook, but will allow the build to succeed even if it does not build -reproducibly: - - enforce-determinism = false - repeat = 1 - -An example output of this configuration: - -```console -$ nix-build ./test.nix -A unstable -this derivation will be built: - /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv -building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... -building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... -output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round -/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable -``` diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 1251888e9..9102aff2d 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -104,10 +104,6 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` - See also the `build-repeat` configuration option, which repeats a - derivation a number of times and prevents its outputs from being - registered as “valid” in the Nix store unless they are identical. - Special exit codes: - `100`\ diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bf51aa1f7..faead26c2 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -46,3 +46,6 @@ * `nix build --json` now prints some statistics about top-level derivations, such as CPU statistics when cgroups are enabled. + +* The `repeat` and `enforce-determinism` options have been removed + since they had been broken under many circumstances for a long time. diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 961f4e18a..d160a83e9 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -180,10 +180,12 @@ public: auto machineName = getS(fields, 1); if (machineName != "") i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); - auto curRound = getI(fields, 2); - auto nrRounds = getI(fields, 3); - if (nrRounds != 1) - i->s += fmt(" (round %d/%d)", curRound, nrRounds); + + // Used to be curRound and nrRounds, but the + // implementation was broken for a long time. + if (getI(fields, 2) != 1 || getI(fields, 3) != 1) { + throw Error("log message indicated repeating builds, but this is not currently implemented"); + } i->name = DrvName(name).name; } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..98f9d681a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -571,10 +571,6 @@ void DerivationGoal::inputsRealised() /* What type of derivation are we building? */ derivationType = drv->type(); - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1; - /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ @@ -589,12 +585,11 @@ void DerivationGoal::started() auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + "building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); } @@ -948,14 +943,6 @@ void DerivationGoal::buildDone() cleanupPostOutputsRegisteredModeNonCheck(); - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 2d8bfd592..d33e04cbc 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -115,11 +115,6 @@ struct DerivationGoal : public Goal BuildMode buildMode; - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d2798888b..6fe3bc49c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2260,7 +2260,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -2688,10 +2687,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); } - if (curRound == nrRounds) { - localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() + worker.markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2720,61 +2717,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Apply output checks. */ checkOutputs(infos); - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - buildResult.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - printError(hint); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - renameFile(path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return {}; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 48dd5c247..12596ba49 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -238,7 +238,6 @@ struct ClientSettings } else if (trusted || name == settings.buildTimeout.name - || name == settings.buildRepeat.name || name == settings.maxSilentTime.name || name == settings.pollInterval.name || name == "connect-timeout" diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca72ad31e..54a5d0fc7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -373,11 +373,6 @@ public: )", {"build-max-log-size"}}; - /* When buildRepeat > 0 and verboseBuild == true, whether to print - repeated builds (i.e. builds other than the first one) to - stderr. Hack to prevent Hydra logs from being polluted. */ - bool printRepeatedBuilds = true; - Setting pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; @@ -501,19 +496,6 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting buildRepeat{ - this, 0, "repeat", - R"( - How many times to repeat builds to check whether they are - deterministic. The default value is 0. If the value is non-zero, - every build is repeated the specified number of times. If the - contents of any of the runs differs from the previous ones and - `enforce-determinism` is true, the build is rejected and the - resulting store paths are not registered as “valid” in Nix’s - database. - )", - {"build-repeat"}}; - #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", @@ -577,10 +559,6 @@ public: configuration file, and cannot be passed at the command line. )"}; - Setting enforceDeterminism{ - this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output. See `repeat`."}; - Setting trustedPublicKeys{ this, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dd34b19c6..4d398b21d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -255,8 +255,8 @@ private: << settings.maxLogSize; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) conn.to - << settings.buildRepeat - << settings.enforceDeterminism; + << 0 // buildRepeat hasn't worked for ages anyway + << 0; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { conn.to << ((int) settings.keepFailed); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b59a6d026..b854ef1e7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -808,14 +808,17 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.maxLogSize = readNum(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.buildRepeat = readInt(in); - settings.enforceDeterminism = readInt(in); + if (readInt(in) != 0) { + throw Error("client requested repeating builds, but this is not currently implemented"); + } + if (readInt(in) != 0) { + throw Error("client requested enforcing determinism, but this is not currently implemented"); + } settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { settings.keepFailed = (bool) readInt(in); } - settings.printRepeatedBuilds = false; }; while (true) { From cc27bd06339283a8189de5c4b1a981bd9c85c211 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 6 Dec 2022 18:00:10 +0100 Subject: [PATCH 81/87] .version: Add newline --- .version | 2 +- flake.nix | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.version b/.version index a3ebb9f51..fb2c0766b 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.13.0 \ No newline at end of file +2.13.0 diff --git a/flake.nix b/flake.nix index d9d01da10..38b469fdb 100644 --- a/flake.nix +++ b/flake.nix @@ -9,14 +9,15 @@ let - version = builtins.readFile ./.version + versionSuffix; + officialRelease = false; + + version = strip (builtins.readFile ./.version) + versionSuffix; + strip = nixpkgs.lib.removeSuffix "\n"; versionSuffix = if officialRelease then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - officialRelease = false; - linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; From c710aa1abd51231ce45fc419de2e0e82c8957fbd Mon Sep 17 00:00:00 2001 From: endgame Date: Wed, 7 Dec 2022 22:55:02 +1000 Subject: [PATCH 82/87] Post build hook signing (#7408) * docs: Use secret-key-files when demonstrating post-build-hooks The docs used to recommend calling `nix store sign` in a post-build hook, but on more recent versions of nix, this results in unsigned store paths being copied into binary caches. See https://github.com/NixOS/nix/issues/6960 for details. Instead, use the `secret-key-files` config option, which signs all locally-built derivations with the private key. Co-authored-by: Valentin Gagarin --- doc/manual/src/advanced-topics/post-build-hook.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index fcb52d878..1479cc3a4 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -33,12 +33,17 @@ distribute the public key for verifying the authenticity of the paths. example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= ``` -Then, add the public key and the cache URL to your `nix.conf`'s -`trusted-public-keys` and `substituters` options: +Then update [`nix.conf`](../command-ref/conf-file.md) on any machine that will access the cache. +Add the cache URL to [`substituters`](../command-ref/conf-file.md#conf-substituters) and the public key to [`trusted-public-keys`](../command-ref/conf-file.md#conf-trusted-public-keys): substituters = https://cache.nixos.org/ s3://example-nix-cache trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +Machines that build for the cache must sign derivations using the private key. +On those machines, add the path to the key file to the [`secret-key-files`](../command-ref/conf-file.md#conf-secret-key-files) field in their [`nix.conf`](../command-ref/conf-file.md): + + secret-key-files = /etc/nix/key.private + We will restart the Nix daemon in a later step. # Implementing the build hook @@ -52,14 +57,12 @@ set -eu set -f # disable globbing export IFS=' ' -echo "Signing paths" $OUT_PATHS -nix store sign --key-file /etc/nix/key.private $OUT_PATHS echo "Uploading paths" $OUT_PATHS -exec nix copy --to 's3://example-nix-cache' $OUT_PATHS +exec nix copy --to "s3://example-nix-cache" $OUT_PATHS ``` > **Note** -> +> > The `$OUT_PATHS` variable is a space-separated list of Nix store > paths. In this case, we expect and want the shell to perform word > splitting to make each output path its own argument to `nix From af8136afd4440bbabb4ea214bfd3dc15ffebf469 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 7 Dec 2022 13:55:31 +0100 Subject: [PATCH 83/87] flake.nix: Use nixpkgs.lib.fileContents --- flake.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 38b469fdb..5127ee2a3 100644 --- a/flake.nix +++ b/flake.nix @@ -11,8 +11,7 @@ officialRelease = false; - version = strip (builtins.readFile ./.version) + versionSuffix; - strip = nixpkgs.lib.removeSuffix "\n"; + version = nixpkgs.lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease then "" From b8a1ff98c1019f36a6425f639bf4c694f42c7edf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 7 Dec 2022 16:12:26 +0100 Subject: [PATCH 84/87] use HTML anchors for config parameters this avoids incorrect rendering on the man pages, since `lowdown` neither parses the anchor syntax nor HTML. this should rather be fixed in lowdown, as adding more anchors would otherwise produce ever more noise and error-prone repetition. --- doc/manual/generate-options.nix | 2 +- doc/manual/src/command-ref/nix-build.md | 12 +++++++----- doc/manual/src/command-ref/nix-store.md | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 814144c20..a4ec36477 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -9,7 +9,7 @@ let let inherit (optionsInfo.${name}) description documentDefault defaultValue aliases; result = squash '' - - [`${name}`]{#conf-${name}} + - [`${name}`](#conf-${name}) ${indent " " body} ''; diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 49c6f3f55..3a47feaae 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -53,16 +53,18 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - [`--no-out-link`]{#opt-no-out-link}\ + - [`--no-out-link`](#opt-no-out-link) + Do not create a symlink to the output path. Note that as a result the output does not become a root of the garbage collector, and so - might be deleted by `nix-store - --gc`. + might be deleted by `nix-store --gc`. + + - [`--dry-run`](#opt-dry-run) - - [`--dry-run`]{#opt-dry-run}\ Show what store paths would be built or downloaded. - - [`--out-link`]{#opt-out-link} / `-o` *outlink*\ + - [`--out-link`](#opt-out-link) / `-o` *outlink* + Change the name of the symlink to the output path created from `result` to *outlink*. diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 1251888e9..ec1da72a4 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -22,7 +22,8 @@ This section lists the options that are common to all operations. These options are allowed for every subcommand, though they may not always have an effect. - - [`--add-root`]{#opt-add-root} *path*\ + - [`--add-root`](#opt-add-root) *path* + Causes the result of a realisation (`--realise` and `--force-realise`) to be registered as a root of the garbage collector. *path* will be created as a symlink to the resulting From ebeaf03558caa62cba6f0bfbd1170dbd5c5944b8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 7 Dec 2022 16:20:25 +0100 Subject: [PATCH 85/87] do not render links in man pages this is a follow-up on e7dcacb. most links are relative and this should not be too much of a detriment. --- doc/manual/local.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 486dbd7a2..c0f69e00f 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -29,19 +29,19 @@ nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command - $(d)/%.1: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man -M section=1 $^.tmp -o $@ + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ @rm $^.tmp $(d)/%.8: $(d)/src/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man -M section=8 $^.tmp -o $@ + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=8 $^.tmp -o $@ @rm $^.tmp $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp - $(trace-gen) lowdown -sT man -M section=5 $^.tmp -o $@ + $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli From 6f61f4667fd39d4fa2e02d18a9b818f39cbfd023 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Dec 2022 17:35:40 +0100 Subject: [PATCH 86/87] Remove tests for --repeat https://hydra.nixos.org/build/201125739 --- tests/check.sh | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index fbd784fc5..e77c0405d 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -40,14 +40,6 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log -nix build -f check.nix deterministic --rebuild --repeat 1 \ - --argstr checkBuildId $checkBuildId --keep-failed --no-link \ - 2> $TEST_ROOT/log -if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi -# Repeat is set to 1, ie. nix should build deterministic twice. -if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi -checkBuildTempDirRemoved $TEST_ROOT/log - nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log @@ -58,12 +50,6 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] checkBuildTempDirRemoved $TEST_ROOT/log -nix build -f check.nix nondeterministic --rebuild --repeat 1 \ - --argstr checkBuildId $checkBuildId --keep-failed --no-link \ - 2> $TEST_ROOT/log || status=$? -grep 'may not be deterministic' $TEST_ROOT/log -checkBuildTempDirRemoved $TEST_ROOT/log - nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log @@ -72,12 +58,6 @@ if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi clearStore -nix-build dependencies.nix --no-out-link --repeat 3 - -nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? -[ "$status" = "1" ] -grep 'differs from previous round' $TEST_ROOT/log - path=$(nix-build check.nix -A fetchurl --no-out-link) chmod +w $path From dbc854766498818917c47ebce302266e92b41433 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sat, 10 Dec 2022 11:26:59 +0100 Subject: [PATCH 87/87] Ignore the enforceDeterminism value We used to set enforceDeterminism to true in the settings (by default) and thus did send a non-zero value over the wire. The value should probably be ignored as it should only matter if nrRounds is non-zero as well. Having the old code here where the value is expected to be zero only works with the same version of Nix where we are sending zero. We should always test this against older Nix versions being client or server as otherwise upgrade in larger networks might be a pain. Fixes 8e0946e8df968391d1430af8377bdb51204e4666 --- src/nix-store/nix-store.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b854ef1e7..85f142c39 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -808,12 +808,15 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.maxLogSize = readNum(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - if (readInt(in) != 0) { + auto nrRepeats = readInt(in); + if (nrRepeats != 0) { throw Error("client requested repeating builds, but this is not currently implemented"); } - if (readInt(in) != 0) { - throw Error("client requested enforcing determinism, but this is not currently implemented"); - } + // Ignore. It used to be true by default, but also only never had any effect when `nrRepeats == 0`. + // We have already asserted that `nrRepeats` in fact is 0, so we can safely ignore this without + // doing something other than what the client asked for. + auto _enforceDeterminism = readInt(in); + settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) {