From fe76b9f4b461c6a25f49a80a2476b7d60537c455 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 16 Jun 2022 09:37:52 -0500 Subject: [PATCH 001/191] doc: fix some darwin uninstall gaps --- doc/manual/src/installation/installing-binary.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 9fb9c80c3..762402948 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -148,7 +148,8 @@ and `/etc/zshrc` which you may remove. This will remove all the build users that no longer serve a purpose. 4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like this, + volume on `/nix`, which looks like + `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic mounting of the Nix Store volume. @@ -175,6 +176,16 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. + If this command indicates that it couldn't remove the volume, your Nix Store + volume may not be mounted. Run the following to double-check: + + ```console + diskutil list + ``` + + If you see a Nix Store volume, re-run the diskutil deleteVolume command, but + replace `/nix` with the volume's `diskXsY` identifier. + > **Note** > > After you complete the steps here, you will still have an empty `/nix` From 649c9d9b4c0b2327b11c44cda7b5738efa245c08 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 16 Jun 2022 09:47:15 -0500 Subject: [PATCH 002/191] doc: acknowledge post-rsync reality Before #5150 the copy-to-store phase of the install was idempotent, but the recursive cp isn't. This is probably baiting a few people into trying corrective installs that will fail. --- doc/manual/src/installation/installing-binary.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 762402948..a2f284d5a 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -176,15 +176,17 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. - If this command indicates that it couldn't remove the volume, your Nix Store - volume may not be mounted. Run the following to double-check: + If this command indicates that it couldn't remove the volume, you should + make sure you don't have an _unmounted_ Nix Store volume. Look for a + "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you see a Nix Store volume, re-run the diskutil deleteVolume command, but - replace `/nix` with the volume's `diskXsY` identifier. + If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil + deleteVolume command, but replace `/nix` with the store volume's `diskXsY` + identifier. > **Note** > @@ -202,8 +204,7 @@ and `/etc/zshrc` which you may remove. We believe we have ironed out how to cleanly support the read-only root -on modern macOS. New installs will do this automatically, and you can -also re-run a new installer to convert your existing setup. +on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, but it now only outlines what the installer does. You don't need to know From 8ea3a911aa81d41efdff231f4b42b11d8861a000 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 14:39:22 -0700 Subject: [PATCH 003/191] local-derivation-goal.cc: improve error messages when sandboxing fails The failure modes for nix's sandboxing setup are pretty complicated. When nix is unable to set up the sandbox, let's provide more detail about what went wrong. Specifically: * Make sure the error message includes the word "sandbox" so the user knows that the failure was related to sandboxing. * If `--option sandbox-fallback false` was provided, and removing it would have allowed further attempts to make progress, let the user know. --- src/libstore/build/local-derivation-goal.cc | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d1ec91ed5..86a59e427 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -850,13 +850,23 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - + if (child == -1) + switch(errno) { + case EPERM: + case EINVAL: { + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); + /* Mention sandbox-fallback in the error message so the user + knows that having it disabled contributed to the + unrecoverability of this failure */ + throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + } + default: + throw SysError("creating sandboxed builder process using clone()"); + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); From 90830b1074cd09b58adde859fb1741a33390412f Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:28:13 -0700 Subject: [PATCH 004/191] local-derivation-goal.cc: warn if failing due to max_user_namespaces==0 This commit uses `warn()` to notify the user if sandbox setup fails with errno==EPERM and /proc/sys/user/max_user_namespaces is missing or zero, since that is at least part of the reason why sandbox setup failed. Note that `echo -n 0 > /proc/sys/user/max_user_namespaces` or equivalent at boot time has been the recommended mitigation for several Linux LPE vulnerabilities over the past few years. Many users have applied this mitigation and then forgotten that they have done so. --- src/libstore/build/local-derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 86a59e427..674b2eaa3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -859,6 +859,8 @@ void LocalDerivationGoal::startBuilder() to true (the default). */ if (settings.sandboxFallback) _exit(1); + if (!userNamespacesEnabled && errno==EPERM) + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 8d35f387dcfa61c59f898de88ef45f3f97817267 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:37:27 -0700 Subject: [PATCH 005/191] local-derivation-goal.cc: warn if failing and /proc/self/ns/user missing This commit causes nix to `warn()` if sandbox setup has failed and `/proc/self/ns/user` does not exist. This is usually a sign that the kernel was compiled without `CONFIG_USER_NS=y`, which is required for sandboxing. --- src/libstore/build/local-derivation-goal.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 674b2eaa3..3aa85e264 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,6 +861,9 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + Path procSelfNsUser = "/proc/self/ns/user"; + if (!pathExists(procSelfNsUser)) + warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 6fc56318bf32f715de8634c199c0fb812f813a8c Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:23:27 -0700 Subject: [PATCH 006/191] local-derivation-goal.cc: add comment re: CLONE_NEWUSER local-derivation-goal.cc contains a comment stating that "Some distros patch Linux to not allow unprivileged user namespaces." Let's give a pointer to a common version of this patch for those who want more details about this failure mode. --- src/libstore/build/local-derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3aa85e264..1c7618045 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -845,6 +845,7 @@ void LocalDerivationGoal::startBuilder() /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. + * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 */ usingUserNamespace = false; flags &= ~CLONE_NEWUSER; From c8c6203c2c6912a52c5f08881c453a04e7fc3f58 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:27:22 -0700 Subject: [PATCH 007/191] local-derivation-goal.cc: detect unprivileged_userns_clone failure mode The workaround for "Some distros patch Linux" mentioned in local-derivation-goal.cc will not help in the `--option sandbox-fallback false` case. To provide the user more helpful guidance on how to get the sandbox working, let's check to see if the `/proc` node created by the aforementioned patch is present and configured in a way that will cause us problems. If so, give the user a suggestion for how to troubleshoot the problem. --- src/libstore/build/local-derivation-goal.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 1c7618045..047c5c8ea 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -862,6 +862,13 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + if (userNamespacesEnabled) { + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + } + } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); From 5f51539f88227285866843f1383fd47d80fd5918 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:30:52 -0700 Subject: [PATCH 008/191] change warn() to notice() --- src/libstore/build/local-derivation-goal.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 047c5c8ea..595149f0a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,17 +861,17 @@ void LocalDerivationGoal::startBuilder() if (settings.sandboxFallback) _exit(1); if (!userNamespacesEnabled && errno==EPERM) - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); } } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) - warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 99fcc91f67ece5a9646065665395f496d6a0cb84 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:33:12 -0700 Subject: [PATCH 009/191] as requested by @thufschmitt https://github.com/NixOS/nix/pull/6814#discussion_r924275777 --- src/libstore/build/local-derivation-goal.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 595149f0a..43df41e34 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -855,11 +855,6 @@ void LocalDerivationGoal::startBuilder() switch(errno) { case EPERM: case EINVAL: { - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -872,6 +867,11 @@ void LocalDerivationGoal::startBuilder() Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From a9e75eca00f7efe361545c6e77ecb65244230dc3 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:33 -0700 Subject: [PATCH 010/191] error.hh: add additional constructor with explicit errno argument --- src/libutil/error.hh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a53e9802e..3d1479c54 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -204,13 +204,19 @@ public: int errNo; template - SysError(const Args & ... args) + SysError(int errNo_, const Args & ... args) : Error("") { - errNo = errno; + errNo = errNo_; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } + + template + SysError(const Args & ... args) + : SysError(errno, args ...) + { + } }; } From 36e1383b6b32abd414c8402dd66f7ef78f93f4e1 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:52 -0700 Subject: [PATCH 011/191] local-derivation-goal.cc: save global errno to the stack before performing tests which might clobber it --- src/libstore/build/local-derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 43df41e34..79a241ae0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -851,10 +851,11 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - if (child == -1) + if (child == -1) { switch(errno) { case EPERM: case EINVAL: { + int errno_ = errno; if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -875,11 +876,12 @@ void LocalDerivationGoal::startBuilder() /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ - throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); } default: throw SysError("creating sandboxed builder process using clone()"); } + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); From 1af5d798a4d10a07e995d420a759f2fe752b583a Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Fri, 15 Jul 2022 00:14:29 -0400 Subject: [PATCH 012/191] libstore/globals.cc: Automatically set cores based on cgroup CPU limit By default, Nix sets the "cores" setting to the number of CPUs which are physically present on the machine. If cgroups are used to limit the CPU and memory consumption of a large Nix build, the OOM killer may be invoked. For example, consider a GitLab CI pipeline which builds a large software package. The GitLab runner spawns a container whose CPU is limited to 4 cores and whose memory is limited to 16 GiB. If the underlying machine has 64 cores, Nix will invoke the build with -j64. In many cases, that level of parallelism will invoke the OOM killer and the build will completely fail. This change sets the default value of "cores" to be ceil(cpu_quota / cpu_period), with a fallback to std::thread::hardware_concurrency() if cgroups v2 is not detected. --- src/libstore/globals.cc | 50 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0f2ca4b15..48df839fa 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,6 +11,11 @@ #include #include +#if __linux__ +#include +#include +#endif + #include @@ -114,7 +119,50 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - return std::max(1U, std::thread::hardware_concurrency()); + unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + + #if __linux__ + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return concurrency; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + concurrency = std::ceil(std::stoi(quota) / std::stof(period)); + } + } + #endif + + return concurrency; } StringSet Settings::getDefaultSystemFeatures() From 722de8ddcc875c7e8e9a228f9d88454bae31fd40 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 19 Jul 2022 02:09:46 -0400 Subject: [PATCH 013/191] libstore/globals.cc: Move cgroup detection to libutil --- src/libstore/globals.cc | 54 +++++------------------------------------ src/libutil/util.cc | 51 ++++++++++++++++++++++++++++++++++++++ src/libutil/util.hh | 3 +++ 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 48df839fa..d724897bb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,11 +11,6 @@ #include #include -#if __linux__ -#include -#include -#endif - #include @@ -119,50 +114,13 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int maxCPU = getMaxCPU(); - #if __linux__ - FILE *fp = fopen("/proc/mounts", "r"); - if (!fp) - return concurrency; - - Strings cgPathParts; - - struct mntent *ent; - while ((ent = getmntent(fp))) { - std::string mountType, mountPath; - - mountType = ent->mnt_type; - mountPath = ent->mnt_dir; - - if (mountType == "cgroup2") { - cgPathParts.push_back(mountPath); - break; - } - } - - fclose(fp); - - if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { - std::string currentCgroup = readFile("/proc/self/cgroup"); - Strings cgValues = tokenizeString(currentCgroup, ":"); - cgPathParts.push_back(trim(cgValues.back(), "\n")); - cgPathParts.push_back("cpu.max"); - std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); - - if (pathExists(fullCgPath)) { - std::string cpuMax = readFile(fullCgPath); - std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); - std::string quota = cpuMaxParts[0]; - std::string period = trim(cpuMaxParts[1], "\n"); - - if (quota != "max") - concurrency = std::ceil(std::stoi(quota) / std::stof(period)); - } - } - #endif - - return concurrency; + if (maxCPU > 0) + return maxCPU; + else + return concurrency; } StringSet Settings::getDefaultSystemFeatures() diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 28df30fef..be6fe091f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -35,6 +35,9 @@ #ifdef __linux__ #include #include + +#include +#include #endif @@ -788,7 +791,55 @@ void drainFD(int fd, Sink & sink, bool block) } } +////////////////////////////////////////////////////////////////////// +unsigned int getMaxCPU() +{ + #if __linux__ + try { + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return 0; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } + } + } catch (Error &) { ignoreException(); } + #endif + + return 0; +} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d3ed15b0b..29227ecc6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -182,6 +182,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); +/* If cgroups are active, attempt to calculate the number of CPUs available. + If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +unsigned int getMaxCPU(); /* Automatic cleanup of resources. */ From 228028fc1aaad20a217387fbe9d4aa2d8698a048 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Thu, 28 Jul 2022 03:36:39 -0400 Subject: [PATCH 014/191] docker.nix: Allow Nix configuration to be customized --- docker.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index ddf6feff5..8e6aa227f 100644 --- a/docker.nix +++ b/docker.nix @@ -6,6 +6,7 @@ , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] , maxLayers ? 100 +, nixConf ? {} }: let defaultPkgs = with pkgs; [ @@ -123,12 +124,17 @@ let (lib.attrValues (lib.mapAttrs groupToGroup groups)) ); - nixConf = { + defaultNixConf = { sandbox = "false"; build-users-group = "nixbld"; - trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; }; - nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: + let + vStr = if builtins.isList v then lib.concatStringsSep " " v else v; + in + "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n"; baseSystem = let From be4654c344f0745d4c2eefb33a878bd1f23f6b40 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 15:55:41 +0200 Subject: [PATCH 015/191] manual: fix section title in table of contents --- doc/manual/src/SUMMARY.md.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9728728aa..5df4e2d75 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,7 +33,7 @@ - [Arguments and Variables](expressions/arguments-variables.md) - [Building and Testing](expressions/simple-building-testing.md) - [Generic Builder Syntax](expressions/generic-builder.md) - - [Writing Nix Expressions](expressions/expression-language.md) + - [Nix Expression Language](expressions/expression-language.md) - [Values](expressions/language-values.md) - [Language Constructs](expressions/language-constructs.md) - [Operators](expressions/language-operators.md) From 85cdaebcd69c4f6abd15b77888f0093be9c5ada4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 16:10:22 +0200 Subject: [PATCH 016/191] manual: set -> attribute set reword description to have shorter sentences. --- doc/manual/src/expressions/language-values.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 75ae9f2eb..abbe1fd35 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -172,25 +172,27 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. -## Sets +## Attribute Sets -Sets are really the core of the language, since ultimately the Nix -language is all about creating derivations, which are really just sets -of attributes to be passed to build scripts. +Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). -Sets are just a list of name/value pairs (called *attributes*) enclosed -in curly brackets, where each value is an arbitrary expression -terminated by a semicolon. For example: +Names and values are separated by an equal sign (`=`). +Each value is an arbitrary expression terminated by a semicolon (`;`). + +Attributes can appear in any order. +An attribute name may only occur once. + +Example: ```nix -{ x = 123; +{ + x = 123; text = "Hello"; y = f { bla = 456; }; } ``` -This defines a set with attributes named `x`, `text`, `y`. The order of -the attributes is irrelevant. An attribute name may only occur once. +This defines a set with attributes named `x`, `text`, `y`. Attributes can be selected from a set using the `.` operator. For instance, From 3063e5b94c1cc1f64cea6af792513c1a04e12155 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:09:53 +0200 Subject: [PATCH 017/191] manual: use subheadings for primitive types this gives us HTML anchors for each of them --- doc/manual/src/expressions/language-values.md | 288 +++++++++--------- 1 file changed, 149 insertions(+), 139 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index abbe1fd35..fa5743222 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -4,151 +4,161 @@ Nix has the following basic data types: - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the result of an expression into a string by - enclosing it in `${...}`, a feature known as *antiquotation*. The - enclosed expression must evaluate to something that can be coerced - into a string (meaning that it must be a string, a path, or a - derivation). For instance, rather than writing - - ```nix - "--with-freetype2-library=" + freetype + "/lib" - ``` - - (where `freetype` is a derivation), you can instead write the more - natural - - ```nix - "--with-freetype2-library=${freetype}/lib" - ``` - - The latter is automatically translated to the former. A more - complicated example (from the Nix expression for - [Qt](http://www.trolltech.com/products/qt)): - - ```nix - configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} - "; - ``` - - Note that Nix expressions and strings can be arbitrarily nested; in - this case the outer string contains various antiquotations that - themselves contain strings (e.g., `"-thread"`), some of which in - turn contain expressions (e.g., `${mesa}`). - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix +### Strings + +*Strings* can be written in three ways. + +The most common way is to enclose the string between double quotes, +e.g., `"foo bar"`. Strings can span multiple lines. The special +characters `"` and `\` and the character sequence `${` must be +escaped by prefixing them with a backslash (`\`). Newlines, carriage +returns and tabs can be written as `\n`, `\r` and `\t`, +respectively. + +You can include the result of an expression into a string by +enclosing it in `${...}`, a feature known as *antiquotation*. The +enclosed expression must evaluate to something that can be coerced +into a string (meaning that it must be a string, a path, or a +derivation). For instance, rather than writing + +```nix +"--with-freetype2-library=" + freetype + "/lib" +``` + +(where `freetype` is a derivation), you can instead write the more +natural + +```nix +"--with-freetype2-library=${freetype}/lib" +``` + +The latter is automatically translated to the former. A more +complicated example (from the Nix expression for +[Qt](http://www.trolltech.com/products/qt)): + +```nix +configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} +"; +``` + +Note that Nix expressions and strings can be arbitrarily nested; in +this case the outer string contains various antiquotations that +themselves contain strings (e.g., `"-thread"`), some of which in +turn contain expressions (e.g., `${mesa}`). + +The second way to write string literals is as an *indented string*, +which is enclosed between pairs of *double single-quotes*, like so: + +```nix +'' + This is the first line. + This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +Note that the whitespace and newline following the opening `''` is +ignored if there is no non-whitespace text on the initial line. + +Antiquotation (`${expr}`) is supported in indented strings. + +Since `${` and `''` have special meaning in indented strings, you +need a way to quote them. `$` can be escaped by prefixing it with +`''` (that is, two single quotes), i.e., `''$`. `''` can be escaped +by prefixing it with `'`, i.e., `'''`. `$` removes any special +meaning from the following `$`. Linefeed, carriage-return and tab +characters can be written as `''\n`, `''\r`, `''\t`, and `''\` +escapes any other character. + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { + ... + postInstall = '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Antiquotation (`${expr}`) is supported in indented strings. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... +} +``` - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - Numbers are type-compatible: pure integer operations will always - return integers, whereas any operation involving at least one - floating point number will have a floating point number as a result. +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. +### Numbers - Antiquotation is supported in any paths except those in angle brackets. - `./${foo}-${bar}.nix` is a more convenient way of writing - `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At - least one slash must appear *before* any antiquotations for this to be - recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division - operation. `./a.${foo}/b.${bar}` is a path. +Numbers, which can be *integers* (like `123`) or *floating point* +(like `123.43` or `.27e13`). - - *Booleans* with values `true` and `false`. +Numbers are type-compatible: pure integer operations will always +return integers, whereas any operation involving at least one +floating point number will have a floating point number as a result. - - The null value, denoted as `null`. +### Paths + +*Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at +least one slash to be recognised as such. For instance, `builder.sh` +is not a path: it's parsed as an expression that selects the +attribute `sh` from the variable `builder`. If the file name is +relative, i.e., if it does not begin with a slash, it is made +absolute at parse time relative to the directory of the Nix +expression that contained it. For instance, if a Nix expression in +`/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path +is `/foo/xyzzy/fnord.nix`. + +If the first component of a path is a `~`, it is interpreted as if +the rest of the path were relative to the user's home directory. +e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user +whose home directory is `/home/edolstra`. + +Paths can also be specified between angle brackets, e.g. +``. This means that the directories listed in the +environment variable `NIX_PATH` will be searched for the given file +or directory name. + +Antiquotation is supported in any paths except those in angle brackets. +`./${foo}-${bar}.nix` is a more convenient way of writing +`./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At +least one slash must appear *before* any antiquotations for this to be +recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division +operation. `./a.${foo}/b.${bar}` is a path. + +### Booleans + +*Booleans* with values `true` and `false`. + +### Null + +The null value, denoted as `null`. ## Lists From 4ff48854b85f5af6b7448957beae448939c56884 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:11:23 +0200 Subject: [PATCH 018/191] manual: simple values -> primitives "simple" is a loaded term --- doc/manual/src/expressions/language-values.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index fa5743222..19d2f7c0d 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -1,8 +1,6 @@ # Values -## Simple Values - -Nix has the following basic data types: +## Primitives ### Strings From 8f4fab8fab9c321aea62b2953cbe16083781f099 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:25:18 +0200 Subject: [PATCH 019/191] manual: use singular for headings --- doc/manual/src/expressions/language-values.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 19d2f7c0d..6b1047c71 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -2,7 +2,7 @@ ## Primitives -### Strings +### String *Strings* can be written in three ways. @@ -112,7 +112,7 @@ is*, without quotes. For instance, the string `"http://example.org/foo.tar.bz2"` can also be written as `http://example.org/foo.tar.bz2`. -### Numbers +### Number Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). @@ -121,7 +121,7 @@ Numbers are type-compatible: pure integer operations will always return integers, whereas any operation involving at least one floating point number will have a floating point number as a result. -### Paths +### Path *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at least one slash to be recognised as such. For instance, `builder.sh` @@ -150,7 +150,7 @@ least one slash must appear *before* any antiquotations for this to be recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. -### Booleans +### Boolean *Booleans* with values `true` and `false`. @@ -158,7 +158,7 @@ operation. `./a.${foo}/b.${bar}` is a path. The null value, denoted as `null`. -## Lists +## List Lists are formed by enclosing a whitespace-separated list of values between square brackets. For example, @@ -180,7 +180,7 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. -## Attribute Sets +## Attribute Set Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). From 41a3b315fd24b83d7e570e1dae8d384317fb89e6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:25:25 +0200 Subject: [PATCH 020/191] manual: values -> data types --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/expressions/language-values.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5df4e2d75..c8cb72fc0 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -34,7 +34,7 @@ - [Building and Testing](expressions/simple-building-testing.md) - [Generic Builder Syntax](expressions/generic-builder.md) - [Nix Expression Language](expressions/expression-language.md) - - [Values](expressions/language-values.md) + - [Data Types](expressions/language-values.md) - [Language Constructs](expressions/language-constructs.md) - [Operators](expressions/language-operators.md) - [Derivations](expressions/derivations.md) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 6b1047c71..e4e3bf181 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -1,4 +1,4 @@ -# Values +# Data Types ## Primitives From 27138f1ec69c7a6f09ab467de317909780e4ae7a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 23:30:07 +0200 Subject: [PATCH 021/191] manual: use singular in body, too Co-authored-by: Cole Helbling --- doc/manual/src/expressions/language-values.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index e4e3bf181..4aa354ba6 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -182,7 +182,7 @@ Note that lists are only lazy in values, and they are strict in length. ## Attribute Set -Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). From 297f6b5d56204c5ece34c88ba09c50d4e12ccf89 Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Thu, 14 Jul 2022 09:45:48 -0700 Subject: [PATCH 022/191] Fix link to hacking doc Right now, https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html redirects to https://hydra.nixos.org/build/183877779/download/1/manual/contributing/hacking.html, which gives me a "500 Internal Server Error". Not super useful =( Feel free to ignore if someone's working to fix the 500 I was running into. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80d6f128c..8a02c4c75 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to +See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources From 1467a98d4c2014e512825c87e9056de79408421a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Mon, 1 Aug 2022 11:37:21 +0200 Subject: [PATCH 023/191] derivation-goal.cc: remove bmCheck custom return branch on buildDone Once a derivation goal has been completed, we check whether or not this goal was meant to be repeated to check its output. An early return branch was preventing the worker to reach that repeat code branch, hence breaking the --check command (#2619). It seems like this early return branch is an artifact of a passed refactoring. As far as I can tell, buildDone's main branch also cleanup the tmp directory before returning. --- src/libstore/build/derivation-goal.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..3c0b14029 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -914,12 +914,6 @@ void DerivationGoal::buildDone() outputPaths ); - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built, std::move(builtOutputs)); - return; - } - cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ From f675ba53310f1bfbe6e9867184b3a5177532b370 Mon Sep 17 00:00:00 2001 From: K900 Date: Mon, 1 Aug 2022 13:50:35 +0300 Subject: [PATCH 024/191] doc/distributed-builds: don't use deprecated alias `nix ping-store` -> `nix store ping`. --- doc/manual/src/advanced-topics/distributed-builds.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index b0d7fbf1a..fefd10100 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix ping-store --store ssh://mac +$ nix store ping --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a From 8119390abcbb25e849acc50d0af0b37d85adfb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:08 +0100 Subject: [PATCH 025/191] Move some fs-related functions to their own file Unclutter `util.cc` a bit --- src/libutil/filesystem.cc | 44 +++++++++++++++++++++++++++++++++++++++ src/libutil/util.cc | 38 --------------------------------- 2 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 src/libutil/filesystem.cc diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc new file mode 100644 index 000000000..e67383e6f --- /dev/null +++ b/src/libutil/filesystem.cc @@ -0,0 +1,44 @@ +#include + +#include "util.hh" +#include "types.hh" + +namespace nix { + +void createSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } +} + +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp, mtime); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmp, link); + + break; + } +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..3f3695f0d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -681,44 +681,6 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - - -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); - - break; - } -} - - void readFull(int fd, char * buf, size_t count) { while (count) { From c2de0a232c1cfddb1f1385ffd23dd43a2099458e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:46 +0100 Subject: [PATCH 026/191] =?UTF-8?q?Create=20a=20wrapper=20around=20stdlib?= =?UTF-8?q?=E2=80=99s=20`rename`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Directly takes some c++ strings, and gently throws an exception on error (rather than having to inline this logic everywhere) --- src/libstore/build/derivation-goal.cc | 3 +-- src/libstore/build/local-derivation-goal.cc | 8 +++----- src/libstore/builtins/unpack-channel.cc | 3 +-- src/libstore/gc.cc | 4 +--- src/libstore/local-binary-cache-store.cc | 3 +-- src/libstore/local-store.cc | 6 ++---- src/libstore/optimise-store.cc | 6 ++++-- src/libutil/filesystem.cc | 9 +++++++-- src/libutil/util.hh | 2 ++ 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..93f923f18 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 79a241ae0..2435377e2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); + moveFile((chrootRootDir + p), p); } return diskFull; @@ -2625,8 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); + moveFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 426d58a53..a8417d7ff 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) - throw SysError("renaming channel directory"); + moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d58ed78b1..3682f81cc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError("cannot rename '%1%' to '%2%'", - tempLink , link); + moveFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ba4416f6d..ff7403f9d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,8 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - if (rename(tmp.c_str(), path2.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path2); + moveFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eba3b0fa5..7f708b243 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - if (rename(tmpFile.c_str(), logPath.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); + moveFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8af9b1dde..20b9c7611 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { + try { + moveFile(tempLink, path); + } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { @@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); + throw; } stats.filesLinked++; diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index e67383e6f..33a8d81a6 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -34,11 +34,16 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); + moveFile(tmp, link); break; } } +void moveFile(const Path & oldName, const Path & newName) +{ + if (::rename(oldName.c_str(), newName.c_str())) + throw SysError("renaming '%1%' to '%2%'", oldName, newName); +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29227ecc6..564d36e79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,6 +168,8 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ From 6f89fb60088c4bc1513a005f0350c2bc13068892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 16:13:29 +0100 Subject: [PATCH 027/191] rename: Fallback to a copy if the filesystems mismatch In `nix::rename`, if the call to `rename` fails with `EXDEV` (failure because the source and the destination are in a different filesystems) switch to copying and removing the source. To avoid having to re-implement the copy manually, I switched the function to use the c++17 `filesystem` library (which has a `copy` function that should do what we want). Fix #6262 --- src/libutil/filesystem.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 33a8d81a6..cf99b848a 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,8 +1,11 @@ #include +#include #include "util.hh" #include "types.hh" +namespace fs = std::filesystem; + namespace nix { void createSymlink(const Path & target, const Path & link, @@ -42,8 +45,16 @@ void replaceSymlink(const Path & target, const Path & link, void moveFile(const Path & oldName, const Path & newName) { - if (::rename(oldName.c_str(), newName.c_str())) - throw SysError("renaming '%1%' to '%2%'", oldName, newName); + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + try { + fs::rename(oldPath, newPath); + } catch (fs::filesystem_error & e) { + if (e.code().value() == EXDEV) { + fs::copy(oldName, newName, fs::copy_options::copy_symlinks); + fs::remove_all(oldName); + } + } } } From c5db1821a94406eaff86341220bc301ba1dac82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Mar 2022 14:25:56 +0100 Subject: [PATCH 028/191] Re-implement the recursive directory copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recursive copy from the stl doesn’t exactly do what we need because 1. It doesn’t delete things as we go 2. It doesn’t keep the mtime, which change the nars So re-implement it ourselves. A bit dull, but that way we have what we want --- src/libutil/filesystem.cc | 52 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index cf99b848a..198db2832 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -43,6 +43,53 @@ void replaceSymlink(const Path & target, const Path & link, } } +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + void moveFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); @@ -51,8 +98,9 @@ void moveFile(const Path & oldName, const Path & newName) fs::rename(oldPath, newPath); } catch (fs::filesystem_error & e) { if (e.code().value() == EXDEV) { - fs::copy(oldName, newName, fs::copy_options::copy_symlinks); - fs::remove_all(oldName); + fs::remove(newPath); + warn("Copying %s to %s", oldName, newName); + copy(fs::directory_entry(oldPath), newPath, true); } } } From a4f0fd633c9ec865d385b34bdc51923c204e7ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 11:52:39 +0200 Subject: [PATCH 029/191] Link against c++fs on darwin Required by the old clang version --- src/libutil/local.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..13e8d426a 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,3 +11,7 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif + +ifdef HOST_DARWIN + libutil_LDFLAGS += -lc++fs +endif From d71d9e9fbfc972514b0b940181ab7f9e766f41f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:10:36 +0200 Subject: [PATCH 030/191] moveFile -> renameFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `move` tends to have this `mv` connotation of “I will copy it for you if needs be” --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +++--- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- src/libutil/filesystem.cc | 4 ++-- src/libutil/util.hh | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 93f923f18..459fdae79 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,7 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2435377e2..6843173a7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,7 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -310,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - moveFile((chrootRootDir + p), p); + renameFile((chrootRootDir + p), p); } return diskFull; @@ -2624,7 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - moveFile(path, dst); + renameFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index a8417d7ff..ba04bb16c 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,7 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); + renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3682f81cc..4c1a82279 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,7 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - moveFile(tempLink, link); + renameFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ff7403f9d..f20b1fa02 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,7 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - moveFile(tmp, path2); + renameFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7f708b243..2d076f404 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - moveFile(tempPath, realPath); + renameFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1941,7 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - moveFile(tmpFile, logPath); + renameFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 20b9c7611..4d2781180 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -230,7 +230,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ try { - moveFile(tempLink, path); + renameFile(tempLink, path); } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 198db2832..fee40d09e 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -37,7 +37,7 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - moveFile(tmp, link); + renameFile(tmp, link); break; } @@ -90,7 +90,7 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) } } -void moveFile(const Path & oldName, const Path & newName) +void renameFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 564d36e79..186513cea 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,7 +168,7 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); -void moveFile(const Path & src, const Path & dst); +void renameFile(const Path & src, const Path & dst); /* Wrappers arount read()/write() that read/write exactly the From 90f968073338d6ba276994bc281fa5efa5306e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:19:42 +0200 Subject: [PATCH 031/191] Only use `renameFile` where needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In most places the fallback to copying isn’t needed and can actually be bad, so we’d rather not transparently fallback --- src/libstore/local-store.cc | 2 +- src/libutil/filesystem.cc | 14 +++++++++----- src/libutil/util.hh | 9 +++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d076f404..a272e4301 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - renameFile(tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index fee40d09e..9cc18507b 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -54,7 +54,6 @@ void setWriteTime(const fs::path & p, const struct stat & st) .tv_sec = st.st_mtime, .tv_usec = 0, }; - warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); if (lutimes(p.c_str(), times) != 0) throw SysError("changing modification time of '%s'", p); } @@ -92,14 +91,19 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) void renameFile(const Path & oldName, const Path & newName) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ try { - fs::rename(oldPath, newPath); + renameFile(oldName, newName); } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); if (e.code().value() == EXDEV) { fs::remove(newPath); - warn("Copying %s to %s", oldName, newName); + warn("Can’t rename %s as %s, copying instead", oldName, newName); copy(fs::directory_entry(oldPath), newPath, true); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 186513cea..cd83f250f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -170,6 +170,15 @@ void replaceSymlink(const Path & target, const Path & link, void renameFile(const Path & src, const Path & dst); +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ From 1ba5b3e001c3da3c2e2a4dc7d475da1b20f53638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:48:31 +0200 Subject: [PATCH 032/191] Make `moveFile` more atomic Rather than directly copying the source to its dest, copy it first to a temporary location, and eventually move that temporary. That way, the move is at least atomic from the point-of-view of the destination --- src/libutil/filesystem.cc | 62 ++++++++++++++++++++++++++++++++++++++- src/libutil/util.cc | 55 ---------------------------------- 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 9cc18507b..403389e60 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,6 +1,7 @@ #include #include +#include "finally.hh" #include "util.hh" #include "types.hh" @@ -8,6 +9,59 @@ namespace fs = std::filesystem; namespace nix { +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + void createSymlink(const Path & target, const Path & link, std::optional mtime) { @@ -101,10 +155,16 @@ void moveFile(const Path & oldName, const Path & newName) } catch (fs::filesystem_error & e) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), newPath, true); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); } } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3f3695f0d..2a1bb770a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -508,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed) } -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); From d1cda07ce47064bda2c609a0290c867295ddd0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 25 May 2022 11:37:01 +0200 Subject: [PATCH 033/191] Don't use -load_all on darwin That flag breaks `-lc++fs` (introducing a duplicate symbol for some reason). Besides, it was apparently needed for bzip2, but we're not using bzip2 anymore. --- configure.ac | 9 --------- 1 file changed, 9 deletions(-) diff --git a/configure.ac b/configure.ac index f0210ab78..64fa12fc7 100644 --- a/configure.ac +++ b/configure.ac @@ -296,15 +296,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) -# This is needed if bzip2 is a static library, and the Nix libraries -# are dynamic. -case "${host_os}" in - darwin*) - LDFLAGS="-all_load $LDFLAGS" - ;; -esac - - AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) From ceed4d41426f6e2dc74473d5c137a8f061c49061 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 3 Aug 2022 11:23:40 +0200 Subject: [PATCH 034/191] encode primitive as list with anchors to make it consistent with builtins and configuration options --- doc/manual/src/expressions/language-values.md | 248 +++++++++--------- 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 4aa354ba6..a53b2cb22 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -2,161 +2,161 @@ ## Primitives -### String +- String -*Strings* can be written in three ways. + *Strings* can be written in three ways. -The most common way is to enclose the string between double quotes, -e.g., `"foo bar"`. Strings can span multiple lines. The special -characters `"` and `\` and the character sequence `${` must be -escaped by prefixing them with a backslash (`\`). Newlines, carriage -returns and tabs can be written as `\n`, `\r` and `\t`, -respectively. + The most common way is to enclose the string between double quotes, + e.g., `"foo bar"`. Strings can span multiple lines. The special + characters `"` and `\` and the character sequence `${` must be + escaped by prefixing them with a backslash (`\`). Newlines, carriage + returns and tabs can be written as `\n`, `\r` and `\t`, + respectively. -You can include the result of an expression into a string by -enclosing it in `${...}`, a feature known as *antiquotation*. The -enclosed expression must evaluate to something that can be coerced -into a string (meaning that it must be a string, a path, or a -derivation). For instance, rather than writing + You can include the result of an expression into a string by + enclosing it in `${...}`, a feature known as *antiquotation*. The + enclosed expression must evaluate to something that can be coerced + into a string (meaning that it must be a string, a path, or a + derivation). For instance, rather than writing -```nix -"--with-freetype2-library=" + freetype + "/lib" -``` + ```nix + "--with-freetype2-library=" + freetype + "/lib" + ``` -(where `freetype` is a derivation), you can instead write the more -natural + (where `freetype` is a derivation), you can instead write the more + natural -```nix -"--with-freetype2-library=${freetype}/lib" -``` + ```nix + "--with-freetype2-library=${freetype}/lib" + ``` -The latter is automatically translated to the former. A more -complicated example (from the Nix expression for -[Qt](http://www.trolltech.com/products/qt)): + The latter is automatically translated to the former. A more + complicated example (from the Nix expression for + [Qt](http://www.trolltech.com/products/qt)): -```nix -configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} -"; -``` + ```nix + configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} + "; + ``` -Note that Nix expressions and strings can be arbitrarily nested; in -this case the outer string contains various antiquotations that -themselves contain strings (e.g., `"-thread"`), some of which in -turn contain expressions (e.g., `${mesa}`). + Note that Nix expressions and strings can be arbitrarily nested; in + this case the outer string contains various antiquotations that + themselves contain strings (e.g., `"-thread"`), some of which in + turn contain expressions (e.g., `${mesa}`). -The second way to write string literals is as an *indented string*, -which is enclosed between pairs of *double single-quotes*, like so: + The second way to write string literals is as an *indented string*, + which is enclosed between pairs of *double single-quotes*, like so: -```nix -'' - This is the first line. - This is the second line. - This is the third line. -'' -``` + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` -This kind of string literal intelligently strips indentation from -the start of each line. To be precise, it strips from each line a -number of spaces equal to the minimal indentation of the string as a -whole (disregarding the indentation of empty lines). For instance, -the first and second line are indented two spaces, while the third -line is indented four spaces. Thus, two spaces are stripped from -each line, so the resulting string is + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is -```nix -"This is the first line.\nThis is the second line.\n This is the third line.\n" -``` + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` -Note that the whitespace and newline following the opening `''` is -ignored if there is no non-whitespace text on the initial line. + Note that the whitespace and newline following the opening `''` is + ignored if there is no non-whitespace text on the initial line. -Antiquotation (`${expr}`) is supported in indented strings. + Antiquotation (`${expr}`) is supported in indented strings. -Since `${` and `''` have special meaning in indented strings, you -need a way to quote them. `$` can be escaped by prefixing it with -`''` (that is, two single quotes), i.e., `''$`. `''` can be escaped -by prefixing it with `'`, i.e., `'''`. `$` removes any special -meaning from the following `$`. Linefeed, carriage-return and tab -characters can be written as `''\n`, `''\r`, `''\t`, and `''\` -escapes any other character. + Since `${` and `''` have special meaning in indented strings, you + need a way to quote them. `$` can be escaped by prefixing it with + `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped + by prefixing it with `'`, i.e., `'''`. `$` removes any special + meaning from the following `$`. Linefeed, carriage-return and tab + characters can be written as `''\n`, `''\r`, `''\t`, and `''\` + escapes any other character. -Indented strings are primarily useful in that they allow multi-line -string literals to follow the indentation of the enclosing Nix -expression, and that less escaping is typically necessary for -strings representing languages such as shell scripts and -configuration files because `''` is much less common than `"`. -Example: + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: -```nix -stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... -} -``` + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` -Finally, as a convenience, *URIs* as defined in appendix B of -[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as -is*, without quotes. For instance, the string -`"http://example.org/foo.tar.bz2"` can also be written as -`http://example.org/foo.tar.bz2`. + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. -### Number +- Number -Numbers, which can be *integers* (like `123`) or *floating point* -(like `123.43` or `.27e13`). + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). -Numbers are type-compatible: pure integer operations will always -return integers, whereas any operation involving at least one -floating point number will have a floating point number as a result. + Numbers are type-compatible: pure integer operations will always + return integers, whereas any operation involving at least one + floating point number will have a floating point number as a result. -### Path +- Path -*Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at -least one slash to be recognised as such. For instance, `builder.sh` -is not a path: it's parsed as an expression that selects the -attribute `sh` from the variable `builder`. If the file name is -relative, i.e., if it does not begin with a slash, it is made -absolute at parse time relative to the directory of the Nix -expression that contained it. For instance, if a Nix expression in -`/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path -is `/foo/xyzzy/fnord.nix`. + *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at + least one slash to be recognised as such. For instance, `builder.sh` + is not a path: it's parsed as an expression that selects the + attribute `sh` from the variable `builder`. If the file name is + relative, i.e., if it does not begin with a slash, it is made + absolute at parse time relative to the directory of the Nix + expression that contained it. For instance, if a Nix expression in + `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path + is `/foo/xyzzy/fnord.nix`. -If the first component of a path is a `~`, it is interpreted as if -the rest of the path were relative to the user's home directory. -e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user -whose home directory is `/home/edolstra`. + If the first component of a path is a `~`, it is interpreted as if + the rest of the path were relative to the user's home directory. + e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user + whose home directory is `/home/edolstra`. -Paths can also be specified between angle brackets, e.g. -``. This means that the directories listed in the -environment variable `NIX_PATH` will be searched for the given file -or directory name. + Paths can also be specified between angle brackets, e.g. + ``. This means that the directories listed in the + environment variable `NIX_PATH` will be searched for the given file + or directory name. -Antiquotation is supported in any paths except those in angle brackets. -`./${foo}-${bar}.nix` is a more convenient way of writing -`./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At -least one slash must appear *before* any antiquotations for this to be -recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division -operation. `./a.${foo}/b.${bar}` is a path. + Antiquotation is supported in any paths except those in angle brackets. + `./${foo}-${bar}.nix` is a more convenient way of writing + `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At + least one slash must appear *before* any antiquotations for this to be + recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division + operation. `./a.${foo}/b.${bar}` is a path. -### Boolean +- Boolean -*Booleans* with values `true` and `false`. + *Booleans* with values `true` and `false`. -### Null +- Null -The null value, denoted as `null`. + The null value, denoted as `null`. ## List From b47b6a418d656ee5408b7773dd9b76770ca0d2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Wed, 3 Aug 2022 12:05:41 +0200 Subject: [PATCH 035/191] tests/check.sh: add nix3-build check test --- tests/check.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/check.sh b/tests/check.sh index ab48ff865..495202781 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -40,6 +40,14 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix deterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log +if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi +# Repeat is set to 1, ie. nix should build deterministic twice. +if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log @@ -50,6 +58,12 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix nondeterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log From c55bea420402fcf995688ef6a12cfa413bc5e35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 3 Aug 2022 14:16:00 +0200 Subject: [PATCH 036/191] Fix the html id of the list headers --- doc/manual/src/expressions/language-values.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index a53b2cb22..f09400d02 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -2,7 +2,7 @@ ## Primitives -- String +- String *Strings* can be written in three ways. @@ -112,7 +112,7 @@ `"http://example.org/foo.tar.bz2"` can also be written as `http://example.org/foo.tar.bz2`. -- Number +- Number Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). @@ -121,7 +121,7 @@ return integers, whereas any operation involving at least one floating point number will have a floating point number as a result. -- Path +- Path *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at least one slash to be recognised as such. For instance, `builder.sh` @@ -150,11 +150,11 @@ recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. -- Boolean +- Boolean *Booleans* with values `true` and `false`. -- Null +- Null The null value, denoted as `null`. From ccbd906c86cb10e5c7b49753e057fec4ab46233c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Aug 2022 17:45:11 +0200 Subject: [PATCH 037/191] Fix NIX_COUNT_CALLS=1 Also, make the JSON writer support std::string_view. Fixes #6857. --- src/libexpr/eval.cc | 12 ++++++------ src/libutil/json.cc | 21 +++++++++------------ src/libutil/json.hh | 11 +++++------ src/libutil/tests/json.cc | 4 ++-- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f485e2fed..e3716f217 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2501,18 +2501,18 @@ void EvalState::printStats() } { auto list = topObj.list("functions"); - for (auto & i : functionCalls) { + for (auto & [fun, count] : functionCalls) { auto obj = list.object(); - if (i.first->name) - obj.attr("name", (const std::string &) i.first->name); + if (fun->name) + obj.attr("name", (std::string_view) symbols[fun->name]); else obj.attr("name", nullptr); - if (auto pos = positions[i.first->pos]) { - obj.attr("file", (const std::string &) pos.file); + if (auto pos = positions[fun->pos]) { + obj.attr("file", (std::string_view) pos.file); obj.attr("line", pos.line); obj.attr("column", pos.column); } - obj.attr("count", i.second); + obj.attr("count", count); } } { diff --git a/src/libutil/json.cc b/src/libutil/json.cc index b0a5d7e75..abe0e6e74 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -6,7 +6,8 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end) +template<> +void toJSON(std::ostream & str, const std::string_view & s) { constexpr size_t BUF_SIZE = 4096; char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts @@ -21,7 +22,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) }; put('"'); - for (auto i = start; i != end; i++) { + for (auto i = s.begin(); i != s.end(); i++) { if (bufPos >= BUF_SIZE) flush(); if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } else if (*i == '\n') { put('\\'); put('n'); } @@ -44,7 +45,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) void toJSON(std::ostream & str, const char * s) { - if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); + if (!s) str << "null"; else toJSON(str, std::string_view(s)); } template<> void toJSON(std::ostream & str, const int & n) { str << n; } @@ -55,11 +56,7 @@ template<> void toJSON(std::ostream & str, const long long & n) { str template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } - -template<> void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} +template<> void toJSON(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); } template<> void toJSON(std::ostream & str, const bool & b) { @@ -154,7 +151,7 @@ JSONObject::~JSONObject() } } -void JSONObject::attr(const std::string & s) +void JSONObject::attr(std::string_view s) { comma(); toJSON(state->str, s); @@ -162,19 +159,19 @@ void JSONObject::attr(const std::string & s) if (state->indent) state->str << ' '; } -JSONList JSONObject::list(const std::string & name) +JSONList JSONObject::list(std::string_view name) { attr(name); return JSONList(state); } -JSONObject JSONObject::object(const std::string & name) +JSONObject JSONObject::object(std::string_view name) { attr(name); return JSONObject(state); } -JSONPlaceholder JSONObject::placeholder(const std::string & name) +JSONPlaceholder JSONObject::placeholder(std::string_view name) { attr(name); return JSONPlaceholder(state); diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 83213ca66..3790b1a2e 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -6,7 +6,6 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end); void toJSON(std::ostream & str, const char * s); template @@ -107,7 +106,7 @@ private: open(); } - void attr(const std::string & s); + void attr(std::string_view s); public: @@ -128,18 +127,18 @@ public: ~JSONObject(); template - JSONObject & attr(const std::string & name, const T & v) + JSONObject & attr(std::string_view name, const T & v) { attr(name); toJSON(state->str, v); return *this; } - JSONList list(const std::string & name); + JSONList list(std::string_view name); - JSONObject object(const std::string & name); + JSONObject object(std::string_view name); - JSONPlaceholder placeholder(const std::string & name); + JSONPlaceholder placeholder(std::string_view name); }; class JSONPlaceholder : JSONWriter diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc index dea73f53a..156286999 100644 --- a/src/libutil/tests/json.cc +++ b/src/libutil/tests/json.cc @@ -102,8 +102,8 @@ namespace nix { TEST(toJSON, substringEscape) { std::stringstream out; - const char *s = "foo\t"; - toJSON(out, s+3, s + strlen(s)); + std::string_view s = "foo\t"; + toJSON(out, s.substr(3)); ASSERT_EQ(out.str(), "\"\\t\""); } From c9f446ede1b09a9daff59e479601f49382071a22 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 4 Aug 2022 09:45:28 +0200 Subject: [PATCH 038/191] flakeref: fix comment --- src/libexpr/flake/flakeref.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a9182f4bf..fe4f67193 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -28,7 +28,7 @@ typedef std::string FlakeId; * object that fetcher generates (usually via * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). * - * The actual fetch not have been performed yet (i.e. a FlakeRef may + * The actual fetch may not have been performed yet (i.e. a FlakeRef may * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ From 53833dfb40ddfc978f47f5c180a73bc7e0400320 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 4 Aug 2022 09:45:29 +0200 Subject: [PATCH 039/191] libexpr/flake: remove `FIXME` Line 593 checks that all overrides (i.e. all elements of `lockFlags.inputOverrides`) are members of `overridesUsed`. --- src/libexpr/flake/flake.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index cc9be1336..105e76bc6 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -341,7 +341,6 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - // FIXME: check whether all overrides are used. std::map overrides; std::set overridesUsed, updatesUsed; From 4c8441be0a59cead901acdfc3285d835c400c615 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 4 Aug 2022 09:45:30 +0200 Subject: [PATCH 040/191] docs/flake-update: fix example --- src/nix/flake-update.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 03b50e38e..2ee8a707d 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -6,7 +6,7 @@ R""( lock file: ```console - # nix flake update + # nix flake update --commit-lock-file * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' … From 499ed265088948e823c83cc95d1097a6362d205b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 11:36:32 +0200 Subject: [PATCH 041/191] manual: remove "Writing Nix Expressions" chapter it is out of date, all over the place in level of detail, is really about `nixpkgs`, and in general instructions should not be part of a reference manual. also: - update redirects and internal links - use "Nix language" consistently --- doc/manual/local.mk | 8 +- doc/manual/redirects.js | 207 +++++++++--------- doc/manual/src/SUMMARY.md.in | 23 +- doc/manual/src/command-ref/nix-env.md | 2 +- doc/manual/src/command-ref/nix-instantiate.md | 4 +- doc/manual/src/command-ref/nix-store.md | 2 +- doc/manual/src/command-ref/opt-common.md | 4 +- .../src/expressions/arguments-variables.md | 80 ------- doc/manual/src/expressions/build-script.md | 70 ------ .../src/expressions/expression-syntax.md | 93 -------- doc/manual/src/expressions/generic-builder.md | 66 ------ .../expressions/simple-building-testing.md | 61 ------ .../src/expressions/simple-expression.md | 23 -- .../expressions/writing-nix-expressions.md | 12 - doc/manual/src/glossary.md | 2 +- .../advanced-attributes.md | 0 .../builtin-constants.md | 0 .../builtins-prefix.md | 0 .../builtins-suffix.md | 0 .../constructs.md} | 0 .../{expressions => language}/derivations.md | 0 .../index.md} | 5 +- .../operators.md} | 2 +- .../language-values.md => language/values.md} | 0 .../package-management/package-management.md | 3 +- 25 files changed, 124 insertions(+), 543 deletions(-) delete mode 100644 doc/manual/src/expressions/arguments-variables.md delete mode 100644 doc/manual/src/expressions/build-script.md delete mode 100644 doc/manual/src/expressions/expression-syntax.md delete mode 100644 doc/manual/src/expressions/generic-builder.md delete mode 100644 doc/manual/src/expressions/simple-building-testing.md delete mode 100644 doc/manual/src/expressions/simple-expression.md delete mode 100644 doc/manual/src/expressions/writing-nix-expressions.md rename doc/manual/src/{expressions => language}/advanced-attributes.md (100%) rename doc/manual/src/{expressions => language}/builtin-constants.md (100%) rename doc/manual/src/{expressions => language}/builtins-prefix.md (100%) rename doc/manual/src/{expressions => language}/builtins-suffix.md (100%) rename doc/manual/src/{expressions/language-constructs.md => language/constructs.md} (100%) rename doc/manual/src/{expressions => language}/derivations.md (100%) rename doc/manual/src/{expressions/expression-language.md => language/index.md} (84%) rename doc/manual/src/{expressions/language-operators.md => language/operators.md} (99%) rename doc/manual/src/{expressions/language-values.md => language/values.md} (100%) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 371ed6f21..02a88e4fb 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -61,10 +61,10 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix - @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp +$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp - @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp + @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -97,7 +97,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(call rwildcard, $(d)/src, *.md) $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 19f928c7e..167e221b8 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -132,113 +132,106 @@ var redirects = { "#sec-common-options": "command-ref/opt-common.html", "#ch-utilities": "command-ref/utilities.html", "#chap-hacking": "contributing/hacking.html", - "#adv-attr-allowSubstitutes": "expressions/advanced-attributes.html#adv-attr-allowSubstitutes", - "#adv-attr-allowedReferences": "expressions/advanced-attributes.html#adv-attr-allowedReferences", - "#adv-attr-allowedRequisites": "expressions/advanced-attributes.html#adv-attr-allowedRequisites", - "#adv-attr-disallowedReferences": "expressions/advanced-attributes.html#adv-attr-disallowedReferences", - "#adv-attr-disallowedRequisites": "expressions/advanced-attributes.html#adv-attr-disallowedRequisites", - "#adv-attr-exportReferencesGraph": "expressions/advanced-attributes.html#adv-attr-exportReferencesGraph", - "#adv-attr-impureEnvVars": "expressions/advanced-attributes.html#adv-attr-impureEnvVars", - "#adv-attr-outputHash": "expressions/advanced-attributes.html#adv-attr-outputHash", - "#adv-attr-outputHashAlgo": "expressions/advanced-attributes.html#adv-attr-outputHashAlgo", - "#adv-attr-outputHashMode": "expressions/advanced-attributes.html#adv-attr-outputHashMode", - "#adv-attr-passAsFile": "expressions/advanced-attributes.html#adv-attr-passAsFile", - "#adv-attr-preferLocalBuild": "expressions/advanced-attributes.html#adv-attr-preferLocalBuild", - "#fixed-output-drvs": "expressions/advanced-attributes.html#adv-attr-outputHash", - "#sec-advanced-attributes": "expressions/advanced-attributes.html", - "#sec-arguments": "expressions/arguments-variables.html", - "#sec-build-script": "expressions/build-script.html", - "#builtin-abort": "expressions/builtins.html#builtins-abort", - "#builtin-add": "expressions/builtins.html#builtins-add", - "#builtin-all": "expressions/builtins.html#builtins-all", - "#builtin-any": "expressions/builtins.html#builtins-any", - "#builtin-attrNames": "expressions/builtins.html#builtins-attrNames", - "#builtin-attrValues": "expressions/builtins.html#builtins-attrValues", - "#builtin-baseNameOf": "expressions/builtins.html#builtins-baseNameOf", - "#builtin-bitAnd": "expressions/builtins.html#builtins-bitAnd", - "#builtin-bitOr": "expressions/builtins.html#builtins-bitOr", - "#builtin-bitXor": "expressions/builtins.html#builtins-bitXor", - "#builtin-builtins": "expressions/builtins.html#builtins-builtins", - "#builtin-compareVersions": "expressions/builtins.html#builtins-compareVersions", - "#builtin-concatLists": "expressions/builtins.html#builtins-concatLists", - "#builtin-concatStringsSep": "expressions/builtins.html#builtins-concatStringsSep", - "#builtin-currentSystem": "expressions/builtins.html#builtins-currentSystem", - "#builtin-deepSeq": "expressions/builtins.html#builtins-deepSeq", - "#builtin-derivation": "expressions/builtins.html#builtins-derivation", - "#builtin-dirOf": "expressions/builtins.html#builtins-dirOf", - "#builtin-div": "expressions/builtins.html#builtins-div", - "#builtin-elem": "expressions/builtins.html#builtins-elem", - "#builtin-elemAt": "expressions/builtins.html#builtins-elemAt", - "#builtin-fetchGit": "expressions/builtins.html#builtins-fetchGit", - "#builtin-fetchTarball": "expressions/builtins.html#builtins-fetchTarball", - "#builtin-fetchurl": "expressions/builtins.html#builtins-fetchurl", - "#builtin-filterSource": "expressions/builtins.html#builtins-filterSource", - "#builtin-foldl-prime": "expressions/builtins.html#builtins-foldl-prime", - "#builtin-fromJSON": "expressions/builtins.html#builtins-fromJSON", - "#builtin-functionArgs": "expressions/builtins.html#builtins-functionArgs", - "#builtin-genList": "expressions/builtins.html#builtins-genList", - "#builtin-getAttr": "expressions/builtins.html#builtins-getAttr", - "#builtin-getEnv": "expressions/builtins.html#builtins-getEnv", - "#builtin-hasAttr": "expressions/builtins.html#builtins-hasAttr", - "#builtin-hashFile": "expressions/builtins.html#builtins-hashFile", - "#builtin-hashString": "expressions/builtins.html#builtins-hashString", - "#builtin-head": "expressions/builtins.html#builtins-head", - "#builtin-import": "expressions/builtins.html#builtins-import", - "#builtin-intersectAttrs": "expressions/builtins.html#builtins-intersectAttrs", - "#builtin-isAttrs": "expressions/builtins.html#builtins-isAttrs", - "#builtin-isBool": "expressions/builtins.html#builtins-isBool", - "#builtin-isFloat": "expressions/builtins.html#builtins-isFloat", - "#builtin-isFunction": "expressions/builtins.html#builtins-isFunction", - "#builtin-isInt": "expressions/builtins.html#builtins-isInt", - "#builtin-isList": "expressions/builtins.html#builtins-isList", - "#builtin-isNull": "expressions/builtins.html#builtins-isNull", - "#builtin-isString": "expressions/builtins.html#builtins-isString", - "#builtin-length": "expressions/builtins.html#builtins-length", - "#builtin-lessThan": "expressions/builtins.html#builtins-lessThan", - "#builtin-listToAttrs": "expressions/builtins.html#builtins-listToAttrs", - "#builtin-map": "expressions/builtins.html#builtins-map", - "#builtin-match": "expressions/builtins.html#builtins-match", - "#builtin-mul": "expressions/builtins.html#builtins-mul", - "#builtin-parseDrvName": "expressions/builtins.html#builtins-parseDrvName", - "#builtin-path": "expressions/builtins.html#builtins-path", - "#builtin-pathExists": "expressions/builtins.html#builtins-pathExists", - "#builtin-placeholder": "expressions/builtins.html#builtins-placeholder", - "#builtin-readDir": "expressions/builtins.html#builtins-readDir", - "#builtin-readFile": "expressions/builtins.html#builtins-readFile", - "#builtin-removeAttrs": "expressions/builtins.html#builtins-removeAttrs", - "#builtin-replaceStrings": "expressions/builtins.html#builtins-replaceStrings", - "#builtin-seq": "expressions/builtins.html#builtins-seq", - "#builtin-sort": "expressions/builtins.html#builtins-sort", - "#builtin-split": "expressions/builtins.html#builtins-split", - "#builtin-splitVersion": "expressions/builtins.html#builtins-splitVersion", - "#builtin-stringLength": "expressions/builtins.html#builtins-stringLength", - "#builtin-sub": "expressions/builtins.html#builtins-sub", - "#builtin-substring": "expressions/builtins.html#builtins-substring", - "#builtin-tail": "expressions/builtins.html#builtins-tail", - "#builtin-throw": "expressions/builtins.html#builtins-throw", - "#builtin-toFile": "expressions/builtins.html#builtins-toFile", - "#builtin-toJSON": "expressions/builtins.html#builtins-toJSON", - "#builtin-toPath": "expressions/builtins.html#builtins-toPath", - "#builtin-toString": "expressions/builtins.html#builtins-toString", - "#builtin-toXML": "expressions/builtins.html#builtins-toXML", - "#builtin-trace": "expressions/builtins.html#builtins-trace", - "#builtin-tryEval": "expressions/builtins.html#builtins-tryEval", - "#builtin-typeOf": "expressions/builtins.html#builtins-typeOf", - "#ssec-builtins": "expressions/builtins.html", - "#attr-system": "expressions/derivations.html#attr-system", - "#ssec-derivation": "expressions/derivations.html", - "#ch-expression-language": "expressions/expression-language.html", - "#sec-expression-syntax": "expressions/expression-syntax.html", - "#sec-generic-builder": "expressions/generic-builder.html", - "#sec-constructs": "expressions/language-constructs.html", - "#sect-let-expressions": "expressions/language-constructs.html#let-expressions", - "#ss-functions": "expressions/language-constructs.html#functions", - "#sec-language-operators": "expressions/language-operators.html", - "#table-operators": "expressions/language-operators.html", - "#ssec-values": "expressions/language-values.html", - "#sec-building-simple": "expressions/simple-building-testing.html", - "#ch-simple-expression": "expressions/simple-expression.html", - "#chap-writing-nix-expressions": "expressions/writing-nix-expressions.html", + "#adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", + "#adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", + "#adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", + "#adv-attr-disallowedReferences": "language/advanced-attributes.html#adv-attr-disallowedReferences", + "#adv-attr-disallowedRequisites": "language/advanced-attributes.html#adv-attr-disallowedRequisites", + "#adv-attr-exportReferencesGraph": "language/advanced-attributes.html#adv-attr-exportReferencesGraph", + "#adv-attr-impureEnvVars": "language/advanced-attributes.html#adv-attr-impureEnvVars", + "#adv-attr-outputHash": "language/advanced-attributes.html#adv-attr-outputHash", + "#adv-attr-outputHashAlgo": "language/advanced-attributes.html#adv-attr-outputHashAlgo", + "#adv-attr-outputHashMode": "language/advanced-attributes.html#adv-attr-outputHashMode", + "#adv-attr-passAsFile": "language/advanced-attributes.html#adv-attr-passAsFile", + "#adv-attr-preferLocalBuild": "language/advanced-attributes.html#adv-attr-preferLocalBuild", + "#fixed-output-drvs": "language/advanced-attributes.html#adv-attr-outputHash", + "#sec-advanced-attributes": "language/advanced-attributes.html", + "#builtin-abort": "language/builtins.html#builtins-abort", + "#builtin-add": "language/builtins.html#builtins-add", + "#builtin-all": "language/builtins.html#builtins-all", + "#builtin-any": "language/builtins.html#builtins-any", + "#builtin-attrNames": "language/builtins.html#builtins-attrNames", + "#builtin-attrValues": "language/builtins.html#builtins-attrValues", + "#builtin-baseNameOf": "language/builtins.html#builtins-baseNameOf", + "#builtin-bitAnd": "language/builtins.html#builtins-bitAnd", + "#builtin-bitOr": "language/builtins.html#builtins-bitOr", + "#builtin-bitXor": "language/builtins.html#builtins-bitXor", + "#builtin-builtins": "language/builtins.html#builtins-builtins", + "#builtin-compareVersions": "language/builtins.html#builtins-compareVersions", + "#builtin-concatLists": "language/builtins.html#builtins-concatLists", + "#builtin-concatStringsSep": "language/builtins.html#builtins-concatStringsSep", + "#builtin-currentSystem": "language/builtins.html#builtins-currentSystem", + "#builtin-deepSeq": "language/builtins.html#builtins-deepSeq", + "#builtin-derivation": "language/builtins.html#builtins-derivation", + "#builtin-dirOf": "language/builtins.html#builtins-dirOf", + "#builtin-div": "language/builtins.html#builtins-div", + "#builtin-elem": "language/builtins.html#builtins-elem", + "#builtin-elemAt": "language/builtins.html#builtins-elemAt", + "#builtin-fetchGit": "language/builtins.html#builtins-fetchGit", + "#builtin-fetchTarball": "language/builtins.html#builtins-fetchTarball", + "#builtin-fetchurl": "language/builtins.html#builtins-fetchurl", + "#builtin-filterSource": "language/builtins.html#builtins-filterSource", + "#builtin-foldl-prime": "language/builtins.html#builtins-foldl-prime", + "#builtin-fromJSON": "language/builtins.html#builtins-fromJSON", + "#builtin-functionArgs": "language/builtins.html#builtins-functionArgs", + "#builtin-genList": "language/builtins.html#builtins-genList", + "#builtin-getAttr": "language/builtins.html#builtins-getAttr", + "#builtin-getEnv": "language/builtins.html#builtins-getEnv", + "#builtin-hasAttr": "language/builtins.html#builtins-hasAttr", + "#builtin-hashFile": "language/builtins.html#builtins-hashFile", + "#builtin-hashString": "language/builtins.html#builtins-hashString", + "#builtin-head": "language/builtins.html#builtins-head", + "#builtin-import": "language/builtins.html#builtins-import", + "#builtin-intersectAttrs": "language/builtins.html#builtins-intersectAttrs", + "#builtin-isAttrs": "language/builtins.html#builtins-isAttrs", + "#builtin-isBool": "language/builtins.html#builtins-isBool", + "#builtin-isFloat": "language/builtins.html#builtins-isFloat", + "#builtin-isFunction": "language/builtins.html#builtins-isFunction", + "#builtin-isInt": "language/builtins.html#builtins-isInt", + "#builtin-isList": "language/builtins.html#builtins-isList", + "#builtin-isNull": "language/builtins.html#builtins-isNull", + "#builtin-isString": "language/builtins.html#builtins-isString", + "#builtin-length": "language/builtins.html#builtins-length", + "#builtin-lessThan": "language/builtins.html#builtins-lessThan", + "#builtin-listToAttrs": "language/builtins.html#builtins-listToAttrs", + "#builtin-map": "language/builtins.html#builtins-map", + "#builtin-match": "language/builtins.html#builtins-match", + "#builtin-mul": "language/builtins.html#builtins-mul", + "#builtin-parseDrvName": "language/builtins.html#builtins-parseDrvName", + "#builtin-path": "language/builtins.html#builtins-path", + "#builtin-pathExists": "language/builtins.html#builtins-pathExists", + "#builtin-placeholder": "language/builtins.html#builtins-placeholder", + "#builtin-readDir": "language/builtins.html#builtins-readDir", + "#builtin-readFile": "language/builtins.html#builtins-readFile", + "#builtin-removeAttrs": "language/builtins.html#builtins-removeAttrs", + "#builtin-replaceStrings": "language/builtins.html#builtins-replaceStrings", + "#builtin-seq": "language/builtins.html#builtins-seq", + "#builtin-sort": "language/builtins.html#builtins-sort", + "#builtin-split": "language/builtins.html#builtins-split", + "#builtin-splitVersion": "language/builtins.html#builtins-splitVersion", + "#builtin-stringLength": "language/builtins.html#builtins-stringLength", + "#builtin-sub": "language/builtins.html#builtins-sub", + "#builtin-substring": "language/builtins.html#builtins-substring", + "#builtin-tail": "language/builtins.html#builtins-tail", + "#builtin-throw": "language/builtins.html#builtins-throw", + "#builtin-toFile": "language/builtins.html#builtins-toFile", + "#builtin-toJSON": "language/builtins.html#builtins-toJSON", + "#builtin-toPath": "language/builtins.html#builtins-toPath", + "#builtin-toString": "language/builtins.html#builtins-toString", + "#builtin-toXML": "language/builtins.html#builtins-toXML", + "#builtin-trace": "language/builtins.html#builtins-trace", + "#builtin-tryEval": "language/builtins.html#builtins-tryEval", + "#builtin-typeOf": "language/builtins.html#builtins-typeOf", + "#ssec-builtins": "language/builtins.html", + "#attr-system": "language/derivations.html#attr-system", + "#ssec-derivation": "language/derivations.html", + "#ch-expression-language": "language/index.html", + "#sec-constructs": "language/constructs.html", + "#sect-let-language": "language/constructs.html#let-language", + "#ss-functions": "language/constructs.html#functions", + "#sec-language-operators": "language/operators.html", + "#table-operators": "language/operators.html", + "#ssec-values": "language/values.html", "#gloss-closure": "glossary.html#gloss-closure", "#gloss-derivation": "glossary.html#gloss-derivation", "#gloss-deriver": "glossary.html#gloss-deriver", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c8cb72fc0..084c8f442 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,21 +26,14 @@ - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - [Serving a Nix store via S3](package-management/s3-substituter.md) -- [Writing Nix Expressions](expressions/writing-nix-expressions.md) - - [A Simple Nix Expression](expressions/simple-expression.md) - - [Expression Syntax](expressions/expression-syntax.md) - - [Build Script](expressions/build-script.md) - - [Arguments and Variables](expressions/arguments-variables.md) - - [Building and Testing](expressions/simple-building-testing.md) - - [Generic Builder Syntax](expressions/generic-builder.md) - - [Nix Expression Language](expressions/expression-language.md) - - [Data Types](expressions/language-values.md) - - [Language Constructs](expressions/language-constructs.md) - - [Operators](expressions/language-operators.md) - - [Derivations](expressions/derivations.md) - - [Advanced Attributes](expressions/advanced-attributes.md) - - [Built-in Constants](expressions/builtin-constants.md) - - [Built-in Functions](expressions/builtins.md) +- [Nix Language](language/index.md) + - [Data Types](language/values.md) + - [Language Constructs](language/constructs.md) + - [Operators](language/operators.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Built-in Constants](language/builtin-constants.md) + - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index a372c5eae..a5df35d77 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -198,7 +198,7 @@ a number of possible ways: another. - If `--from-expression` is given, *args* are Nix - [functions](../expressions/language-constructs.md#functions) + [functions](../language/constructs.md#functions) that are called with the active Nix expression as their single argument. The derivations returned by those function calls are installed. This allows derivations to be specified in an diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 2e198daed..8f143729e 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -51,7 +51,7 @@ standard input. - `--strict`\ When used with `--eval`, recursively evaluate list elements and attributes. Normally, such sub-expressions are left unevaluated - (since the Nix expression language is lazy). + (since the Nix language is lazy). > **Warning** > @@ -66,7 +66,7 @@ standard input. When used with `--eval`, print the resulting value as an XML representation of the abstract syntax tree rather than as an ATerm. The schema is the same as that used by the [`toXML` - built-in](../expressions/builtins.md). + built-in](../language/builtins.md). - `--read-write-mode`\ When used with `--eval`, perform evaluation in read/write mode so diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index dc8faba68..ecd838e8d 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -121,7 +121,7 @@ Special exit codes: - `102`\ Hash mismatch, the build output was rejected because it does not match the [`outputHash` attribute of the - derivation](../expressions/advanced-attributes.md). + derivation](../language/advanced-attributes.md). - `104`\ Not deterministic, the build succeeded in check mode but the diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 51d7de18a..e612c416f 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -145,7 +145,7 @@ Most Nix commands accept the following command-line options: expression evaluator will automatically try to call functions that it encounters. It can automatically call functions for which every argument has a [default - value](../expressions/language-constructs.md#functions) (e.g., + value](../language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a @@ -164,7 +164,7 @@ Most Nix commands accept the following command-line options: So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](../expressions/builtins.md) for + value [`builtins.currentSystem`](../language/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md deleted file mode 100644 index 12198c879..000000000 --- a/doc/manual/src/expressions/arguments-variables.md +++ /dev/null @@ -1,80 +0,0 @@ -# Arguments and Variables - -The [Nix expression for GNU Hello](expression-syntax.md) is a -function; it is missing some arguments that have to be filled in -somewhere. In the Nix Packages collection this is done in the file -`pkgs/top-level/all-packages.nix`, where all Nix expressions for -packages are imported and called with the appropriate arguments. Here -are some fragments of `all-packages.nix`, with annotations of what -they mean: - -```nix -... - -rec { ① - - hello = import ../applications/misc/hello/ex-1 ② { ③ - inherit fetchurl stdenv perl; - }; - - perl = import ../development/interpreters/perl { ④ - inherit fetchurl stdenv; - }; - - fetchurl = import ../build-support/fetchurl { - inherit stdenv; ... - }; - - stdenv = ...; - -} -``` - -1. This file defines a set of attributes, all of which are concrete - derivations (i.e., not functions). In fact, we define a *mutually - recursive* set of attributes. That is, the attributes can refer to - each other. This is precisely what we want since we want to “plug” - the various packages into each other. - -2. Here we *import* the Nix expression for GNU Hello. The import - operation just loads and returns the specified Nix expression. In - fact, we could just have put the contents of the Nix expression - for GNU Hello in `all-packages.nix` at this point. That would be - completely equivalent, but it would make `all-packages.nix` rather - bulky. - - Note that we refer to `../applications/misc/hello/ex-1`, not - `../applications/misc/hello/ex-1/default.nix`. When you try to - import a directory, Nix automatically appends `/default.nix` to the - file name. - -3. This is where the actual composition takes place. Here we *call* the - function imported from `../applications/misc/hello/ex-1` with a set - containing the things that the function expects, namely `fetchurl`, - `stdenv`, and `perl`. We use inherit again to use the attributes - defined in the surrounding scope (we could also have written - `fetchurl = fetchurl;`, etc.). - - The result of this function call is an actual derivation that can be - built by Nix (since when we fill in the arguments of the function, - what we get is its body, which is the call to `stdenv.mkDerivation` - in the [Nix expression for GNU Hello](expression-syntax.md)). - - > **Note** - > - > Nixpkgs has a convenience function `callPackage` that imports and - > calls a function, filling in any missing arguments by passing the - > corresponding attribute from the Nixpkgs set, like this: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { }; - > ``` - > - > If necessary, you can set or override arguments: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; }; - > ``` - -4. Likewise, we have to instantiate Perl, `fetchurl`, and the standard - environment. diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md deleted file mode 100644 index b1eacae88..000000000 --- a/doc/manual/src/expressions/build-script.md +++ /dev/null @@ -1,70 +0,0 @@ -# Build Script - -Here is the builder referenced from Hello's Nix expression (stored in -`pkgs/applications/misc/hello/ex-1/builder.sh`): - -```bash -source $stdenv/setup ① - -PATH=$perl/bin:$PATH ② - -tar xvfz $src ③ -cd hello-* -./configure --prefix=$out ④ -make ⑤ -make install -``` - -The builder can actually be made a lot shorter by using the *generic -builder* functions provided by `stdenv`, but here we write out the build -steps to elucidate what a builder does. It performs the following steps: - -1. When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the derivation). - This is done to prevent undeclared inputs from being used in the - build process. If for example the `PATH` contained `/usr/bin`, then - you might accidentally use `/usr/bin/gcc`. - - So the first step is to set up the environment. This is done by - calling the `setup` script of the standard environment. The - environment variable `stdenv` points to the location of the - standard environment being used. (It wasn't specified explicitly - as an attribute in Hello's Nix expression, but `mkDerivation` adds - it automatically.) - -2. Since Hello needs Perl, we have to make sure that Perl is in the - `PATH`. The `perl` environment variable points to the location of - the Perl package (since it was passed in as an attribute to the - derivation), so `$perl/bin` is the directory containing the Perl - interpreter. - -3. Now we have to unpack the sources. The `src` attribute was bound to - the result of fetching the Hello source tarball from the network, so - the `src` environment variable points to the location in the Nix - store to which the tarball was downloaded. After unpacking, we `cd` - to the resulting source directory. - - The whole build is performed in a temporary directory created in - `/tmp`, by the way. This directory is removed after the builder - finishes, so there is no need to clean up the sources afterwards. - Also, the temporary directory is always newly created, so you don't - have to worry about files from previous builds interfering with the - current build. - -4. GNU Hello is a typical Autoconf-based package, so we first have to - run its `configure` script. In Nix every package is stored in a - separate location in the Nix store, for instance - `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix - computes this path by cryptographically hashing all attributes of - the derivation. The path is passed to the builder through the `out` - environment variable. So here we give `configure` the parameter - `--prefix=$out` to cause Hello to be installed in the expected - location. - -5. Finally we build Hello (`make`) and install it into the location - specified by `out` (`make install`). - -If you are wondering about the absence of error checking on the result -of various commands called in the builder: this is because the shell -script is evaluated with Bash's `-e` option, which causes the script to -be aborted if any command fails without an error check. diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md deleted file mode 100644 index 6b93e692c..000000000 --- a/doc/manual/src/expressions/expression-syntax.md +++ /dev/null @@ -1,93 +0,0 @@ -# Expression Syntax - -Here is a Nix expression for GNU Hello: - -```nix -{ stdenv, fetchurl, perl }: ① - -stdenv.mkDerivation { ② - name = "hello-2.1.1"; ③ - builder = ./builder.sh; ④ - src = fetchurl { ⑤ - url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - }; - inherit perl; ⑥ -} -``` - -This file is actually already in the Nix Packages collection in -`pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to -place each package in a separate directory and call the single Nix -expression in that directory `default.nix`. The file has the following -elements (referenced from the figure by number): - -1. This states that the expression is a *function* that expects to be - called with three arguments: `stdenv`, `fetchurl`, and `perl`. They - are needed to build Hello, but we don't know how to build them here; - that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages; it provides a - “standard” environment consisting of the things you would expect - in a basic Unix environment: a C/C++ compiler (GCC, to be precise), - the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, - etc. `fetchurl` is a function that downloads files. `perl` is the - Perl interpreter. - - Nix functions generally have the form `{ x, y, ..., z }: e` where - `x`, `y`, etc. are the names of the expected arguments, and where - *e* is the body of the function. So here, the entire remainder of - the file is the body of the function; when given the required - arguments, the body should describe how to build an instance of - the Hello package. - -2. So we have to build a package. Building something from other stuff - is called a *derivation* in Nix (as opposed to sources, which are - built by humans instead of computers). We perform a derivation by - calling `stdenv.mkDerivation`. `mkDerivation` is a function - provided by `stdenv` that builds a package from a set of - *attributes*. A set is just a list of key/value pairs where each - key is a string and each value is an arbitrary Nix - expression. They take the general form `{ name1 = expr1; ... - nameN = exprN; }`. - -3. The attribute `name` specifies the symbolic name and version of - the package. Nix doesn't really care about these things, but they - are used by for instance `nix-env -q` to show a “human-readable” - name for packages. This attribute is required by `mkDerivation`. - -4. The attribute `builder` specifies the builder. This attribute can - sometimes be omitted, in which case `mkDerivation` will fill in a - default builder (which does a `configure; make; make install`, in - essence). Hello is sufficiently simple that the default builder - would suffice, but in this case, we will show an actual builder - for educational purposes. The value `./builder.sh` refers to the - shell script shown in the [next section](build-script.md), - discussed below. - -5. The builder has to know what the sources of the package are. Here, - the attribute `src` is bound to the result of a call to the - `fetchurl` function. Given a URL and a SHA-256 hash of the expected - contents of the file at that URL, this function builds a derivation - that downloads the file and checks its hash. So the sources are a - dependency that like all other dependencies is built before Hello - itself is built. - - Instead of `src` any other name could have been used, and in fact - there can be any number of sources (bound to different attributes). - However, `src` is customary, and it's also expected by the default - builder (which we don't use in this example). - -6. Since the derivation requires Perl, we have to pass the value of the - `perl` function argument to the builder. All attributes in the set - are actually passed as environment variables to the builder, so - declaring an attribute - - ```nix - perl = perl; - ``` - - will do the trick: it binds an attribute `perl` to the function - argument which also happens to be called `perl`. However, it looks a - bit silly, so there is a shorter syntax. The `inherit` keyword - causes the specified attributes to be bound to whatever variables - with the same name happen to be in scope. diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md deleted file mode 100644 index cf26b5f82..000000000 --- a/doc/manual/src/expressions/generic-builder.md +++ /dev/null @@ -1,66 +0,0 @@ -# Generic Builder Syntax - -Recall that the [build script for GNU Hello](build-script.md) looked -something like this: - -```bash -PATH=$perl/bin:$PATH -tar xvfz $src -cd hello-* -./configure --prefix=$out -make -make install -``` - -The builders for almost all Unix packages look like this — set up some -environment variables, unpack the sources, configure, build, and -install. For this reason the standard environment provides some Bash -functions that automate the build process. Here is what a builder using -the generic build facilities looks like: - -```bash -buildInputs="$perl" ① - -source $stdenv/setup ② - -genericBuild ③ -``` - -Here is what each line means: - -1. The `buildInputs` variable tells `setup` to use the indicated - packages as “inputs”. This means that if a package provides a `bin` - subdirectory, it's added to `PATH`; if it has a `include` - subdirectory, it's added to GCC's header search path; and so on. - (This is implemented in a modular way: `setup` tries to source the - file `pkg/nix-support/setup-hook` of all dependencies. These “setup - hooks” can then set up whatever environment variables they want; for - instance, the setup hook for Perl sets the `PERL5LIB` environment - variable to contain the `lib/site_perl` directories of all inputs.) - -2. The function `genericBuild` is defined in the file `$stdenv/setup`. - -3. The final step calls the shell function `genericBuild`, which - performs the steps that were done explicitly in the previous build - script. The generic builder is smart enough to figure out whether - to unpack the sources using `gzip`, `bzip2`, etc. It can be - customised in many ways; see the Nixpkgs manual for details. - -Discerning readers will note that the `buildInputs` could just as well -have been set in the Nix expression, like this: - -```nix - buildInputs = [ perl ]; -``` - -The `perl` attribute can then be removed, and the builder becomes even -shorter: - -```bash -source $stdenv/setup -genericBuild -``` - -In fact, `mkDerivation` provides a default builder that looks exactly -like that, so it is actually possible to omit the builder for Hello -entirely. diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md deleted file mode 100644 index 7f0d8f841..000000000 --- a/doc/manual/src/expressions/simple-building-testing.md +++ /dev/null @@ -1,61 +0,0 @@ -# Building and Testing - -You can now try to build Hello. Of course, you could do `nix-env -f . -iA -hello`, but you may not want to install a possibly broken package just -yet. The best way to test the package is by using the command -`nix-build`, which builds a Nix expression and creates a symlink named -`result` in the current directory: - -```console -$ nix-build -A hello -building path `/nix/store/632d2b22514d...-hello-2.1.1' -hello-2.1.1/ -hello-2.1.1/intl/ -hello-2.1.1/intl/ChangeLog -... - -$ ls -l result -lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1 - -$ ./result/bin/hello -Hello, world! -``` - -The `-A` option selects the `hello` attribute. This is faster than -using the symbolic package name specified by the `name` attribute -(which also happens to be `hello`) and is unambiguous (there can be -multiple packages with the symbolic name `hello`, but there can be -only one attribute in a set named `hello`). - -`nix-build` registers the `./result` symlink as a garbage collection -root, so unless and until you delete the `./result` symlink, the output -of the build will be safely kept on your system. You can use -`nix-build`’s `-o` switch to give the symlink another name. - -Nix has transactional semantics. Once a build finishes successfully, Nix -makes a note of this in its database: it registers that the path denoted -by `out` is now “valid”. If you try to build the derivation again, Nix -will see that the path is already valid and finish immediately. If a -build fails, either because it returns a non-zero exit code, because Nix -or the builder are killed, or because the machine crashes, then the -output paths will not be registered as valid. If you try to build the -derivation again, Nix will remove the output paths if they exist (e.g., -because the builder died half-way through `make -install`) and try again. Note that there is no “negative caching”: Nix -doesn't remember that a build failed, and so a failed build can always -be repeated. This is because Nix cannot distinguish between permanent -failures (e.g., a compiler error due to a syntax error in the source) -and transient failures (e.g., a disk full condition). - -Nix also performs locking. If you run multiple Nix builds -simultaneously, and they try to build the same derivation, the first Nix -instance that gets there will perform the build, while the others block -(or perform other derivations if available) until the build finishes: - -```console -$ nix-build -A hello -waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x' -``` - -So it is always safe to run multiple instances of Nix in parallel (which -isn’t the case with, say, `make`). diff --git a/doc/manual/src/expressions/simple-expression.md b/doc/manual/src/expressions/simple-expression.md deleted file mode 100644 index 857f71b9b..000000000 --- a/doc/manual/src/expressions/simple-expression.md +++ /dev/null @@ -1,23 +0,0 @@ -# A Simple Nix Expression - -This section shows how to add and test the [GNU Hello -package](http://www.gnu.org/software/hello/hello.html) to the Nix -Packages collection. Hello is a program that prints out the text “Hello, -world\!”. - -To add a package to the Nix Packages collection, you generally need to -do three things: - -1. Write a Nix expression for the package. This is a file that - describes all the inputs involved in building the package, such as - dependencies, sources, and so on. - -2. Write a *builder*. This is a shell script that builds the package - from the inputs. (In fact, it can be written in any language, but - typically it's a `bash` shell script.) - -3. Add the package to the file `pkgs/top-level/all-packages.nix`. The - Nix expression written in the first step is a *function*; it - requires other packages in order to build it. In this step you put - it all together, i.e., you call the function with the right - arguments to build the actual package. diff --git a/doc/manual/src/expressions/writing-nix-expressions.md b/doc/manual/src/expressions/writing-nix-expressions.md deleted file mode 100644 index 5664108e7..000000000 --- a/doc/manual/src/expressions/writing-nix-expressions.md +++ /dev/null @@ -1,12 +0,0 @@ -This chapter shows you how to write Nix expressions, which instruct Nix -how to build packages. It starts with a simple example (a Nix expression -for GNU Hello), and then moves on to a more in-depth look at the Nix -expression language. - -> **Note** -> -> This chapter is mostly about the Nix expression language. For more -> extensive information on adding packages to the Nix Packages -> collection (such as functions in the standard environment and coding -> conventions), please consult [its -> manual](http://nixos.org/nixpkgs/manual/). diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 3448b971b..aa0ac78cb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -3,7 +3,7 @@ - [derivation]{#gloss-derivation}\ A description of a build action. The result of a derivation is a store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](expressions/derivations.md). These are + using the [`derivation` primitive](language/derivations.md). These are translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md similarity index 100% rename from doc/manual/src/expressions/advanced-attributes.md rename to doc/manual/src/language/advanced-attributes.md diff --git a/doc/manual/src/expressions/builtin-constants.md b/doc/manual/src/language/builtin-constants.md similarity index 100% rename from doc/manual/src/expressions/builtin-constants.md rename to doc/manual/src/language/builtin-constants.md diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md similarity index 100% rename from doc/manual/src/expressions/builtins-prefix.md rename to doc/manual/src/language/builtins-prefix.md diff --git a/doc/manual/src/expressions/builtins-suffix.md b/doc/manual/src/language/builtins-suffix.md similarity index 100% rename from doc/manual/src/expressions/builtins-suffix.md rename to doc/manual/src/language/builtins-suffix.md diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/language/constructs.md similarity index 100% rename from doc/manual/src/expressions/language-constructs.md rename to doc/manual/src/language/constructs.md diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/language/derivations.md similarity index 100% rename from doc/manual/src/expressions/derivations.md rename to doc/manual/src/language/derivations.md diff --git a/doc/manual/src/expressions/expression-language.md b/doc/manual/src/language/index.md similarity index 84% rename from doc/manual/src/expressions/expression-language.md rename to doc/manual/src/language/index.md index 267fcb983..c4b3abf75 100644 --- a/doc/manual/src/expressions/expression-language.md +++ b/doc/manual/src/language/index.md @@ -1,6 +1,6 @@ -# Nix Expression Language +# Nix Language -The Nix expression language is a pure, lazy, functional language. Purity +The Nix language is a pure, lazy, functional language. Purity means that operations in the language don't have side-effects (for instance, there is no variable assignment). Laziness means that arguments to functions are evaluated only when they are needed. @@ -10,3 +10,4 @@ full-featured, general purpose language. Its main job is to describe packages, compositions of packages, and the variability within packages. This section presents the various features of the language. + diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/language/operators.md similarity index 99% rename from doc/manual/src/expressions/language-operators.md rename to doc/manual/src/language/operators.md index 268b44f4c..32398189d 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/language/operators.md @@ -1,6 +1,6 @@ # Operators -The table below lists the operators in the Nix expression language, in +The table below lists the operators in the Nix language, in order of precedence (from strongest to weakest binding). | Name | Syntax | Associativity | Description | Precedence | diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/language/values.md similarity index 100% rename from doc/manual/src/expressions/language-values.md rename to doc/manual/src/language/values.md diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/package-management.md index bd26a09ab..d528112e2 100644 --- a/doc/manual/src/package-management/package-management.md +++ b/doc/manual/src/package-management/package-management.md @@ -1,5 +1,4 @@ This chapter discusses how to do package management with Nix, i.e., how to obtain, install, upgrade, and erase packages. This is the “user’s” perspective of the Nix system — people who want to *create* -packages should consult the [chapter on writing Nix -expressions](../expressions/writing-nix-expressions.md). +packages should consult the chapter on the [Nix language](../language/index.md). From 523359d133ea8f048f69d8e8b3e5f1b674baf17a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 08:38:12 -0500 Subject: [PATCH 042/191] WIP: Document the design of Nix The current docs are all "how to do things" and no "what is Nix" or "why are things the way they are". I see lots of misconception on the wider internet, and I also think we would benefit from a "living document" to answer some questions people currently turn to the thesis for. I think a new section of the manual can address all these issues. --- doc/manual/local.mk | 6 +++- doc/manual/src/SUMMARY.md.in | 5 +++ doc/manual/src/design/design.md | 5 +++ doc/manual/src/design/overview.md | 13 ++++++++ doc/manual/src/design/store/entries.md | 46 ++++++++++++++++++++++++++ doc/manual/src/design/store/paths.md | 30 +++++++++++++++++ doc/manual/src/design/store/store.md | 5 +++ 7 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 doc/manual/src/design/design.md create mode 100644 doc/manual/src/design/overview.md create mode 100644 doc/manual/src/design/store/entries.md create mode 100644 doc/manual/src/design/store/paths.md create mode 100644 doc/manual/src/design/store/store.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 371ed6f21..a26fb8fcf 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,9 @@ ifeq ($(doc_generate),yes) +MANUAL_SRCS := \ + $(call rwildcard, $(d)/src, *.md) \ + $(call rwildcard, $(d)/src, */*.md) + # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ @@ -97,7 +101,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c8cb72fc0..c320bde3f 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,6 +15,11 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) +- [Design and Data Model](design/design.md) + - [Overview](design/overview.md) + - [The Store Layer](design/store/store.md) + - [Store Entries](design/store/entries.md) + - [Store Paths](design/store/paths.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md new file mode 100644 index 000000000..7d4211764 --- /dev/null +++ b/doc/manual/src/design/design.md @@ -0,0 +1,5 @@ +# Design and Data Model + +Most of the manual is about how to use Nix. +This chapter is about what Nix actually is. +The hope is that it can serve as a reference for key concepts, and also shed light on why things are the way they are. diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md new file mode 100644 index 000000000..05f4a70b2 --- /dev/null +++ b/doc/manual/src/design/overview.md @@ -0,0 +1,13 @@ +# Overview + +Nix is broken into layers that operate fairly independently. + +At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. + +Below that is the Nix *expression language*, in which packages and configurations are written. +These are the layers which users interact with most. + +Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. +The store layer may not be as visible, but this is the heart of Nix. + +This chapter will start there and work up towards the more user-facing interfaces described in the rest of the manual. diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md new file mode 100644 index 000000000..c16c98a36 --- /dev/null +++ b/doc/manual/src/design/store/entries.md @@ -0,0 +1,46 @@ +# Store Entries + +File system data in Nix is organized into *store entries*. +A store entry is the combination of + + - some file system data + - references to store entries + +## File system data + +Nix supports the a similar model of the file system as Git. +Namely, every file system object falls into these three cases: + + - File: arbitrary data + + - Directory: mapping of names to child file system objects. + File children additionally have an executable flag. + + - Symlink: may point anywhere. + In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. + +A bare file as the "root" file system object is allowed. +Note that there is no containing directory object to store its executable bit; it's deemed non-executable by default. + +## References + +Store entries can refer to both other store entries and themselves. + +Store references are normally calculated by scanning the file system data for store paths when a new store entry is created, but this isn't mandatory, as store entries are allowed to have references that don't correspond to contained store paths, and contained store paths that don't correspond to references. + +The references themselves need not be store paths per-se (this is an implementation detail of the store). +But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. +That said, all the references of the store entry must agree on a store dir. +Also the store directory of the references must equal that of any store which contains the store entry doing the referencing. + +## Relocatability + +The two final restrictions of the previous section yield an alternative of view of the same information. +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least once reference. + +This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. + +Lastly, this illustrates the purpose of tracking self references. +Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. +\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md new file mode 100644 index 000000000..1e4369625 --- /dev/null +++ b/doc/manual/src/design/store/paths.md @@ -0,0 +1,30 @@ +# Store Paths + +A store path is a pair of a 20-byte digest and a name. + +Historically it is the triple of those two and also the store directory, but the modern implementation's internal representation is just the pair. +This change is because in the vast majority of cases, the store dir is fully determined by the context in which the store path occurs. + +## String representation + +A store path is rendered as the concatenation of + + - the store directory + + - a path-separator (`/`) + + - the digest rendered as Base-32 (20 bytes becomes 32 bytes) + + - a hyphen (`-`) + + - the name + +Let's take the store path from the very beginning of this manual as an example: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + +This parses like so: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + store dir digest name diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md new file mode 100644 index 000000000..c2431f1c5 --- /dev/null +++ b/doc/manual/src/design/store/store.md @@ -0,0 +1,5 @@ +A Nix store is a collection of *store entries* referred to by *store paths*. +Every store also has a "store directory path", which is a path prefix used for various purposes. + +There are many types of stores, but all of them at least respect this model. +Some however offer additional functionality. From a2b3160f2898c2009d458cc27d5e8ea925cf583b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 09:09:42 -0500 Subject: [PATCH 043/191] Briefly describe the digest of a store path --- doc/manual/src/design/store/paths.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 1e4369625..18ba27ecb 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -28,3 +28,24 @@ This parses like so: /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ store dir digest name + +## The digest + +The calculation of the digest is quite complicated for historical reasons. +Some of the details will be saved for later. + +> Historical note. The 20 byte restriction is because originally a digests were SHA-1 hashes. +> This is no longer true, but longer hashes and other information is still boiled down to 20 bytes. + +Store paths are either content-addressed or "input-addressed". + +Content addressing means that the digest ultimately derives from referred store entry's file system data and references, and thus can be verified (if one knows how it was calculated). + +Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. +Store paths of this sort can not be validated from the content of the store entry. +Rather, the store entry might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. +The signature indicates that someone is vouching for the store entry really being the results of a plan with that digest. + +While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. +That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. +In the parlance of referencing schemes, this means that store paths are not "self-describing". From e64633f98f290cb571df72a667a0a267722fc397 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 10:18:39 -0500 Subject: [PATCH 044/191] Flesh out TOC --- doc/manual/src/SUMMARY.md.in | 6 ++++++ doc/manual/src/design/store/building.md | 0 doc/manual/src/design/store/drvs/ca.md | 0 doc/manual/src/design/store/drvs/drvs.md | 0 doc/manual/src/design/store/drvs/ia.md | 0 doc/manual/src/design/store/input-addressing.md | 1 + doc/manual/src/design/store/nar.md | 1 + doc/manual/src/design/store/netry-ca.md | 1 + 8 files changed, 9 insertions(+) create mode 100644 doc/manual/src/design/store/building.md create mode 100644 doc/manual/src/design/store/drvs/ca.md create mode 100644 doc/manual/src/design/store/drvs/drvs.md create mode 100644 doc/manual/src/design/store/drvs/ia.md create mode 100644 doc/manual/src/design/store/input-addressing.md create mode 100644 doc/manual/src/design/store/nar.md create mode 100644 doc/manual/src/design/store/netry-ca.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c320bde3f..8c4dbcb18 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,6 +20,12 @@ - [The Store Layer](design/store/store.md) - [Store Entries](design/store/entries.md) - [Store Paths](design/store/paths.md) + - [Nix Archives](design/store/nar.md) + - [Content-Addressing Store Entries](design/store/entry-ca.md) + - [Derivations](design/store/drvs/drvs.md) + - [Input-Addressing](design/store/drvs/ia.md) + - [Content-Addressing (Experimental)](design/store/drvs/ca.md) + - [Building](design/store/building.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/ca.md b/doc/manual/src/design/store/drvs/ca.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/ia.md b/doc/manual/src/design/store/drvs/ia.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/input-addressing.md b/doc/manual/src/design/store/input-addressing.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/input-addressing.md @@ -0,0 +1 @@ +TODO diff --git a/doc/manual/src/design/store/nar.md b/doc/manual/src/design/store/nar.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/nar.md @@ -0,0 +1 @@ +TODO diff --git a/doc/manual/src/design/store/netry-ca.md b/doc/manual/src/design/store/netry-ca.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/netry-ca.md @@ -0,0 +1 @@ +TODO From a210504bc7b67defe16fda97a6d2a844ecc9bd2b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 11:41:15 -0400 Subject: [PATCH 045/191] Apply suggestions from code review --- doc/manual/src/design/overview.md | 2 +- doc/manual/src/design/store/entries.md | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 05f4a70b2..792cfefbf 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -10,4 +10,4 @@ These are the layers which users interact with most. Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. The store layer may not be as visible, but this is the heart of Nix. -This chapter will start there and work up towards the more user-facing interfaces described in the rest of the manual. +This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index c16c98a36..2eb4bf90c 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -8,8 +8,8 @@ A store entry is the combination of ## File system data -Nix supports the a similar model of the file system as Git. -Namely, every file system object falls into these three cases: +The nix store uses a simple filesystem model, similar to the one Git uses. +In particular, every file system object falls into these three cases: - File: arbitrary data @@ -20,13 +20,25 @@ Namely, every file system object falls into these three cases: In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. -Note that there is no containing directory object to store its executable bit; it's deemed non-executable by default. +Note that it cannot be executable, though. +This is a consequence of the executable flags being part of the child entries of the directory, rather than the child files themselves. +A root file has no parent directory; so there is no child entry about the root file, and therefore no executable flag for it. +Without a flag saying which, whether root files are executable or non-executable by default must be decided by convention, and the choice of Nix (and git) is to make them non-executable. ## References Store entries can refer to both other store entries and themselves. -Store references are normally calculated by scanning the file system data for store paths when a new store entry is created, but this isn't mandatory, as store entries are allowed to have references that don't correspond to contained store paths, and contained store paths that don't correspond to references. +References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store entries. +For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store entries by searching in all their files. +When we get to building in a future section, this process will be described in precise detail. + +However, scanning for references is not mandatory. +Store entries are allowed to have official references that *don't* correspond to store paths contained in their contents, +and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. +Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. + +This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. The references themselves need not be store paths per-se (this is an implementation detail of the store). But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. @@ -35,8 +47,8 @@ Also the store directory of the references must equal that of any store which co ## Relocatability -The two final restrictions of the previous section yield an alternative of view of the same information. -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least once reference. +The two final restrictions of the previous section yield an alternative view of the same information. +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. From e3a0209a9e279a30ea9381743c77b2737e8ed95e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 12:35:01 -0400 Subject: [PATCH 046/191] Move the bits on relocating store entires to the end They are too advanced for up front. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/design/store/building.md | 15 +++++++++++++++ doc/manual/src/design/store/entries.md | 17 ----------------- doc/manual/src/design/store/relocatability.md | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 doc/manual/src/design/store/relocatability.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8c4dbcb18..1e8b35483 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,6 +26,7 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) + - [Advanced Topic: store dir relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index e69de29bb..5adccbc0b 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -0,0 +1,15 @@ +# Building + +## Scanning for references + +Before in the section on [store entries](../entries.md), we talked abstractly about scanning for references. +Now we can make this concrete. + +After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store entries. +There is a few steps of this, but let's start with the simple case of one input-addressed output first. + +\[Overview of things that need to happen.] + +For example, if Nix thinks `/nix/store/asdfasdfasdf-foo` and `/nix/store/qwerqwerqwer-bar` are paths the data might plausibly reference, Nix will scan all the contents of all files recursively for the "hash parts" `asdfasdfasdf`` and `qwerqwerqwer`. + +\[Explain why whitelist.] diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index 2eb4bf90c..74f429e78 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -39,20 +39,3 @@ and they are also allowed to *not* have references that *do* correspond to store Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. - -The references themselves need not be store paths per-se (this is an implementation detail of the store). -But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. -That said, all the references of the store entry must agree on a store dir. -Also the store directory of the references must equal that of any store which contains the store entry doing the referencing. - -## Relocatability - -The two final restrictions of the previous section yield an alternative view of the same information. -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. - -This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. - -Lastly, this illustrates the purpose of tracking self references. -Store entries without self-references or other references are relocatable, while store paths with self-references aren't. -This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. -\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md new file mode 100644 index 000000000..04b581459 --- /dev/null +++ b/doc/manual/src/design/store/relocatability.md @@ -0,0 +1,15 @@ +## Advanced Topic: store entry relocatability + +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store entry to a store with a different mount point. + +Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. + +~~The two final restrictions of the previous section yield an alternative view of the same information.~~ +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. + +This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. + +Lastly, this illustrates the purpose of tracking self references. +Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. +\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] From 678d75baeaca049042330e543f610d630e8c43b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 13:13:55 -0400 Subject: [PATCH 047/191] Start on the derivations section --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/design/store/drvs/drvs.md | 59 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1e8b35483..a0efdb9cd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,7 +26,7 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store dir relocatability](design/store/relocatability.md) + - [Advanced Topic: store entry relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index e69de29bb..cedb05985 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -0,0 +1,59 @@ +# Derivations + +Derivations are recipes to create store entries. + +Derivations are the heart of Nix. +Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. +This is where Nix comes in. + +Derivations produce data by running arbitrary commands, like Make or Ninja rules. +Unlike those systems, derivations do not produce arbitrary files, but only specific store entries. +They cannot modify the store in any way, other than creating those store entries. +This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. + +Based on the above, we can conceptually break derivations down into 3 parts: + +1. What command will be run? + +2. What existing store entries are needed as inputs? + +3. What store entries will be produced as outputs? + +## What command will be run? + +The original core of Nix was very simple about this, in the mold of traditional Unix. +Commands consist of 3 parts: + +1. Path to executable + +2. Arguments (Excecpt for `argv[0]`, which is taken from the path in the usual way) + +3. Environment variables. + +## What existing store entries are needed as inputs? + +The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" +It's a good questions! + +## What store entries will be produced as outputs? + +## Extra extensions + +### `__structuredAttrs` + +Historically speaking, most users of Nix made GNU Bash with a script the command run, regardless of what they were doing. +Bash variable are automatically created from env vars, but bash also supports array and string-keyed map variables in addition to string variables. +People also usually create derivations using language which also support these richer data types. +It was thus desired a way to get this data from the language "planning" the derivation to language to bash, the language evaluated at "run time". + +`__structuredAttrs` does this by smuggling inside the core derivation format a map of named richer data. +At run time, this becomes two things: + +1. A JSON file containing that map. +2. A bash script setting those variables. + +The bash command can be passed a script which will "source" that Nix-created bash script, setting those variables with the richer data. +The outer script can then do whatever it likes with those richer variables as input. + +However, since derivations can already contain arbitary input sources, the vast majority of `__structuredAttrs` can be handled by upper layers. +We might consider implementing `__structuredAttrs` in higher layers in the future, and simplifying the store layer. From f5386d7059ad2c63b32356082d1f6ecacfc1e93b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 13:15:35 -0400 Subject: [PATCH 048/191] Fix stub file's name --- doc/manual/src/design/store/{netry-ca.md => entry-ca.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/manual/src/design/store/{netry-ca.md => entry-ca.md} (100%) diff --git a/doc/manual/src/design/store/netry-ca.md b/doc/manual/src/design/store/entry-ca.md similarity index 100% rename from doc/manual/src/design/store/netry-ca.md rename to doc/manual/src/design/store/entry-ca.md From a04340f9a1ce3c107b21a11395b4a2348fd19571 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 10:55:30 -0400 Subject: [PATCH 049/191] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 792cfefbf..7790e701e 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -7,7 +7,8 @@ At the top is the *command line interface*, i.e. the argument parsing of the var Below that is the Nix *expression language*, in which packages and configurations are written. These are the layers which users interact with most. -Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. +Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. +It is also used for executing those build plans. The store layer may not be as visible, but this is the heart of Nix. This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." From 75c5191a1f8cf70a11098e89ed33edc203f52dab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 11:04:55 -0400 Subject: [PATCH 050/191] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 7790e701e..bd3d935e6 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -1,6 +1,6 @@ # Overview -Nix is broken into layers that operate fairly independently. +Nix consists of layers that operate fairly independently. At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. From cdb0bf3b65172fd0e366d44f17392b4261c3d925 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 11:15:56 -0400 Subject: [PATCH 051/191] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index bd3d935e6..2d3e9226b 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,7 +5,8 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. Below that is the Nix *expression language*, in which packages and configurations are written. -These are the layers which users interact with most. + +The command line and expression language are what users interact with most. Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. It is also used for executing those build plans. From e308602fdfabf0ff1f0cd95baf1d90ba54a02281 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Apr 2022 20:48:03 -0400 Subject: [PATCH 052/191] Update doc/manual/src/design/store/drvs/drvs.md Co-authored-by: Matthieu Coudron --- doc/manual/src/design/store/drvs/drvs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index cedb05985..98e95809b 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -26,7 +26,7 @@ Commands consist of 3 parts: 1. Path to executable -2. Arguments (Excecpt for `argv[0]`, which is taken from the path in the usual way) +2. Arguments (Except for `argv[0]`, which is taken from the path in the usual way) 3. Environment variables. From 4e2d5ae2027a19ab12d6d024cdf5125c3200a7a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 21:53:39 -0400 Subject: [PATCH 053/191] doc: Store entry -> store object This matches the terminology in Eelco's thesis. --- doc/manual/src/SUMMARY.md.in | 6 +++--- doc/manual/src/design/overview.md | 2 +- doc/manual/src/design/store/building.md | 4 ++-- doc/manual/src/design/store/drvs/drvs.md | 14 ++++++------- doc/manual/src/design/store/entries.md | 20 +++++++++---------- doc/manual/src/design/store/paths.md | 8 ++++---- doc/manual/src/design/store/relocatability.md | 10 +++++----- doc/manual/src/design/store/store.md | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a0efdb9cd..5c4cfc920 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,15 +18,15 @@ - [Design and Data Model](design/design.md) - [Overview](design/overview.md) - [The Store Layer](design/store/store.md) - - [Store Entries](design/store/entries.md) + - [Store Objects](design/store/entries.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Entries](design/store/entry-ca.md) + - [Content-Addressing Store Objects](design/store/entry-ca.md) - [Derivations](design/store/drvs/drvs.md) - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store entry relocatability](design/store/relocatability.md) + - [Advanced Topic: store object relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 2d3e9226b..9329f2ad7 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -2,7 +2,7 @@ Nix consists of layers that operate fairly independently. -At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. +At the top is the *command line interface*. Below that is the Nix *expression language*, in which packages and configurations are written. diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index 5adccbc0b..f97b74952 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -2,10 +2,10 @@ ## Scanning for references -Before in the section on [store entries](../entries.md), we talked abstractly about scanning for references. +Before in the section on [store objects](../entries.md), we talked abstractly about scanning for references. Now we can make this concrete. -After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store entries. +After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. There is a few steps of this, but let's start with the simple case of one input-addressed output first. \[Overview of things that need to happen.] diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index 98e95809b..766a7b47f 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -1,23 +1,23 @@ # Derivations -Derivations are recipes to create store entries. +Derivations are recipes to create store objects. Derivations are the heart of Nix. Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. This is where Nix comes in. Derivations produce data by running arbitrary commands, like Make or Ninja rules. -Unlike those systems, derivations do not produce arbitrary files, but only specific store entries. -They cannot modify the store in any way, other than creating those store entries. +Unlike those systems, derivations do not produce arbitrary files, but only specific store objects. +They cannot modify the store in any way, other than creating those store objects. This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. Based on the above, we can conceptually break derivations down into 3 parts: 1. What command will be run? -2. What existing store entries are needed as inputs? +2. What existing store objects are needed as inputs? -3. What store entries will be produced as outputs? +3. What store objects will be produced as outputs? ## What command will be run? @@ -30,12 +30,12 @@ Commands consist of 3 parts: 3. Environment variables. -## What existing store entries are needed as inputs? +## What existing store objects are needed as inputs? The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" It's a good questions! -## What store entries will be produced as outputs? +## What store objects will be produced as outputs? ## Extra extensions diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index 74f429e78..ddfefa166 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -1,10 +1,10 @@ -# Store Entries +# Store Objects -File system data in Nix is organized into *store entries*. -A store entry is the combination of +File system data in Nix is organized into *store objects*. +A store object is the combination of - some file system data - - references to store entries + - references to store objects ## File system data @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: File children additionally have an executable flag. - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. + In particular, Symlinks that do not point within the containing file system data or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. Note that it cannot be executable, though. @@ -27,15 +27,15 @@ Without a flag saying which, whether root files are executable or non-executable ## References -Store entries can refer to both other store entries and themselves. +Store objects can refer to both other store objects and themselves. -References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store entries. -For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store entries by searching in all their files. +References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store objects. +For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. However, scanning for references is not mandatory. -Store entries are allowed to have official references that *don't* correspond to store paths contained in their contents, +Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. -This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. +This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store object, rather than try to recompute them on the fly by scanning their contents. diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 18ba27ecb..d7d2ef960 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -39,12 +39,12 @@ Some of the details will be saved for later. Store paths are either content-addressed or "input-addressed". -Content addressing means that the digest ultimately derives from referred store entry's file system data and references, and thus can be verified (if one knows how it was calculated). +Content addressing means that the digest ultimately derives from referred store object's file system data and references, and thus can be verified (if one knows how it was calculated). Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. -Store paths of this sort can not be validated from the content of the store entry. -Rather, the store entry might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. -The signature indicates that someone is vouching for the store entry really being the results of a plan with that digest. +Store paths of this sort can not be validated from the content of the store object. +Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. +The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md index 04b581459..5cdbe9486 100644 --- a/doc/manual/src/design/store/relocatability.md +++ b/doc/manual/src/design/store/relocatability.md @@ -1,15 +1,15 @@ -## Advanced Topic: store entry relocatability +## Advanced Topic: store object relocatability -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store entry to a store with a different mount point. +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store object to a store with a different mount point. Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. ~~The two final restrictions of the previous section yield an alternative view of the same information.~~ -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. +Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. -This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. +This corresponds to the observation that a store object with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store object without any references, i.e. thus without a store directory, can exist in any store. Lastly, this illustrates the purpose of tracking self references. -Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +Store objects without self-references or other references are relocatable, while store paths with self-references aren't. This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. \[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index c2431f1c5..e91f0f6e4 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,4 +1,4 @@ -A Nix store is a collection of *store entries* referred to by *store paths*. +A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. There are many types of stores, but all of them at least respect this model. From 838ba26fda23649545cc61a274e20076a2e27892 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 21:57:10 -0400 Subject: [PATCH 054/191] Rename files after store entry -> store object rename --- doc/manual/src/SUMMARY.md.in | 4 ++-- doc/manual/src/design/store/building.md | 2 +- doc/manual/src/design/store/{entry-ca.md => object-ca.md} | 0 doc/manual/src/design/store/{entries.md => objects.md} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename doc/manual/src/design/store/{entry-ca.md => object-ca.md} (100%) rename doc/manual/src/design/store/{entries.md => objects.md} (100%) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5c4cfc920..323a569d1 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,10 +18,10 @@ - [Design and Data Model](design/design.md) - [Overview](design/overview.md) - [The Store Layer](design/store/store.md) - - [Store Objects](design/store/entries.md) + - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Objects](design/store/entry-ca.md) + - [Content-Addressing Store Objects](design/store/object-ca.md) - [Derivations](design/store/drvs/drvs.md) - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index f97b74952..f4f2649a3 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -2,7 +2,7 @@ ## Scanning for references -Before in the section on [store objects](../entries.md), we talked abstractly about scanning for references. +Before in the section on [store objects](../objects.md), we talked abstractly about scanning for references. Now we can make this concrete. After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. diff --git a/doc/manual/src/design/store/entry-ca.md b/doc/manual/src/design/store/object-ca.md similarity index 100% rename from doc/manual/src/design/store/entry-ca.md rename to doc/manual/src/design/store/object-ca.md diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/objects.md similarity index 100% rename from doc/manual/src/design/store/entries.md rename to doc/manual/src/design/store/objects.md From 1bbad62c7d5f3b0d18c3fdb5a8e947ae5232139f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:02:21 -0400 Subject: [PATCH 055/191] doc: File system data -> file system object, to match Nix --- doc/manual/src/design/store/objects.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index ddfefa166..8783a1648 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -3,10 +3,10 @@ File system data in Nix is organized into *store objects*. A store object is the combination of - - some file system data + - A (root) file system object - references to store objects -## File system data +## File system objects The nix store uses a simple filesystem model, similar to the one Git uses. In particular, every file system object falls into these three cases: @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: File children additionally have an executable flag. - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing file system data or that of another store object referenced by the containing store object are allowed, but might not function as intended. + In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. Note that it cannot be executable, though. @@ -29,7 +29,7 @@ Without a flag saying which, whether root files are executable or non-executable Store objects can refer to both other store objects and themselves. -References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store objects. +References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. From 5f4d2ac091342c26d1a3f447961932139ed568b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:21:29 -0400 Subject: [PATCH 056/191] Improve store object section In particular, Nix is *not* like Git, so that needs to be fixed. --- doc/manual/src/design/store/objects.md | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 8783a1648..2c2ed2bdb 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -1,34 +1,46 @@ # Store Objects File system data in Nix is organized into *store objects*. -A store object is the combination of +A store object is the pair of - A (root) file system object - - references to store objects + - A set of references to store objects ## File system objects -The nix store uses a simple filesystem model, similar to the one Git uses. +The nix store uses a simple filesystem model. In particular, every file system object falls into these three cases: - - File: arbitrary data + - File: an executable flag, and arbitrary data - Directory: mapping of names to child file system objects. - File children additionally have an executable flag. - Symlink: may point anywhere. + In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. -A bare file as the "root" file system object is allowed. -Note that it cannot be executable, though. -This is a consequence of the executable flags being part of the child entries of the directory, rather than the child files themselves. -A root file has no parent directory; so there is no child entry about the root file, and therefore no executable flag for it. -Without a flag saying which, whether root files are executable or non-executable by default must be decided by convention, and the choice of Nix (and git) is to make them non-executable. +A bare file or symlink as the "root" file system object is allowed. + +### Comparison with Git + +This is close to Git's model, but with one crucial difference: +Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + +So long as the root object is a directory, the representations are isomorphic. +There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. + +However, if the root object is a file, there is loss of fidelity. +Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. + +Git's model matches Unix tradition, but Nix's model is more natural. ## References Store objects can refer to both other store objects and themselves. +Self-reference may seem pointless, but tracking them is in fact useful. +We can best explain why later after more concepts have been established. + References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. From b4df351880bb019f6b3176f739059440f128dc43 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:28:26 -0400 Subject: [PATCH 057/191] Relocability -> relocation in store object title --- doc/manual/src/design/store/relocatability.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md index 5cdbe9486..c7f869135 100644 --- a/doc/manual/src/design/store/relocatability.md +++ b/doc/manual/src/design/store/relocatability.md @@ -1,8 +1,8 @@ -## Advanced Topic: store object relocatability +## Advanced Topic: Store object relocation -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store object to a store with a different mount point. +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissible to relocate a store object to a store with a different mount point. -Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. +Recall from the section on [store paths](./store-paths.md) that concrete store paths look like `/-`. ~~The two final restrictions of the previous section yield an alternative view of the same information.~~ Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. From 55b437b5516dfdcd4358de48e7496b08d7be39c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:56:21 -0400 Subject: [PATCH 058/191] Improve store path section --- doc/manual/src/design/store/objects.md | 2 +- doc/manual/src/design/store/paths.md | 37 +++++++++++++++++++------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 2c2ed2bdb..8431958a5 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. + In particular, symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file or symlink as the "root" file system object is allowed. diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index d7d2ef960..1a00de5cb 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -2,18 +2,15 @@ A store path is a pair of a 20-byte digest and a name. -Historically it is the triple of those two and also the store directory, but the modern implementation's internal representation is just the pair. -This change is because in the vast majority of cases, the store dir is fully determined by the context in which the store path occurs. - ## String representation A store path is rendered as the concatenation of - - the store directory + - a store directory - a path-separator (`/`) - - the digest rendered as Base-32 (20 bytes becomes 32 bytes) + - the digest rendered as Base-32 (20 arbitrary bytes becomes 32 ASCII chars) - a hyphen (`-`) @@ -21,14 +18,32 @@ A store path is rendered as the concatenation of Let's take the store path from the very beginning of this manual as an example: - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 This parses like so: - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ store dir digest name +We then can discard the store dir to recover the conceptual pair that is a store path: + + { + digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", + name: "firefox-33.1", + } + +### Where did the "store directory" come from? + +If you notice, the above references a "store directory", but that is *not* part of the definition of a store path. +We can discard it when parsing, but what about when printing? +We need to get a store directory from *somewhere*. + +The answer is, the store directory is a property of the store that contains the store path. +The explanation for this is simple enough: a store is notionally mounted as a directory at some location, and the store object's root file system likewise mounted at this path within that directory. + +This does, however, mean the string representation of a store path is not derived just from the store path itself, but is in fact "context dependent". + ## The digest The calculation of the digest is quite complicated for historical reasons. @@ -39,10 +54,14 @@ Some of the details will be saved for later. Store paths are either content-addressed or "input-addressed". -Content addressing means that the digest ultimately derives from referred store object's file system data and references, and thus can be verified (if one knows how it was calculated). +Content addressing means that the digest ultimately derives from referred store object's file system objects and references, and thus can be verified. +There is more than one *method* of content-addressing, however. +Still, if one does know the the content addressing schema that was used, +(or guesses, there isn't that many yet!) +one can recalcuate the store path and thus verify the store object. Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. -Store paths of this sort can not be validated from the content of the store object. +Store paths of this sort can *not* be validated from the content of the store object. Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. From b98dc3b19c62abb3b8edc778cee0ac8234c5577d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 23:03:38 -0400 Subject: [PATCH 059/191] store objects, better opining sentances --- doc/manual/src/design/store/objects.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 8431958a5..38c73bc7a 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -1,6 +1,6 @@ # Store Objects -File system data in Nix is organized into *store objects*. +Data in Nix is chiefly organized into *store objects*. A store object is the pair of - A (root) file system object @@ -8,7 +8,7 @@ A store object is the pair of ## File system objects -The nix store uses a simple filesystem model. +The Nix store uses a simple filesystem model. In particular, every file system object falls into these three cases: - File: an executable flag, and arbitrary data From e4eea5e84e78cc57859cca9295da450258e8f55b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 01:44:02 -0400 Subject: [PATCH 060/191] Include abstract syntax based on the thesis for FSOs See https://edolstra.github.io/pubs/phd-thesis.pdf, page 91. --- doc/manual/src/design/store/objects.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 38c73bc7a..0f3d2a499 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -9,6 +9,16 @@ A store object is the pair of ## File system objects The Nix store uses a simple filesystem model. + + data FileSystemObject + = Regular Executable ByteString + | Directory (Map FileName FSO) + | SymLink ByteString + + data Executable + = Executable + | NonExecutable + In particular, every file system object falls into these three cases: - File: an executable flag, and arbitrary data @@ -26,10 +36,21 @@ A bare file or symlink as the "root" file system object is allowed. This is close to Git's model, but with one crucial difference: Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + data GitObject + = Blob ByteString + | Tree (Map FileName (Persission, FSO)) + + data Persission + = Directory -- IFF paired with tree + -- Iff paired with blob, one of: + | RegFile + | ExecutableFile + | Symlink + So long as the root object is a directory, the representations are isomorphic. There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. -However, if the root object is a file, there is loss of fidelity. +However, if the root object is a blob, there is loss of fidelity. Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. Git's model matches Unix tradition, but Nix's model is more natural. From 4e4bbd9e832e810b4e4cf33aced0ec0322cf55b6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 02:02:11 -0400 Subject: [PATCH 061/191] Improve store objects session more --- doc/manual/src/design/store/paths.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 1a00de5cb..cf51eb866 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -47,20 +47,28 @@ This does, however, mean the string representation of a store path is not derive ## The digest The calculation of the digest is quite complicated for historical reasons. -Some of the details will be saved for later. +The details of the algorithms will be discussed later once more concepts have been introduced. +For now, we just concern ourselves with the *key properties* of those algorithms. -> Historical note. The 20 byte restriction is because originally a digests were SHA-1 hashes. -> This is no longer true, but longer hashes and other information is still boiled down to 20 bytes. +::: {.note} +**Historical note** The 20 byte restriction is because originally a digests were SHA-1 hashes. +This is no longer true, but longer hashes and other information are still boiled down to 20 bytes. +::: -Store paths are either content-addressed or "input-addressed". +Store paths are either *content-addressed* or *input-addressed*. -Content addressing means that the digest ultimately derives from referred store object's file system objects and references, and thus can be verified. +::: {.note} +The former is a standard term used elsewhere. +The later is our own creation to evoke a contrast with content addressing. +::: + +Content addressing means that the store path digest ultimately derives from referred store object's contents, namely its file system objects and references. There is more than one *method* of content-addressing, however. -Still, if one does know the the content addressing schema that was used, +Still, if one does know the content addressing schema that was used, (or guesses, there isn't that many yet!) -one can recalcuate the store path and thus verify the store object. +one can recalculate the store path and thus verify the store object. -Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. +Input addressing means that the store path digest derives from how the store path was produced, namely the "inputs" and plan that it was built from. Store paths of this sort can *not* be validated from the content of the store object. Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. From c86c1ec7e31c2725be20e4087845e96878134846 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 02:20:15 -0400 Subject: [PATCH 062/191] Make refernces sneak preview more concise --- doc/manual/src/design/store/objects.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 0f3d2a499..e4f49a170 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -62,9 +62,8 @@ Store objects can refer to both other store objects and themselves. Self-reference may seem pointless, but tracking them is in fact useful. We can best explain why later after more concepts have been established. -References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. -For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. -When we get to building in a future section, this process will be described in precise detail. +References are normally calculated so as to to record the presence of textual references in store object's file systems obejcts. +This process will be described precisely in the section on [building](./building.md), once more concepts are explained, as building is the primary path new store objects with non-trivial references are created. However, scanning for references is not mandatory. Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, From 0737094161cc2b13a6ec06a58a78200cb2116b88 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Apr 2022 01:39:30 -0400 Subject: [PATCH 063/191] Add draft "Rosetta stone" by @fricklerhandwerk and stub commentary The idea and most of the execution are @fricklerhandwerk's. I changed a few things best I could based on @edolstra's corrections, and a Bazel glossary. Valentin Gagarin --- doc/manual/src/SUMMARY.md.in | 3 +- doc/manual/src/design/store/related-work.md | 37 +++++++++++++++++++++ doc/manual/src/design/store/store.md | 22 ++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 doc/manual/src/design/store/related-work.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 323a569d1..9cde44a0d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,7 +26,8 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store object relocatability](design/store/relocatability.md) + - [Advanced Topic: Store object relocatability](design/store/relocatability.md) + - [Advanced Topic: Related work](design/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/related-work.md b/doc/manual/src/design/store/related-work.md new file mode 100644 index 000000000..b64b41988 --- /dev/null +++ b/doc/manual/src/design/store/related-work.md @@ -0,0 +1,37 @@ +# Advanced Topic: Related Work + +## Bazel + +TODO skylark and layering. + +TODO being monadic, if RFC 92. + +## Build Systems à la Carte + +TODO user-choosen keys vs keys chosen automatically? +Purity in face of dynamic tasks (no conflicts, guaranteed). + +TODO Does Nix constitute a different way to be be monadic? +Purity of keys, as mentioned. +Dynamic tasks/keys vs dynamic dependencies. +(Not sure yet.) + +## Lazy evaluation + +We clearly have thunks that produce thunks, but less clearly functions that produce functions. + +Do we have open terms? + +Do we hve thunks vs expressions distinction? +c.f. John Shutt's modern fexprs, when the syntax can "leak". + +## Machine models + +TODO +Derivations as store objects via drv files makes Nix a "Von Neumann" archicture. +Can also imagine a "Harvard" archicture where derivations are stored separately? +Can we in general imagine N heaps for N different sorts of objects? + +TODO +Also, leaning on the notion of "builtin builders" more, having multiple different sorts of user-defined builders too. +The builder is a black box as far as the Nix model is concerned. diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index e91f0f6e4..458c1e2b8 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,5 +1,27 @@ +# Nix Store + A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. There are many types of stores, but all of them at least respect this model. Some however offer additional functionality. + +## A Rosetta stone for the Nix store. + +The design of Nix is comparable to other build systems, even programming languages in general. +Here is a rough [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology. +If you are familiar with one of these columns, this might help the following sections make more sense. + +generic build system | Nix | Bazel | Build Systems à la Carte | lazy programming language +-- | -- | -- | -- | -- +data (build input, build result) | component | file (source, target) | value | value +build plan | derivation graph | action graph | `Tasks` | thunk +build step | derivation | rule | `Task` | thunk +build instructions | builder | (depends on action type) | `Task` | function +build | build | build | `Build` applied to arguments | evaluation +persistence layer | store | file system | `Store` | heap + +(n.b. Bazel terms gotten from https://docs.bazel.build/versions/main/glossary.html.) + +Plenty more could be said elaborating these comparisons. +We will save that for the end of this chapter, in the [Related Work](./related-work.md) section. From 0eae4bfad1bfa94347252b51dec8c87fa73644fb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Apr 2022 14:39:35 +0200 Subject: [PATCH 064/191] reword overview with clear terminology trying to capture alternative terms in one go here, mirroring everyday use: derivation - build plan realise - execute build there will be more of that sort. --- doc/manual/src/design/overview.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 9329f2ad7..6a5555dff 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -2,14 +2,22 @@ Nix consists of layers that operate fairly independently. -At the top is the *command line interface*. +At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the Nix *expression language*, in which packages and configurations are written. +Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +It is used to compose expressions which ultimately evaluate to *derivations* — self-contained *build plans* to derive new data from referenced input data. -The command line and expression language are what users interact with most. +::: {.note} +The Nix language itself does not have a notion of *packages* or *configurations*. +As far as we are concerned here, the result of a derivation is just data. +In practice this amounts to a set of files in a file system. +::: -Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. -It is also used for executing those build plans. -The store layer may not be as visible, but this is the heart of Nix. +The command line and Nix language are what users interact with most. + +Underlying everything is the *Nix store*, a mechanism to keep track of derivations, data, and references between them. +It can also *realise derivations*, that is, *execute build instructions* to produce new data. +It uses the file system as a persistence layer, and a database to keep track of references. + +This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. -This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." From 327ccd3b0787f7803a51aa5124cf0cbab694523c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Apr 2022 15:06:05 +0200 Subject: [PATCH 065/191] only use generic build system terminology we will use a translation table to introduce nix-specific terms --- doc/manual/src/design/overview.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 6a5555dff..c60121a01 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,18 +5,18 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to *derivations* — self-contained *build plans* to derive new data from referenced input data. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the result of a derivation is just data. +As far as we are concerned here, the inputs and results of a derivation are just data. In practice this amounts to a set of files in a file system. ::: The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of derivations, data, and references between them. -It can also *realise derivations*, that is, *execute build instructions* to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. +It can also execute *build instructions* captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. From 804e8bd7473516b087f39161bd34730e26480517 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:43:19 +0200 Subject: [PATCH 066/191] indicate sequence with "then" Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index c60121a01..20a3fe653 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -19,5 +19,5 @@ Underlying everything is the *Nix store*, a mechanism to keep track of build pla It can also execute *build instructions* captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. -This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. +This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 23ee0b24f7c0fd43ccf6f99b87941df4cb071689 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:44:00 +0200 Subject: [PATCH 067/191] correctly use comma for nesting Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 20a3fe653..b1adb143b 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -16,7 +16,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions* captured in the build plans, to produce new data. +It can also execute *build instructions*, captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 51e6bed25e073afac5a7b5e11033769a615d0473 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:54:43 +0200 Subject: [PATCH 068/191] do not mention implementation details Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index b1adb143b..0cead62ab 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -17,7 +17,6 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build plans, to produce new data. -It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 89a7c956ffadbc9c17849051f3c91a1225dcfce2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Apr 2022 14:05:50 -0400 Subject: [PATCH 069/191] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 0cead62ab..15ddb562d 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,7 +5,7 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. From b387d809437b0c787691b814bf65e1647fd18009 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 24 Apr 2022 21:13:45 +0200 Subject: [PATCH 070/191] remove sentence for chapter transition idea: sections could be read in different orders by linking them in different ways (e.g. depth-first or breadth-first). adding hard-coded transitions makes that confusing. --- doc/manual/src/design/overview.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 15ddb562d..b5f8b6aad 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -18,5 +18,3 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build plans, to produce new data. -This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. - From 34ea74c9ec21641399a96386094b021d7a2b30f4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Apr 2022 11:46:15 +0200 Subject: [PATCH 071/191] reword introductory section there should be a meta section for each chapter to give motivation of the presented structure. the structure itself is visible from the table of contents. --- doc/manual/src/design/design.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index 7d4211764..f5135bc9a 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -1,5 +1,6 @@ # Design and Data Model Most of the manual is about how to use Nix. -This chapter is about what Nix actually is. -The hope is that it can serve as a reference for key concepts, and also shed light on why things are the way they are. +This chapter is about the technical principles behind Nix. + +It describes each architectural layer and its components in its own section, starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 75981263912805a77890bcdb63c4aa3bbb0d8e09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 21:17:44 +0200 Subject: [PATCH 072/191] remove separate meta-section, add architecture diagram the diagram is a first approximation and only covers that same section. of course there is much more going on, and other features should at some point also be illustrated. we also have to think about presentation format and technicalities behind it. the manual has to render to `man`, but we may want something more refined for web view. --- doc/manual/src/SUMMARY.md.in | 3 +-- doc/manual/src/design/design.md | 39 ++++++++++++++++++++++++++-- doc/manual/src/design/overview.md | 20 -------------- doc/manual/src/design/store/store.md | 2 +- 4 files changed, 39 insertions(+), 25 deletions(-) delete mode 100644 doc/manual/src/design/overview.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9cde44a0d..ec457126a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,8 +16,7 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Design and Data Model](design/design.md) - - [Overview](design/overview.md) - - [The Store Layer](design/store/store.md) + - [Store](design/store/store.md) - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index f5135bc9a..960a68f41 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -1,6 +1,41 @@ # Design and Data Model -Most of the manual is about how to use Nix. This chapter is about the technical principles behind Nix. -It describes each architectural layer and its components in its own section, starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. +## Architecture + +Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). + +``` + [ commmand line interface ] + | + | evaluates + V + [ configuration language ] + | + | evaluates to + | + reference V build +[ build inputs ] --> [ build plans ] --> [ build results ] + \ | / + \ | persisted to / + \ V / + [ store ] +``` + +At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. + +Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. + +::: {.note} +The Nix language itself does not have a notion of *packages* or *configurations*. +As far as we are concerned here, the inputs and results of a derivation are just data. +In practice this amounts to a set of files in a file system. +::: + +The command line and Nix language are what users interact with most. + +Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. +It can also execute *build instructions*, captured in the build plans, to produce new data. + diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md deleted file mode 100644 index b5f8b6aad..000000000 --- a/doc/manual/src/design/overview.md +++ /dev/null @@ -1,20 +0,0 @@ -# Overview - -Nix consists of layers that operate fairly independently. - -At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. - -Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. - -::: {.note} -The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a derivation are just data. -In practice this amounts to a set of files in a file system. -::: - -The command line and Nix language are what users interact with most. - -Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build plans, to produce new data. - diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index 458c1e2b8..d4add52f5 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,4 +1,4 @@ -# Nix Store +# Store A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. From d30033759a07a0d5df7aac4c22bfa274b74baf0e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 21:21:46 +0200 Subject: [PATCH 073/191] address Nix language consistently as configuration language --- doc/manual/src/design/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index 960a68f41..16c51533b 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -25,7 +25,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. ::: {.note} From 39f01176a713bad4e090e19c8a25c2447056640d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 23:25:33 +0200 Subject: [PATCH 074/191] design -> architecture, add motivation following ideas found in Architecture of Gazelle[1] [1]: https://github.com/bazelbuild/bazel-gazelle/blob/56d35f8db086bb65ef876f96f7baa7b71516daf8/Design.rst --- doc/manual/src/SUMMARY.md.in | 2 +- .../src/{design/design.md => architecture/architecture.md} | 7 ++++--- doc/manual/src/{design => architecture}/store/building.md | 0 doc/manual/src/{design => architecture}/store/drvs/ca.md | 0 doc/manual/src/{design => architecture}/store/drvs/drvs.md | 0 doc/manual/src/{design => architecture}/store/drvs/ia.md | 0 .../src/{design => architecture}/store/input-addressing.md | 0 doc/manual/src/{design => architecture}/store/nar.md | 0 doc/manual/src/{design => architecture}/store/object-ca.md | 0 doc/manual/src/{design => architecture}/store/objects.md | 0 doc/manual/src/{design => architecture}/store/paths.md | 0 .../src/{design => architecture}/store/related-work.md | 0 .../src/{design => architecture}/store/relocatability.md | 0 doc/manual/src/{design => architecture}/store/store.md | 0 14 files changed, 5 insertions(+), 4 deletions(-) rename doc/manual/src/{design/design.md => architecture/architecture.md} (88%) rename doc/manual/src/{design => architecture}/store/building.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/ca.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/drvs.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/ia.md (100%) rename doc/manual/src/{design => architecture}/store/input-addressing.md (100%) rename doc/manual/src/{design => architecture}/store/nar.md (100%) rename doc/manual/src/{design => architecture}/store/object-ca.md (100%) rename doc/manual/src/{design => architecture}/store/objects.md (100%) rename doc/manual/src/{design => architecture}/store/paths.md (100%) rename doc/manual/src/{design => architecture}/store/related-work.md (100%) rename doc/manual/src/{design => architecture}/store/relocatability.md (100%) rename doc/manual/src/{design => architecture}/store/store.md (100%) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index ec457126a..c029e30bf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,7 +15,7 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Design and Data Model](design/design.md) +- [Architecture](architecture/architecture.md) - [Store](design/store/store.md) - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/architecture/architecture.md similarity index 88% rename from doc/manual/src/design/design.md rename to doc/manual/src/architecture/architecture.md index 16c51533b..b17eacd2e 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/architecture/architecture.md @@ -1,8 +1,9 @@ -# Design and Data Model +# Architecture -This chapter is about the technical principles behind Nix. +This chapter describes how Nix works. +It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. -## Architecture +## Overview Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/architecture/store/building.md similarity index 100% rename from doc/manual/src/design/store/building.md rename to doc/manual/src/architecture/store/building.md diff --git a/doc/manual/src/design/store/drvs/ca.md b/doc/manual/src/architecture/store/drvs/ca.md similarity index 100% rename from doc/manual/src/design/store/drvs/ca.md rename to doc/manual/src/architecture/store/drvs/ca.md diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md similarity index 100% rename from doc/manual/src/design/store/drvs/drvs.md rename to doc/manual/src/architecture/store/drvs/drvs.md diff --git a/doc/manual/src/design/store/drvs/ia.md b/doc/manual/src/architecture/store/drvs/ia.md similarity index 100% rename from doc/manual/src/design/store/drvs/ia.md rename to doc/manual/src/architecture/store/drvs/ia.md diff --git a/doc/manual/src/design/store/input-addressing.md b/doc/manual/src/architecture/store/input-addressing.md similarity index 100% rename from doc/manual/src/design/store/input-addressing.md rename to doc/manual/src/architecture/store/input-addressing.md diff --git a/doc/manual/src/design/store/nar.md b/doc/manual/src/architecture/store/nar.md similarity index 100% rename from doc/manual/src/design/store/nar.md rename to doc/manual/src/architecture/store/nar.md diff --git a/doc/manual/src/design/store/object-ca.md b/doc/manual/src/architecture/store/object-ca.md similarity index 100% rename from doc/manual/src/design/store/object-ca.md rename to doc/manual/src/architecture/store/object-ca.md diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/architecture/store/objects.md similarity index 100% rename from doc/manual/src/design/store/objects.md rename to doc/manual/src/architecture/store/objects.md diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/architecture/store/paths.md similarity index 100% rename from doc/manual/src/design/store/paths.md rename to doc/manual/src/architecture/store/paths.md diff --git a/doc/manual/src/design/store/related-work.md b/doc/manual/src/architecture/store/related-work.md similarity index 100% rename from doc/manual/src/design/store/related-work.md rename to doc/manual/src/architecture/store/related-work.md diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/architecture/store/relocatability.md similarity index 100% rename from doc/manual/src/design/store/relocatability.md rename to doc/manual/src/architecture/store/relocatability.md diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/architecture/store/store.md similarity index 100% rename from doc/manual/src/design/store/store.md rename to doc/manual/src/architecture/store/store.md From c8c1b705ad5cdf115997cef387d83c4b04fe0660 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 24 Apr 2022 21:34:33 +0200 Subject: [PATCH 075/191] reword section on Nix store --- doc/manual/src/architecture/store/store.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d4add52f5..b52fe8255 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,10 +1,14 @@ # Store -A Nix store is a collection of *store objects* referred to by *store paths*. -Every store also has a "store directory path", which is a path prefix used for various purposes. +A Nix store is a collection of [store objects](objects.md) and associated operations. -There are many types of stores, but all of them at least respect this model. -Some however offer additional functionality. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. + +Store objects are accessible in a file system through [store paths](paths.md). +Every store has a *store directory*, which contains that store’s objects and is a prefix of their store paths. +It defaults to `/nix/store`, but is in principle arbitrary. + +A Nix store can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. ## A Rosetta stone for the Nix store. From 7b5c00f67f729bb3c0d026f3f4d14e727c4cf420 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 01:15:27 +0200 Subject: [PATCH 076/191] add concrete store examples, reword note on file system --- doc/manual/src/architecture/store/store.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b52fe8255..688c7e1a2 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,11 +4,18 @@ A Nix store is a collection of [store objects](objects.md) and associated operat These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -Store objects are accessible in a file system through [store paths](paths.md). -Every store has a *store directory*, which contains that store’s objects and is a prefix of their store paths. -It defaults to `/nix/store`, but is in principle arbitrary. +A Nix store allows to add and retrieve store objects. +It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. +It also keeps track of *references* between data and can therefore garbage-collect unused store objects. -A Nix store can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. +There exist different types of stores, which all follow this model. +Examples: +- store on the local file system +- remote store accessible via SSH +- binary cache store accessible via HTTP + +Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). +The store directory defaults to `/nix/store`, but is in principle arbitrary. ## A Rosetta stone for the Nix store. From 070c85499b370bf886db1a87c04ed9c319bb6323 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 09:44:46 +0200 Subject: [PATCH 077/191] fix grammar Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 688c7e1a2..96796f26c 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,7 +4,7 @@ A Nix store is a collection of [store objects](objects.md) and associated operat These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -A Nix store allows to add and retrieve store objects. +A Nix store allows adding and retrieving store objects. It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. From 5f96a0b4e830bef59b5986c916727d07b244c22c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:18:03 +0200 Subject: [PATCH 078/191] associated operations are not collected Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 96796f26c..624439f73 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,6 +1,6 @@ # Store -A Nix store is a collection of [store objects](objects.md) and associated operations. +A Nix store is a collection of [store objects](objects.md) with associated operations. These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. From 610ddf44aa86924834bf2ac3735c7cb75a6a1f1c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 25 Apr 2022 13:25:43 +0200 Subject: [PATCH 079/191] reword introduction to rosetta stone, add links attempt to explain used and documented terminology, as well as how the declarative programming paradigm relates to building software. in the future one could highlight encouraged terms to shape future material into higher consistency. --- doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/architecture/store/store.md | 29 ++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index b17eacd2e..836b40c91 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 624439f73..af0609d4b 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -17,22 +17,21 @@ Examples: Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). The store directory defaults to `/nix/store`, but is in principle arbitrary. -## A Rosetta stone for the Nix store. +## A [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology -The design of Nix is comparable to other build systems, even programming languages in general. -Here is a rough [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology. -If you are familiar with one of these columns, this might help the following sections make more sense. +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. -generic build system | Nix | Bazel | Build Systems à la Carte | lazy programming language +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- -data (build input, build result) | component | file (source, target) | value | value -build plan | derivation graph | action graph | `Tasks` | thunk -build step | derivation | rule | `Task` | thunk -build instructions | builder | (depends on action type) | `Task` | function -build | build | build | `Build` applied to arguments | evaluation -persistence layer | store | file system | `Store` | heap +data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value +build plan | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build graph | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) +build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build | realisation | build | application of `Build` | evaluation +persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap -(n.b. Bazel terms gotten from https://docs.bazel.build/versions/main/glossary.html.) - -Plenty more could be said elaborating these comparisons. -We will save that for the end of this chapter, in the [Related Work](./related-work.md) section. +All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software +Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf) (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) (2018). From ca5ebf63827f82ae63796d775070b2f9cf828625 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:34:53 +0200 Subject: [PATCH 080/191] revert build plan/step distinction, reorder rows --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index af0609d4b..f4f5a3e16 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -27,9 +27,9 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build plan | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) -build graph | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | realisation | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 40efe5b30b24e75b26017eb68c62acfc6b7ebf93 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:36:36 +0200 Subject: [PATCH 081/191] build instrcution: Task -> function --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f4f5a3e16..e851444c9 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -27,7 +27,7 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | realisation | build | application of `Build` | evaluation From a145007a577fda8a8f3b65b66f4c359410b4af82 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:48:59 +0200 Subject: [PATCH 082/191] component -> store object, realisation -> build --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index e851444c9..74d3d7f84 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,11 +26,11 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- -data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value +data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) -build | realisation | build | application of `Build` | evaluation +build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software From e5e48593c85d052585ff8a275c54823caf575a9d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:16:27 +0200 Subject: [PATCH 083/191] move git comparison to related work it should be pulled out of the branch before we go for merging --- doc/manual/src/architecture/store/objects.md | 23 ------------------ .../src/architecture/store/related-work.md | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index e4f49a170..caa00d862 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -31,29 +31,6 @@ In particular, every file system object falls into these three cases: A bare file or symlink as the "root" file system object is allowed. -### Comparison with Git - -This is close to Git's model, but with one crucial difference: -Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. - - data GitObject - = Blob ByteString - | Tree (Map FileName (Persission, FSO)) - - data Persission - = Directory -- IFF paired with tree - -- Iff paired with blob, one of: - | RegFile - | ExecutableFile - | Symlink - -So long as the root object is a directory, the representations are isomorphic. -There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. - -However, if the root object is a blob, there is loss of fidelity. -Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. - -Git's model matches Unix tradition, but Nix's model is more natural. ## References diff --git a/doc/manual/src/architecture/store/related-work.md b/doc/manual/src/architecture/store/related-work.md index b64b41988..92b7d480e 100644 --- a/doc/manual/src/architecture/store/related-work.md +++ b/doc/manual/src/architecture/store/related-work.md @@ -25,6 +25,30 @@ Do we have open terms? Do we hve thunks vs expressions distinction? c.f. John Shutt's modern fexprs, when the syntax can "leak". +## Comparison with Git file system model + +This is close to Git's model, but with one crucial difference: +Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + + data GitObject + = Blob ByteString + | Tree (Map FileName (Persission, FSO)) + + data Persission + = Directory -- IFF paired with tree + -- Iff paired with blob, one of: + | RegFile + | ExecutableFile + | Symlink + +So long as the root object is a directory, the representations are isomorphic. +There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. + +However, if the root object is a blob, there is loss of fidelity. +Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. + +Git's model matches Unix tradition, but Nix's model is more natural. + ## Machine models TODO From 90fc5b41a83575c0bd008868f989b6e1497ce3a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:18:23 +0200 Subject: [PATCH 084/191] reword file system objects - use singular for the "class" - more consistency in type definition - minor fixes in wording --- doc/manual/src/architecture/store/objects.md | 40 ++++++++------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index caa00d862..872a209c3 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -1,40 +1,32 @@ -# Store Objects +# Store Object -Data in Nix is chiefly organized into *store objects*. +Nix organizes the data it manages into *store objects*. A store object is the pair of - - A (root) file system object - - A set of references to store objects + - a [file system object](#file-system-object) + - a set of [references](#reference) to store objects. -## File system objects +## File system object {#file-system-object} -The Nix store uses a simple filesystem model. +The Nix store uses a simple file system model. data FileSystemObject - = Regular Executable ByteString - | Directory (Map FileName FSO) - | SymLink ByteString - - data Executable - = Executable - | NonExecutable - -In particular, every file system object falls into these three cases: + = File Executable Data + | Directory (Map FileName FileSystemObject) + | SymLink Path +Every file system object is one of the following: - File: an executable flag, and arbitrary data + - Directory: mapping of names to child file system objects + - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - - Directory: mapping of names to child file system objects. + In particular, symlinks pointing outside of their own root file system object, or to a store object without a matching reference, are allowed, but might not function as intended. - - Symlink: may point anywhere. +A bare file or symlink can be a root file system object. - In particular, symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. +## Reference {#reference} -A bare file or symlink as the "root" file system object is allowed. - - -## References - -Store objects can refer to both other store objects and themselves. +A store object can refer to both other store objects and itself. Self-reference may seem pointless, but tracking them is in fact useful. We can best explain why later after more concepts have been established. From fb2ec7e4ec758e17f247218625d23ceba685e8a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:54:51 +0200 Subject: [PATCH 085/191] reword section on references use file Contents instead of Data, as that flows more naturally in the prose. simplify explanation of the idea behind scanning for store paths remove references to unfinished sections. --- doc/manual/src/architecture/store/objects.md | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 872a209c3..d7a27b528 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -11,12 +11,12 @@ A store object is the pair of The Nix store uses a simple file system model. data FileSystemObject - = File Executable Data + = File Executable Contents | Directory (Map FileName FileSystemObject) | SymLink Path Every file system object is one of the following: - - File: an executable flag, and arbitrary data + - File: an executable flag, and arbitrary data for contents - Directory: mapping of names to child file system objects - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. @@ -26,17 +26,16 @@ A bare file or symlink can be a root file system object. ## Reference {#reference} -A store object can refer to both other store objects and itself. +A store object can refer to other store objects or itself. -Self-reference may seem pointless, but tracking them is in fact useful. -We can best explain why later after more concepts have been established. +Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. -References are normally calculated so as to to record the presence of textual references in store object's file systems obejcts. -This process will be described precisely in the section on [building](./building.md), once more concepts are explained, as building is the primary path new store objects with non-trivial references are created. +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness: +Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. -However, scanning for references is not mandatory. -Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, -and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. -Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. +However, having references match store paths in files is not enforced by the data model: +Store objects could have excess or incomplete references with respect to store paths found in their file contents. + +Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. -This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store object, rather than try to recompute them on the fly by scanning their contents. From 5fda995491f1f6b32b46b45c2b47f3e329c997f6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 00:24:23 +0200 Subject: [PATCH 086/191] formalize file system objects convention: describe every data type in prose, and illustrate with a class diagram, and a textual representation of an abstract data type. right now we save ourselves the trouble of doing class diagrams, we can add them later. but they are important. --- doc/manual/src/architecture/store/objects.md | 25 ++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index d7a27b528..7f341c87c 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -6,31 +6,42 @@ A store object is the pair of - a [file system object](#file-system-object) - a set of [references](#reference) to store objects. +We call a store object's outermost file system object the *root*. + +```haskell +data StoreOject = StoreObject { + root :: FileSystemObject +, references :: Set StoreObject +} +``` + ## File system object {#file-system-object} The Nix store uses a simple file system model. - data FileSystemObject - = File Executable Contents - | Directory (Map FileName FileSystemObject) - | SymLink Path - Every file system object is one of the following: - File: an executable flag, and arbitrary data for contents - Directory: mapping of names to child file system objects - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - In particular, symlinks pointing outside of their own root file system object, or to a store object without a matching reference, are allowed, but might not function as intended. +```haskell +data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } +``` A bare file or symlink can be a root file system object. +Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. + ## Reference {#reference} A store object can refer to other store objects or itself. Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness: +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. However, having references match store paths in files is not enforced by the data model: From 07d490fd895b09144684dd8b389ef02f0f1256c4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 00:58:40 +0200 Subject: [PATCH 087/191] stores can also delete objects --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 74d3d7f84..cbb5a4169 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,7 +4,7 @@ A Nix store is a collection of [store objects](objects.md) with associated opera These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -A Nix store allows adding and retrieving store objects. +A Nix store allows adding, retrieving, and deleting store objects. It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. From e90586c0a40cf59df9d39407dcc49d5944a8b853 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 01:15:33 +0200 Subject: [PATCH 088/191] add motivation for references --- doc/manual/src/architecture/store/objects.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 7f341c87c..30683d22d 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -37,9 +37,20 @@ Symlinks pointing outside of their own root, or to a store object without a matc ## Reference {#reference} -A store object can refer to other store objects or itself. +A store object can reference other store objects. -Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. +Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. + +Building, copying and deleting store objects must be done in a way that obeys this property: + +- We can only safely delete unreferenced objects. + +- When copying, to maintain correctness, either the result must be "revealed" atomically to the destination store, or objects must be copied in reference-dependency order. + +- Newly built store objects must only refer to store objects in the closure of the build inputs. + This ensures the purity of the build. + +### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. From b5ca3d12b6f3414302363c8ae362334c020448c7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 02:10:24 +0200 Subject: [PATCH 089/191] reword details on keeping closure property --- doc/manual/src/architecture/store/objects.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 30683d22d..f7587e112 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -43,13 +43,13 @@ Nix stores have the *closure property*: for each store object in the store, all Building, copying and deleting store objects must be done in a way that obeys this property: +- Build results must only refer to store objects in the closure of the build inputs. + +- Store objects being copied must refer to objects already in the destination store. + Recursive copying must either proceed in dependency order or be atomic. + - We can only safely delete unreferenced objects. -- When copying, to maintain correctness, either the result must be "revealed" atomically to the destination store, or objects must be copied in reference-dependency order. - -- Newly built store objects must only refer to store objects in the closure of the build inputs. - This ensures the purity of the build. - ### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. From b01bb65d3023281c601024578a9c19f32e7ad011 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Apr 2022 21:41:04 -0400 Subject: [PATCH 090/191] Fix rel path in doc --- doc/manual/src/architecture/store/building.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/building.md b/doc/manual/src/architecture/store/building.md index f4f2649a3..d43904126 100644 --- a/doc/manual/src/architecture/store/building.md +++ b/doc/manual/src/architecture/store/building.md @@ -2,7 +2,7 @@ ## Scanning for references -Before in the section on [store objects](../objects.md), we talked abstractly about scanning for references. +Before in the section on [store objects](./objects.md), we talked abstractly about scanning for references. Now we can make this concrete. After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. From 3d8f2f5cc1faa4d42138267f21d2996da99c49a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Apr 2022 17:53:10 -0400 Subject: [PATCH 091/191] Fix manual TOC links --- doc/manual/src/SUMMARY.md.in | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c029e30bf..80a228741 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,17 +16,17 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - - [Store](design/store/store.md) - - [Store Objects](design/store/objects.md) - - [Store Paths](design/store/paths.md) - - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Objects](design/store/object-ca.md) - - [Derivations](design/store/drvs/drvs.md) - - [Input-Addressing](design/store/drvs/ia.md) - - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - - [Building](design/store/building.md) - - [Advanced Topic: Store object relocatability](design/store/relocatability.md) - - [Advanced Topic: Related work](design/store/related-work.md) + - [Store](architecture/store/store.md) + - [Store Objects](architecture/store/objects.md) + - [Store Paths](architecture/store/paths.md) + - [Nix Archives](architecture/store/nar.md) + - [Content-Addressing Store Objects](architecture/store/object-ca.md) + - [Derivations](architecture/store/drvs/drvs.md) + - [Input-Addressing](architecture/store/drvs/ia.md) + - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) + - [Building](architecture/store/building.md) + - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) + - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) From 1ba6d8fb1d59f4d40825141f75098dfbfd3b0153 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:53:37 +0200 Subject: [PATCH 092/191] remove incomplete section: building --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/building.md | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 doc/manual/src/architecture/store/building.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 80a228741..596843057 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - - [Building](architecture/store/building.md) - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) diff --git a/doc/manual/src/architecture/store/building.md b/doc/manual/src/architecture/store/building.md deleted file mode 100644 index d43904126..000000000 --- a/doc/manual/src/architecture/store/building.md +++ /dev/null @@ -1,15 +0,0 @@ -# Building - -## Scanning for references - -Before in the section on [store objects](./objects.md), we talked abstractly about scanning for references. -Now we can make this concrete. - -After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. -There is a few steps of this, but let's start with the simple case of one input-addressed output first. - -\[Overview of things that need to happen.] - -For example, if Nix thinks `/nix/store/asdfasdfasdf-foo` and `/nix/store/qwerqwerqwer-bar` are paths the data might plausibly reference, Nix will scan all the contents of all files recursively for the "hash parts" `asdfasdfasdf`` and `qwerqwerqwer`. - -\[Explain why whitelist.] From 96876b1eaeae95e3f2f338c6b3495040fc843c57 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:54:06 +0200 Subject: [PATCH 093/191] remove incomplete section: related work --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/related-work.md | 61 ------------------- 2 files changed, 62 deletions(-) delete mode 100644 doc/manual/src/architecture/store/related-work.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 596843057..ee7f5d7ac 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -25,7 +25,6 @@ - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/related-work.md b/doc/manual/src/architecture/store/related-work.md deleted file mode 100644 index 92b7d480e..000000000 --- a/doc/manual/src/architecture/store/related-work.md +++ /dev/null @@ -1,61 +0,0 @@ -# Advanced Topic: Related Work - -## Bazel - -TODO skylark and layering. - -TODO being monadic, if RFC 92. - -## Build Systems à la Carte - -TODO user-choosen keys vs keys chosen automatically? -Purity in face of dynamic tasks (no conflicts, guaranteed). - -TODO Does Nix constitute a different way to be be monadic? -Purity of keys, as mentioned. -Dynamic tasks/keys vs dynamic dependencies. -(Not sure yet.) - -## Lazy evaluation - -We clearly have thunks that produce thunks, but less clearly functions that produce functions. - -Do we have open terms? - -Do we hve thunks vs expressions distinction? -c.f. John Shutt's modern fexprs, when the syntax can "leak". - -## Comparison with Git file system model - -This is close to Git's model, but with one crucial difference: -Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. - - data GitObject - = Blob ByteString - | Tree (Map FileName (Persission, FSO)) - - data Persission - = Directory -- IFF paired with tree - -- Iff paired with blob, one of: - | RegFile - | ExecutableFile - | Symlink - -So long as the root object is a directory, the representations are isomorphic. -There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. - -However, if the root object is a blob, there is loss of fidelity. -Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. - -Git's model matches Unix tradition, but Nix's model is more natural. - -## Machine models - -TODO -Derivations as store objects via drv files makes Nix a "Von Neumann" archicture. -Can also imagine a "Harvard" archicture where derivations are stored separately? -Can we in general imagine N heaps for N different sorts of objects? - -TODO -Also, leaning on the notion of "builtin builders" more, having multiple different sorts of user-defined builders too. -The builder is a black box as far as the Nix model is concerned. From 7cec9ee3612c7255e6dc4fb94fc238f3d81ced03 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:54:24 +0200 Subject: [PATCH 094/191] remove incomplete section: relocatability --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/relocatability.md | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 doc/manual/src/architecture/store/relocatability.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index ee7f5d7ac..c35505bbe 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/relocatability.md b/doc/manual/src/architecture/store/relocatability.md deleted file mode 100644 index c7f869135..000000000 --- a/doc/manual/src/architecture/store/relocatability.md +++ /dev/null @@ -1,15 +0,0 @@ -## Advanced Topic: Store object relocation - -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissible to relocate a store object to a store with a different mount point. - -Recall from the section on [store paths](./store-paths.md) that concrete store paths look like `/-`. - -~~The two final restrictions of the previous section yield an alternative view of the same information.~~ -Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. - -This corresponds to the observation that a store object with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store object without any references, i.e. thus without a store directory, can exist in any store. - -Lastly, this illustrates the purpose of tracking self references. -Store objects without self-references or other references are relocatable, while store paths with self-references aren't. -This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. -\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] From b18852eb3fc1d62ec307d3b2f102c6bd8574d069 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:55:11 +0200 Subject: [PATCH 095/191] remove incomplete section: content-addressed objects --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/object-ca.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 doc/manual/src/architecture/store/object-ca.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c35505bbe..11a974b47 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,7 +20,6 @@ - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - [Nix Archives](architecture/store/nar.md) - - [Content-Addressing Store Objects](architecture/store/object-ca.md) - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) diff --git a/doc/manual/src/architecture/store/object-ca.md b/doc/manual/src/architecture/store/object-ca.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/object-ca.md +++ /dev/null @@ -1 +0,0 @@ -TODO From 3bd125ebbeae38c9efb5790e13265b7534e66c45 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:55:52 +0200 Subject: [PATCH 096/191] remove incomplete section: nix archives --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/nar.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 doc/manual/src/architecture/store/nar.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 11a974b47..38277aaf2 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,7 +19,6 @@ - [Store](architecture/store/store.md) - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - - [Nix Archives](architecture/store/nar.md) - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) diff --git a/doc/manual/src/architecture/store/nar.md b/doc/manual/src/architecture/store/nar.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/nar.md +++ /dev/null @@ -1 +0,0 @@ -TODO From ad8c2ed7f0566a5fe1b4a3591240eb06804bd958 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:59:03 +0200 Subject: [PATCH 097/191] remove incomplete section: input/content-addressing --- doc/manual/src/SUMMARY.md.in | 2 -- doc/manual/src/architecture/store/drvs/ca.md | 0 doc/manual/src/architecture/store/drvs/ia.md | 0 doc/manual/src/architecture/store/input-addressing.md | 1 - 4 files changed, 3 deletions(-) delete mode 100644 doc/manual/src/architecture/store/drvs/ca.md delete mode 100644 doc/manual/src/architecture/store/drvs/ia.md delete mode 100644 doc/manual/src/architecture/store/input-addressing.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 38277aaf2..257880efe 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,8 +20,6 @@ - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - [Derivations](architecture/store/drvs/drvs.md) - - [Input-Addressing](architecture/store/drvs/ia.md) - - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/ca.md b/doc/manual/src/architecture/store/drvs/ca.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/manual/src/architecture/store/drvs/ia.md b/doc/manual/src/architecture/store/drvs/ia.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/manual/src/architecture/store/input-addressing.md b/doc/manual/src/architecture/store/input-addressing.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/input-addressing.md +++ /dev/null @@ -1 +0,0 @@ -TODO From d3effd014b17bc957b9af8ed35b2f25a3b54e02c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 14:05:46 +0200 Subject: [PATCH 098/191] update architecture diagram --- doc/manual/src/architecture/architecture.md | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 836b40c91..25b6a9616 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -8,20 +8,20 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). ``` - [ commmand line interface ] - | - | evaluates - V - [ configuration language ] - | - | evaluates to - | - reference V build -[ build inputs ] --> [ build plans ] --> [ build results ] - \ | / - \ | persisted to / - \ V / - [ store ] + [ commmand line interface ]--------+ + | | + | evaluates | manages + V | + [ configuration language ] | + | V ++-------------------------------|---------------------------------+ +| store | | +| | evaluates to | +| | | +| references V builds | +| [ build input ] --> [ build plan ] --> [ build result ] | +| | ++-----------------------------------------------------------------+ ``` At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. From 87523f01e3dc27d4cf57354a67346f8473e93cc5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:27:44 +0200 Subject: [PATCH 099/191] match grammatical case to arrow direction --- doc/manual/src/architecture/architecture.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 25b6a9616..a0aa713f3 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -18,8 +18,8 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | store | | | | evaluates to | | | | -| references V builds | -| [ build input ] --> [ build plan ] --> [ build result ] | +| referenced by V builds | +| [ build input ] ----> [ build plan ] ----> [ build result ] | | | +-----------------------------------------------------------------+ ``` From 902638c519fb46b0f0a3f7092ccb7cd2a622b206 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 14:06:51 +0200 Subject: [PATCH 100/191] build step -> build rule "step" sounds atomic, while "rule" hints at internal structure, which in our case consists of mapping inputs to outputs using build instructions. --- doc/manual/src/architecture/architecture.md | 7 ++++--- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index a0aa713f3..81348c570 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build rules*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -37,6 +37,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build plans, to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build rules, data, and references between them. +It can also execute *build instructions*, captured in the build rules, to produce new data. +A series of build rules is a *build plan*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cbb5a4169..b03aa690e 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,10 +2,10 @@ A Nix store is a collection of [store objects](objects.md) with associated operations. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build rules. A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. +It can perform builds, that is, transform build inputs using instructions from the build rules into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. There exist different types of stores, which all follow this model. @@ -28,7 +28,7 @@ generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [B -- | -- | -- | -- | -- data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build rule | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 2a8532fb61ba2b988448e93f5074590e40df7e04 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 May 2022 16:39:26 +0200 Subject: [PATCH 101/191] build rule -> build task closer to "build systems a la carte", satisfies all other complaints --- doc/manual/src/architecture/architecture.md | 10 +++++----- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 81348c570..407e8d0f8 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -19,7 +19,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | | evaluates to | | | | | referenced by V builds | -| [ build input ] ----> [ build plan ] ----> [ build result ] | +| [ build input ] --> [ build task ] --> [ build result ] | | | +-----------------------------------------------------------------+ ``` @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build rules*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build tasks*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -37,7 +37,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build rules, data, and references between them. -It can also execute *build instructions*, captured in the build rules, to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. +It can also execute *build instructions*, captured in the build tasks, to produce new data. -A series of build rules is a *build plan*. +A series of build tasks is a *build plan*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b03aa690e..cc6698a40 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,10 +2,10 @@ A Nix store is a collection of [store objects](objects.md) with associated operations. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build rules. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build tasks. A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build rules into build outputs. +It can perform builds, that is, transform build inputs using instructions from the build tasks into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. There exist different types of stores, which all follow this model. @@ -28,7 +28,7 @@ generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [B -- | -- | -- | -- | -- data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build rule | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build task | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 689b32a543240db992c9e8240401f32bd39e6736 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 12 May 2022 12:30:28 +0200 Subject: [PATCH 102/191] clarify relation of tasks and plans --- doc/manual/src/architecture/architecture.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 407e8d0f8..e8d87f8ae 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -13,21 +13,21 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | evaluates | manages V | [ configuration language ] | - | V -+-------------------------------|---------------------------------+ -| store | | -| | evaluates to | -| | | -| referenced by V builds | -| [ build input ] --> [ build task ] --> [ build result ] | -| | + | | ++-------------------------------|---------------------V-----------+ +| store | evaluates to | +| .............................V............................... | +| : build plan : | +| : referenced by builds : | +| : [ build input ] --> [ build task ] --> [ build result ] : | +| :...........................................................: | +-----------------------------------------------------------------+ ``` At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build tasks*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -40,4 +40,3 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. -A series of build tasks is a *build plan*. From 75ce32405235614235dbe343beb13bb0d17934eb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 May 2022 09:55:24 +0200 Subject: [PATCH 103/191] use singular for class names consistently --- doc/manual/src/SUMMARY.md.in | 6 +++--- doc/manual/src/architecture/store/drvs/drvs.md | 2 +- doc/manual/src/architecture/store/paths.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 257880efe..f356bd07d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,9 +17,9 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Objects](architecture/store/objects.md) - - [Store Paths](architecture/store/paths.md) - - [Derivations](architecture/store/drvs/drvs.md) + - [Store Object](architecture/store/objects.md) + - [Store Path](architecture/store/paths.md) + - [Derivation](architecture/store/drvs/drvs.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md index 766a7b47f..5f374357d 100644 --- a/doc/manual/src/architecture/store/drvs/drvs.md +++ b/doc/manual/src/architecture/store/drvs/drvs.md @@ -1,4 +1,4 @@ -# Derivations +# Derivation Derivations are recipes to create store objects. diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index cf51eb866..402e55e69 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -1,4 +1,4 @@ -# Store Paths +# Store Path A store path is a pair of a 20-byte digest and a name. From 68d26010f6d7341b73675aa30b24b67dcd479a7b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:40:00 +0200 Subject: [PATCH 104/191] architecture overview: add link to Nix expression language reference update summary title to match file contents --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index e8d87f8ae..b818b7aa1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -26,7 +26,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} From ef81276cc161b0ce38cb2681867529c1cbdbd7cb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:40:28 +0200 Subject: [PATCH 105/191] architecture overview: add link to command line reference --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index b818b7aa1..77d5ebc2c 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -24,7 +24,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ +-----------------------------------------------------------------+ ``` -At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. +At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. From 0e63b9bf8872f3556ea8925cea0c639f329d7c6e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 May 2022 19:02:28 +0200 Subject: [PATCH 106/191] add link from overview to store section the overview should only link to the three main concepts presented. the store is now fairly fleshed out. others can follow later. --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 77d5ebc2c..ae977a7e1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,6 +37,6 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. +Underlying everything is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. From 25926c5fc673264288b5d9f7f175178e5b1fdad4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 09:37:28 +0200 Subject: [PATCH 107/191] Nix store does not underly literally everything Co-authored-by: Robert Hensing --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index ae977a7e1..fa992e563 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,6 +37,6 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. +Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. From 2303f84a684e41139a549595edf5d57aada4c685 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 23:46:34 +0200 Subject: [PATCH 108/191] revert to "build plan" in overview diagram this displays correct composition again. build inputs and build results are not part of build plans in terms of data objects. also this is a much less complicated setup. this will be the first impression of architecture, and we want to get it right. --- doc/manual/src/architecture/architecture.md | 29 ++++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index fa992e563..c0fd1b0b5 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -8,19 +8,22 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). ``` - [ commmand line interface ]--------+ - | | - | evaluates | manages - V | - [ configuration language ] | - | | -+-------------------------------|---------------------V-----------+ -| store | evaluates to | -| .............................V............................... | -| : build plan : | -| : referenced by builds : | -| : [ build input ] --> [ build task ] --> [ build result ] : | -| :...........................................................: | ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | | | +| +-------------------------------------------------------------+ | +-----------------------------------------------------------------+ ``` From 4639b36b53fd369024d90e95b0178e4fae0a4203 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 23:56:47 +0200 Subject: [PATCH 109/191] use reference links for URLs --- doc/manual/src/architecture/architecture.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index c0fd1b0b5..4dfe34fd6 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -5,7 +5,7 @@ It should help users understand why Nix behaves as it does, and it should help d ## Overview -Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). +Nix consists of [hierarchical layers][layer-architecture]. ``` +-----------------------------------------------------------------+ @@ -29,7 +29,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} @@ -43,3 +43,5 @@ The command line and Nix language are what users interact with most. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. +[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers +[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From 7c3bca1372aeeb5074fa4038df2984b316e99bb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 21 May 2022 00:16:24 +0200 Subject: [PATCH 110/191] revert to build plans in top-level overview do not introduce build tasks yet, that is the next level of detail. --- doc/manual/src/architecture/architecture.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 4dfe34fd6..6d4b86935 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -30,7 +30,9 @@ Nix consists of [hierarchical layers][layer-architecture]. At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. + +The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -38,10 +40,8 @@ As far as we are concerned here, the inputs and results of a derivation are just In practice this amounts to a set of files in a file system. ::: -The command line and Nix language are what users interact with most. - Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build tasks, to produce new data. +It can also execute build plans to produce new data. [layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers [purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From d5eea66615266e7c341e8d4e982af12d6cb82887 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:52:10 +0200 Subject: [PATCH 111/191] introduce build tasks while it appears a bit much for the overview, this way we set the stage for going directly into data types when describing the store, instead of first having to say what build tasks are and how they relate to build plans. --- doc/manual/src/architecture/architecture.md | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 6d4b86935..8000aecd1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -43,5 +43,37 @@ In practice this amounts to a set of files in a file system. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data. +A build plan is a series of *build tasks*. +Each build task has a special build input which is used as *build instructions*. +The result of a build task can be input to another build task. + +``` ++-----------------------------------------------------------------------------------------+ +| store | +| ................................................. | +| : build plan : | +| : : | +| [ build input ]-----instructions-, : | +| : | : | +| : v : | +| [ build input ]----------->[ build task ]--instructions-, : | +| : | : | +| : | : | +| : v : | +| : [ build task ]----->[ build result ] | +| [ build input ]-----instructions-, ^ : | +| : | | : | +| : v | : | +| [ build input ]----------->[ build task ]---------------' : | +| : ^ : | +| : | : | +| [ build input ]------------------' : | +| : : | +| : : | +| :...............................................: | +| | ++-----------------------------------------------------------------------------------------+ +``` + [layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers [purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From b6b112b366bed363f94231410946ad82122b317c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:28:40 +0200 Subject: [PATCH 112/191] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cc6698a40..d710ca4c0 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -17,21 +17,35 @@ Examples: Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). The store directory defaults to `/nix/store`, but is in principle arbitrary. -## A [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology +## A [Rosetta stone][rosetta-stone] for build system terminology The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. -generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language +generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language -- | -- | -- | -- | -- -data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build task | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) -build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) +data (build input, build result) | store object | [artifact][bazel-artifact] | value | value +build instructions | builder | ([depends on action type][bazel-actions]) | function | function +build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] +build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] build | build | build | application of `Build` | evaluation -persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap +persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap -All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software -Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf) (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) (2018). +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf From e72a7874dc59e7dd9783fb2783cfaf88806bf085 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:29:59 +0200 Subject: [PATCH 113/191] beautify rosetta table while this may eventually introduce ugly diffs, the table will now render readably on the terminal (e.g. for `man nix` or `nix --help`) without further intervention. --- doc/manual/src/architecture/store/store.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d710ca4c0..808e87581 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -24,14 +24,14 @@ Usage of terms is, for historic reasons, not entirely consistent within the Nix The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. -generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language --- | -- | -- | -- | -- -data (build input, build result) | store object | [artifact][bazel-artifact] | value | value -build instructions | builder | ([depends on action type][bazel-actions]) | function | function -build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] -build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] -build | build | build | application of `Build` | evaluation -persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). From 207992a71d5d9c9ee5d09c90f30a9dd35991691d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 13:38:47 +0200 Subject: [PATCH 114/191] introduce store and store objects without file system details this leaves open implementation details, especially about store paths and file system objects, and allows explaining them together were it is more appropriate. also leaves room to carefully introduce the key insight behind Nix: applying results from programming language theory to the operating system paradigm of files and processes. --- doc/manual/src/architecture/store/store.md | 57 ++++++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 808e87581..f421a9ec3 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,12 +1,59 @@ # Store -A Nix store is a collection of [store objects](objects.md) with associated operations. +A Nix store is a collection of *store objects*. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build tasks. +Store objects can hold arbitrary *data* and *references* to one another. +Nix makes no distinction if they are used as build inputs, build results, or build tasks. -A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build tasks into build outputs. -It also keeps track of *references* between data and can therefore garbage-collect unused store objects. +```haskell +data Store = Set StoreObject + +data StoreObject = StoreObject { + data :: Data +, references :: Set Reference +} +``` + +A Nix store can *add*, *retrieve*, and *delete* store objects. + +It can *perform builds*, that is, create new store objects by transforming build inputs, using instructions from the build tasks, into build outputs. + +As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. + +```haskell +add :: Store -> Data -> (Store, Reference) +get :: Store -> Reference -> StoreObject +delete :: Store -> Reference -> Store + +build :: Store -> Reference -> Maybe (Store, Reference) + +collectGarbage :: Store -> Store +``` + +Store objects are [immutable](https://en.m.wikipedia.org/wiki/Immutable_object): once created, they do not change until they are deleted. + +References are [opaque](https://en.m.wikipedia.org/wiki/Opaque_data_type), [unique identifiers](https://en.m.wikipedia.org/wiki/Unique_identifier): +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +An added store object cannot have references, unless it is a build task. + +Building a store object will add appropriate references, according to provided build instructions. +These references can only come from declared build inputs, and are not known by build instructions a priori. + +```haskell +data Data = Data | Task BuildTask + +data BuildTask = BuildTask { + instructions :: Reference +, inputs :: [Reference] +} +``` + +A store object cannot be deleted as long as it is reachable from a reference still in use. +Garbage collection will delete all store objects that cannot be reached from any reference in use. + + There exist different types of stores, which all follow this model. Examples: From b84f2bdfdd21b9aa65cbfac7e0292971d52c40e4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 14:20:56 +0200 Subject: [PATCH 115/191] introduce mapping to Unix files and processes --- doc/manual/src/architecture/architecture.md | 1 - doc/manual/src/architecture/store/store.md | 51 +++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 8000aecd1..3d17074cc 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,7 +37,6 @@ The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. As far as we are concerned here, the inputs and results of a derivation are just data. -In practice this amounts to a set of files in a file system. ::: Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f421a9ec3..d2d64dd53 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -39,7 +39,7 @@ A reference will always point to exactly one store object. An added store object cannot have references, unless it is a build task. Building a store object will add appropriate references, according to provided build instructions. -These references can only come from declared build inputs, and are not known by build instructions a priori. +These references can only come from declared build inputs, and are not known to build instructions a priori. ```haskell data Data = Data | Task BuildTask @@ -55,14 +55,59 @@ Garbage collection will delete all store objects that cannot be reached from any +## Files and Processes + +Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) on the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). + +Nix encodes immutable store objects and opaque identifiers as file system primitives: files, directories, and paths. +That allows processes to resolve references contained in files and thus access the contents of store objects. + +``` ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | ^ | | | +| +---------|----------------------------------------|----------+ | ++-----------|----------------------------------------|------------+ + | | + file system object store path + | | ++-----------|----------------------------------------|------------+ +| operating system +------------+ | | +| '------------ | | <-----------' | +| | file | | +| ,-- | | <-, | +| | +------------+ | | +| execute as | | read, write, execute | +| | +------------+ | | +| '-> | process | --' | +| +------------+ | ++-----------------------------------------------------------------+ +``` + +Store objects are therefore implemented as the pair of + + - a *file system object* for data + - a set of *store paths* for references. + There exist different types of stores, which all follow this model. Examples: - store on the local file system - remote store accessible via SSH - binary cache store accessible via HTTP -Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). -The store directory defaults to `/nix/store`, but is in principle arbitrary. +Every store ultimately has to make store objects accessible to processes through the file system. ## A [Rosetta stone][rosetta-stone] for build system terminology From 4eb11d45928a412643aaf2c2ed40aa7c35cd345c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 00:30:08 +0200 Subject: [PATCH 116/191] fix grammar for clarity --- doc/manual/src/architecture/store/store.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d2d64dd53..99a92bf8e 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -16,7 +16,7 @@ data StoreObject = StoreObject { A Nix store can *add*, *retrieve*, and *delete* store objects. -It can *perform builds*, that is, create new store objects by transforming build inputs, using instructions from the build tasks, into build outputs. +It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. @@ -57,9 +57,9 @@ Garbage collection will delete all store objects that cannot be reached from any ## Files and Processes -Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) on the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). +Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). -Nix encodes immutable store objects and opaque identifiers as file system primitives: files, directories, and paths. +Nix encodes immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. ``` From 4adb6602bdf274f1b0471d8a7ab53e80eaff4854 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:58:25 +0200 Subject: [PATCH 117/191] clarify first sentence on store objects --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 99a92bf8e..18d25ca4d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,8 +2,8 @@ A Nix store is a collection of *store objects*. -Store objects can hold arbitrary *data* and *references* to one another. -Nix makes no distinction if they are used as build inputs, build results, or build tasks. +A store object can hold arbitrary *data* and *references* to other store objects. +Nix makes no distinction if store objects are used as build inputs, build results, or build tasks. ```haskell data Store = Set StoreObject From db8703bcac78ed2d132e39fe9bc13a2cd0fe6efc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 03:05:44 +0200 Subject: [PATCH 118/191] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 18d25ca4d..1640944ad 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -18,7 +18,7 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. -As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. +As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. ```haskell add :: Store -> Data -> (Store, Reference) @@ -30,9 +30,9 @@ build :: Store -> Reference -> Maybe (Store, Reference) collectGarbage :: Store -> Store ``` -Store objects are [immutable](https://en.m.wikipedia.org/wiki/Immutable_object): once created, they do not change until they are deleted. +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -References are [opaque](https://en.m.wikipedia.org/wiki/Opaque_data_type), [unique identifiers](https://en.m.wikipedia.org/wiki/Unique_identifier): +References are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. @@ -55,6 +55,11 @@ Garbage collection will delete all store objects that cannot be reached from any +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + ## Files and Processes Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). From 445f753a820cb4b6076fec1d69d66c649618c4f0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 03:40:17 +0200 Subject: [PATCH 119/191] replace pseudo code by diagrams change prose description to visually resemble the data structure --- doc/manual/src/architecture/store/store.md | 57 ++++++++++++---------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 1640944ad..5a0f4a2df 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,33 +2,49 @@ A Nix store is a collection of *store objects*. -A store object can hold arbitrary *data* and *references* to other store objects. -Nix makes no distinction if store objects are used as build inputs, build results, or build tasks. +A store object can hold -```haskell -data Store = Set StoreObject +- arbitrary *data* +- *references* to other store objects. -data StoreObject = StoreObject { - data :: Data -, references :: Set Reference -} -``` +Nix makes no distinction if store objects are build inputs, build results, or build tasks. A Nix store can *add*, *retrieve*, and *delete* store objects. + [ data ] + | + V + [ store ] ---> add ----> [ store' ] [ reference ] + + + + [ reference ] + | + V + [ store ] ---> get ----> [ store object ] + + + + [ reference ] + | + V + [ store ] --> delete --> [ store' ] + + It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. + + [ reference ] + | + V + [ store ] --> build --(maybe)--> [ store' ] [ reference' ] + + As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. -```haskell -add :: Store -> Data -> (Store, Reference) -get :: Store -> Reference -> StoreObject -delete :: Store -> Reference -> Store -build :: Store -> Reference -> Maybe (Store, Reference) + [ store ] --> collect garbage --> [ store' ] -collectGarbage :: Store -> Store -``` Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. @@ -41,15 +57,6 @@ An added store object cannot have references, unless it is a build task. Building a store object will add appropriate references, according to provided build instructions. These references can only come from declared build inputs, and are not known to build instructions a priori. -```haskell -data Data = Data | Task BuildTask - -data BuildTask = BuildTask { - instructions :: Reference -, inputs :: [Reference] -} -``` - A store object cannot be deleted as long as it is reachable from a reference still in use. Garbage collection will delete all store objects that cannot be reached from any reference in use. From 4341849193cfd9b8e1e494292516a60734dfcc53 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:15:53 +0200 Subject: [PATCH 120/191] move closure property to discussion references --- doc/manual/src/architecture/store/objects.md | 15 ------------- doc/manual/src/architecture/store/store.md | 23 +++++++++++++++----- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index f7587e112..8ab0b9368 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -35,21 +35,6 @@ A bare file or symlink can be a root file system object. Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. -## Reference {#reference} - -A store object can reference other store objects. - -Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. - -Building, copying and deleting store objects must be done in a way that obeys this property: - -- Build results must only refer to store objects in the closure of the build inputs. - -- Store objects being copied must refer to objects already in the destination store. - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete unreferenced objects. - ### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5a0f4a2df..a3d9e2eeb 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -52,15 +52,26 @@ References are [opaque][opaque-data-type], [unique identifiers][unique-identifie The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -An added store object cannot have references, unless it is a build task. +Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. -Building a store object will add appropriate references, according to provided build instructions. -These references can only come from declared build inputs, and are not known to build instructions a priori. +Adding, building, copying and deleting store objects must be done in a way that obeys this property: -A store object cannot be deleted as long as it is reachable from a reference still in use. -Garbage collection will delete all store objects that cannot be reached from any reference in use. +- A newly added store object cannot have references, unless it is a build task. - +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to provided build instructions. + These references can only come from declared build inputs. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + Garbage collection will delete all store objects that cannot be reached from any reference in use. + + [garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) [immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object From 843288a451461610f67798af2598a2034668153a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:16:33 +0200 Subject: [PATCH 121/191] add subsections for objects and references group description of data instead of spreading it across the section. that should help direct skimming. as it turns out, people do not actually read any of that. --- doc/manual/src/architecture/store/store.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index a3d9e2eeb..de12ac1dc 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,6 +2,8 @@ A Nix store is a collection of *store objects*. +## Store Object + A store object can hold - arbitrary *data* @@ -9,6 +11,16 @@ A store object can hold Nix makes no distinction if store objects are build inputs, build results, or build tasks. +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. + +## Reference + +References to store objects are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +## Operations + A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] @@ -46,11 +58,7 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. - -References are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: -The only way to obtain references is by adding or building store objects. -A reference will always point to exactly one store object. +## Closure Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. From e63a768e21d9bff605e51de54f9ec1a8f74650a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:18:07 +0200 Subject: [PATCH 122/191] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index de12ac1dc..710d3265d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -88,11 +88,12 @@ Adding, building, copying and deleting store objects must be done in a way that ## Files and Processes -Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). - -Nix encodes immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. +Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. +[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file +[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor + ``` +-----------------------------------------------------------------+ | Nix | From 7b7e4c6340513efb838aa576d95c5032a90dc50d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:20:21 +0200 Subject: [PATCH 123/191] use singular to match section heading --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 710d3265d..7367b4626 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -15,7 +15,7 @@ Store objects are [immutable][immutable-object]: once created, they do not chang ## Reference -References to store objects are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: +A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. From 3794618ccecf56a468eb9befd351e3cd39595ab5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:11:58 +0200 Subject: [PATCH 124/191] add commas between output values --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 7367b4626..6ac210856 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,7 +26,7 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] | V - [ store ] ---> add ----> [ store' ] [ reference ] + [ store ] ---> add ----> [ store' ], [ reference ] @@ -49,7 +49,7 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build --(maybe)--> [ store' ] [ reference' ] + [ store ] --> build --(maybe)--> [ store' ], [ reference' ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From 80de4a4804bf16fe3266618cb79ad486dc6659ae Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:20:11 +0200 Subject: [PATCH 125/191] operations diagram: store' to the right this also looks more diverse, hopefully easier to distinguish Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 6ac210856..33777dc0d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,14 +26,20 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] | V - [ store ] ---> add ----> [ store' ], [ reference ] + [ store ] ---> add ----> [ store' ] + | + V + [ reference ] [ reference ] | V - [ store ] ---> get ----> [ store object ] + [ store ] ---> get + | + V + [ store object ] @@ -49,7 +55,12 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build --(maybe)--> [ store' ], [ reference' ] + [ store ] --> build + \ + (maybe) --> [ store' ] + | + V + [ reference ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From 195aa28ff73ee21bfe2d398e570bf458f07f6757 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:29:32 +0200 Subject: [PATCH 126/191] references are added according to build task at this level of abstraction we do not really care about build instructions or what they are, and also build instructions including their arguments really amount to the build task. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 33777dc0d..aff41d511 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -79,7 +79,7 @@ Adding, building, copying and deleting store objects must be done in a way that - Build results must only refer to store objects in the closure of the build inputs. - Building a store object will add appropriate references, according to provided build instructions. + Building a store object will add appropriate references, according to the build task. These references can only come from declared build inputs. - Store objects being copied must refer to objects already in the destination store. From 7993ba1f388e509d7617808b4a947ed1bf1533a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:34:24 +0200 Subject: [PATCH 127/191] constrain garbage collection scope garbage collection is now incremental, and may (in theory) never delete all unreferenced objects if it is slow enough. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index aff41d511..5b4c3072d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -88,7 +88,7 @@ Adding, building, copying and deleting store objects must be done in a way that - We can only safely delete store objects which are not reachable from any reference still in use. - Garbage collection will delete all store objects that cannot be reached from any reference in use. + Garbage collection will delete those store objects that cannot be reached from any reference in use. From a90fc62b618f571bc5e6512e66e9c8c9a8eb6e78 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:50:51 +0200 Subject: [PATCH 128/191] make clear that file system is for processes Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5b4c3072d..3d1ad0bad 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -150,7 +150,7 @@ Examples: - remote store accessible via SSH - binary cache store accessible via HTTP -Every store ultimately has to make store objects accessible to processes through the file system. +To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. ## A [Rosetta stone][rosetta-stone] for build system terminology From 19d8a5d83942f13c3f532b4ac21cc91c7158c21a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:45:04 +0200 Subject: [PATCH 129/191] move first mention of file system object before diagram --- doc/manual/src/architecture/store/store.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 3d1ad0bad..a22735e42 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -102,6 +102,11 @@ Adding, building, copying and deleting store objects must be done in a way that Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. +Store objects are therefore implemented as the pair of + + - a *file system object* for data + - a set of *store paths* for references. + [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor @@ -139,11 +144,6 @@ That allows processes to resolve references contained in files and thus access t +-----------------------------------------------------------------+ ``` -Store objects are therefore implemented as the pair of - - - a *file system object* for data - - a set of *store paths* for references. - There exist different types of stores, which all follow this model. Examples: - store on the local file system From 93f721b0d26b6525d8eaa8a1606ebeb4b71a9b6a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:36:29 +0200 Subject: [PATCH 130/191] remove draft on derivations --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/drvs/drvs.md | 59 ------------------- 2 files changed, 60 deletions(-) delete mode 100644 doc/manual/src/architecture/store/drvs/drvs.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f356bd07d..5e2d26bf7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,7 +19,6 @@ - [Store](architecture/store/store.md) - [Store Object](architecture/store/objects.md) - [Store Path](architecture/store/paths.md) - - [Derivation](architecture/store/drvs/drvs.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md deleted file mode 100644 index 5f374357d..000000000 --- a/doc/manual/src/architecture/store/drvs/drvs.md +++ /dev/null @@ -1,59 +0,0 @@ -# Derivation - -Derivations are recipes to create store objects. - -Derivations are the heart of Nix. -Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. -This is where Nix comes in. - -Derivations produce data by running arbitrary commands, like Make or Ninja rules. -Unlike those systems, derivations do not produce arbitrary files, but only specific store objects. -They cannot modify the store in any way, other than creating those store objects. -This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. - -Based on the above, we can conceptually break derivations down into 3 parts: - -1. What command will be run? - -2. What existing store objects are needed as inputs? - -3. What store objects will be produced as outputs? - -## What command will be run? - -The original core of Nix was very simple about this, in the mold of traditional Unix. -Commands consist of 3 parts: - -1. Path to executable - -2. Arguments (Except for `argv[0]`, which is taken from the path in the usual way) - -3. Environment variables. - -## What existing store objects are needed as inputs? - -The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" -It's a good questions! - -## What store objects will be produced as outputs? - -## Extra extensions - -### `__structuredAttrs` - -Historically speaking, most users of Nix made GNU Bash with a script the command run, regardless of what they were doing. -Bash variable are automatically created from env vars, but bash also supports array and string-keyed map variables in addition to string variables. -People also usually create derivations using language which also support these richer data types. -It was thus desired a way to get this data from the language "planning" the derivation to language to bash, the language evaluated at "run time". - -`__structuredAttrs` does this by smuggling inside the core derivation format a map of named richer data. -At run time, this becomes two things: - -1. A JSON file containing that map. -2. A bash script setting those variables. - -The bash command can be passed a script which will "source" that Nix-created bash script, setting those variables with the richer data. -The outer script can then do whatever it likes with those richer variables as input. - -However, since derivations can already contain arbitary input sources, the vast majority of `__structuredAttrs` can be handled by upper layers. -We might consider implementing `__structuredAttrs` in higher layers in the future, and simplifying the store layer. From 84ddfbf8fd9b1e77fefb3c2c11a9cedab2e530c2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:57:19 +0200 Subject: [PATCH 131/191] remove diagonal from operations diagram --- doc/manual/src/architecture/store/store.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index a22735e42..3b99982e4 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -55,12 +55,10 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build - \ - (maybe) --> [ store' ] - | - V - [ reference ] + [ store ] --> build --(maybe)--> [ store' ] + | + V + [ reference ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From f632816cbaefbba9cc27a8e0de6cffd39fa7a8dd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:09:27 +0200 Subject: [PATCH 132/191] add explanation and examples of file system objects --- doc/manual/src/architecture/store/fso.md | 64 ++++++++++++++++++++++ doc/manual/src/architecture/store/store.md | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 doc/manual/src/architecture/store/fso.md diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md new file mode 100644 index 000000000..958b8f62a --- /dev/null +++ b/doc/manual/src/architecture/store/fso.md @@ -0,0 +1,64 @@ +# File System Object + +The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object). + +Every file system object is one of the following: + + - File: an executable flag, and arbitrary data for contents + - Directory: mapping of names to child file system objects + - [Symbolic link][symlink]: may point anywhere. + +We call a store object's outermost file system object the *root*. + + data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } + +Examples: + +- a directory with contents + + /nix/store/-hello-2.10 + ├── bin + │   └── hello + └── share + ├── info + │   └── hello.info + └── man + └── man1 + └── hello.1.gz + +- a directory with relative symlink and other contents + + /nix/store/-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ + +- a directory with absolute symlink + + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. + +A bare file or symlink can be a root file system object. +Examles: + + /nix/store/-hello-2.10.tar.gz + + /nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc + +Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended. +Examples: + +- an arbitrarily symlinked file may change or not exist at all + + /nix/store/-foo + └── foo -> /home/foo + +- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted + + /nix/store/-bar + └── bar -> /nix/store/abc...-foo + +[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 3b99982e4..68bdadc4a 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,7 +2,7 @@ A Nix store is a collection of *store objects*. -## Store Object +## Store Object {#store-object} A store object can hold @@ -102,7 +102,7 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - - a *file system object* for data + - a [file system object](fso.md) for data - a set of *store paths* for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file From fa7ad4593d09d04afe1d215d2da09be02c2e2836 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 11:07:50 +0200 Subject: [PATCH 133/191] explain store directory --- doc/manual/src/SUMMARY.md.in | 6 +- doc/manual/src/architecture/store/objects.md | 48 ------- doc/manual/src/architecture/store/paths.md | 129 +++++++++++-------- doc/manual/src/architecture/store/store.md | 21 +-- 4 files changed, 94 insertions(+), 110 deletions(-) delete mode 100644 doc/manual/src/architecture/store/objects.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5e2d26bf7..997d75444 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,8 +17,10 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Object](architecture/store/objects.md) - - [Store Path](architecture/store/paths.md) + - [Store Path](architecture/store/path.md) + - [Digest](architecture/store/path.md#digest) + - [Input Addressing](architecture/store/path.md#input-addressing) + - [Content Addressing](architecture/store/path.md#content-addressing) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md deleted file mode 100644 index 8ab0b9368..000000000 --- a/doc/manual/src/architecture/store/objects.md +++ /dev/null @@ -1,48 +0,0 @@ -# Store Object - -Nix organizes the data it manages into *store objects*. -A store object is the pair of - - - a [file system object](#file-system-object) - - a set of [references](#reference) to store objects. - -We call a store object's outermost file system object the *root*. - -```haskell -data StoreOject = StoreObject { - root :: FileSystemObject -, references :: Set StoreObject -} -``` - -## File system object {#file-system-object} - -The Nix store uses a simple file system model. - -Every file system object is one of the following: - - File: an executable flag, and arbitrary data for contents - - Directory: mapping of names to child file system objects - - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - -```haskell -data FileSystemObject - = File { isExecutable :: Bool, contents :: Bytes } - | Directory { entries :: Map FileName FileSystemObject } - | SymLink { target :: Path } -``` - -A bare file or symlink can be a root file system object. - -Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. - -### Reference scanning - -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. -Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. - -However, having references match store paths in files is not enforced by the data model: -Store objects could have excess or incomplete references with respect to store paths found in their file contents. - -Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. -Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. - diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index 402e55e69..4867e7fd3 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -1,78 +1,103 @@ # Store Path -A store path is a pair of a 20-byte digest and a name. +Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*. -## String representation +Store paths are pairs of -A store path is rendered as the concatenation of +- a 20-byte [digest](#digest) for identification +- a symbolic name for people to read. - - a store directory - - - a path-separator (`/`) - - - the digest rendered as Base-32 (20 arbitrary bytes becomes 32 ASCII chars) - - - a hyphen (`-`) - - - the name - -Let's take the store path from the very beginning of this manual as an example: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - -This parses like so: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ - store dir digest name - -We then can discard the store dir to recover the conceptual pair that is a store path: +Example: { digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", name: "firefox-33.1", } -### Where did the "store directory" come from? +It is rendered to a file system path as the concatenation of -If you notice, the above references a "store directory", but that is *not* part of the definition of a store path. -We can discard it when parsing, but what about when printing? -We need to get a store directory from *somewhere*. + - [store directory](#store-directory) + - path-separator (`/`) + - [digest](#digest) rendered in [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - hyphen (`-`) + - name -The answer is, the store directory is a property of the store that contains the store path. -The explanation for this is simple enough: a store is notionally mounted as a directory at some location, and the store object's root file system likewise mounted at this path within that directory. +Example: -This does, however, mean the string representation of a store path is not derived just from the store path itself, but is in fact "context dependent". + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 + |--------| |------------------------------| |----------| + store directory digest name -## The digest +## Store Directory {#store-directory} -The calculation of the digest is quite complicated for historical reasons. -The details of the algorithms will be discussed later once more concepts have been introduced. -For now, we just concern ourselves with the *key properties* of those algorithms. +Every [store](./store.md) has a store directory. + +If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path). + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. ::: {.note} -**Historical note** The 20 byte restriction is because originally a digests were SHA-1 hashes. -This is no longer true, but longer hashes and other information are still boiled down to 20 bytes. +The store directory defaults to `/nix/store`, but is in principle arbitrary. ::: -Store paths are either *content-addressed* or *input-addressed*. +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. + +Therefore one can only copy store objects if + +- the source and target stores' directories match + + or + +- the store object in question has no references, that is, contains no store paths. + +To move a store object to a store with a different store directory, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may break internal offsets or content hashes. + +# Digest {#digest} + +In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. + +Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). ::: {.note} -The former is a standard term used elsewhere. -The later is our own creation to evoke a contrast with content addressing. +**Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +This is no longer true, but longer hashes and other information are still truncated to 20 bytes for compatibility. ::: -Content addressing means that the store path digest ultimately derives from referred store object's contents, namely its file system objects and references. -There is more than one *method* of content-addressing, however. -Still, if one does know the content addressing schema that was used, -(or guesses, there isn't that many yet!) -one can recalculate the store path and thus verify the store object. +[digest]: https://en.m.wiktionary.org/wiki/digest#Noun +[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function +[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 -Input addressing means that the store path digest derives from how the store path was produced, namely the "inputs" and plan that it was built from. -Store paths of this sort can *not* be validated from the content of the store object. -Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. -The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. -While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. -That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. -In the parlance of referencing schemes, this means that store paths are not "self-describing". +### Reference scanning + +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. +Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. + +However, having references match store paths in files is not enforced by the data model: +Store objects could have excess or incomplete references with respect to store paths found in their file contents. + +Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. + +## Input Addressing {#input-addressing} + +Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. + +To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes. + +Nix has a custom serialisation format called Nix Archive (NAR) + +Store object references of this sort can *not* be validated from the content of the store object. +Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. + +## Content Addressing {#content-addressing} + +Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. +If one knows content addressing was used, one can recalculate the reference and thus verify the store object. + +Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance. +Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md). + diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 68bdadc4a..21a876f75 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -67,18 +67,19 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -## Closure +## Closure {#closure} -Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. -Adding, building, copying and deleting store objects must be done in a way that obeys this property: +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - A newly added store object cannot have references, unless it is a build task. - Build results must only refer to store objects in the closure of the build inputs. Building a store object will add appropriate references, according to the build task. - These references can only come from declared build inputs. - Store objects being copied must refer to objects already in the destination store. @@ -86,16 +87,15 @@ Adding, building, copying and deleting store objects must be done in a way that - We can only safely delete store objects which are not reachable from any reference still in use. - Garbage collection will delete those store objects that cannot be reached from any reference in use. - +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity [garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) [immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object [opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -## Files and Processes +## Files and Processes {#files-and-processes} Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. @@ -103,11 +103,16 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - a [file system object](fso.md) for data - - a set of *store paths* for references. + - a set of [store paths](paths.md) for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor +The following diagram shows a radical simplification of how Nix interacts with the operating system: +It uses files as build inputs, and build outputs are files again. +On the operating system, files are either "dead" data, or "live" as processes, which in turn operate on files, or can bring them to life. +A build function also amounts to an operating system process (not depicted). + ``` +-----------------------------------------------------------------+ | Nix | From 1681f4e9f3608973e3247e447fd9236eb32d66f3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 13 Jun 2022 17:10:31 +0200 Subject: [PATCH 134/191] better explain reference scanning --- doc/manual/src/architecture/store/paths.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index 4867e7fd3..956049341 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -70,18 +70,21 @@ This is no longer true, but longer hashes and other information are still trunca [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function [sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 - ### Reference scanning -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. -Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. +When a new store object is built, Nix scans its file contents for store paths to construct its set of references. -However, having references match store paths in files is not enforced by the data model: -Store objects could have excess or incomplete references with respect to store paths found in their file contents. +The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data. +Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives. -Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +This way, scanning files captures run time dependencies without the user having to declare them explicitly. Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. +::: {.note} +In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +This depends on the specifics of the software to build and run. +::: + ## Input Addressing {#input-addressing} Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. From 9c544813d7883fa003d25f528f50c46faf06e1a2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:01:31 +0200 Subject: [PATCH 135/191] paths -> path use singular for terminology uniformly --- doc/manual/src/architecture/store/{paths.md => path.md} | 0 doc/manual/src/architecture/store/store.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename doc/manual/src/architecture/store/{paths.md => path.md} (100%) diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/path.md similarity index 100% rename from doc/manual/src/architecture/store/paths.md rename to doc/manual/src/architecture/store/path.md diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 21a876f75..f662ed3e7 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -103,7 +103,7 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - a [file system object](fso.md) for data - - a set of [store paths](paths.md) for references. + - a set of [store paths](path.md) for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor From c10dcccccda331499db75753ab6fed8938c31cca Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:08:13 +0200 Subject: [PATCH 136/191] make example a simple list using JSON notation is unwarranted and not explained. --- doc/manual/src/architecture/store/path.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 956049341..2cedc4234 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -9,10 +9,8 @@ Store paths are pairs of Example: - { - digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", - name: "firefox-33.1", - } +- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +- name: `firefox-33.1` It is rendered to a file system path as the concatenation of From 631ca18ee62502dd2e51a8f3c9e2ab656b8382ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:08:50 +0200 Subject: [PATCH 137/191] reword notes on copying clarify that we are copying between different stores. we have not introduced that notion or why it would be interesting, but for now it should be fine to keep it in context of the store directory. we could move that later to a more detailed explanation of different store types. --- doc/manual/src/architecture/store/path.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 2cedc4234..3f2a4bbff 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -42,7 +42,7 @@ It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. -Therefore one can only copy store objects if +Therefore one can only copy store objects to a different store if - the source and target stores' directories match @@ -50,8 +50,9 @@ Therefore one can only copy store objects if - the store object in question has no references, that is, contains no store paths. -To move a store object to a store with a different store directory, it has to be rebuilt, together with all its dependencies. -It is in general not enough to replace the store directory string in file contents, as this may break internal offsets or content hashes. +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. # Digest {#digest} From 7c656d90ca2c3364fd72543c0aa27115d96ac463 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:25:30 +0200 Subject: [PATCH 138/191] simplify description of diagram try not to be too fancy, it's just for reading the diagram out loud. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f662ed3e7..b3517f9b6 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -110,7 +110,7 @@ Store objects are therefore implemented as the pair of The following diagram shows a radical simplification of how Nix interacts with the operating system: It uses files as build inputs, and build outputs are files again. -On the operating system, files are either "dead" data, or "live" as processes, which in turn operate on files, or can bring them to life. +On the operating system, files can be run as processes, which in turn operate on files. A build function also amounts to an operating system process (not depicted). ``` From ec43977553e605370d160ad0f852c15d0c754e74 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:10:44 +0200 Subject: [PATCH 139/191] store: match chapter introduction to outline the various levels of detail should describe the same things. --- doc/manual/src/architecture/store/store.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b3517f9b6..403a23bbd 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,6 +1,7 @@ # Store -A Nix store is a collection of *store objects*. +A Nix store is a collection of *store objects* with references between them. +It supports operations to manipulate that collection. ## Store Object {#store-object} From 348432f48ffaf084c7ba1397fe63d805891bb68d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:17:28 +0200 Subject: [PATCH 140/191] store: add concept map this should help nativate the chapter by indicating which terms should be known to understand a given concept. --- doc/manual/src/architecture/store/store.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 403a23bbd..ed250ce0c 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -3,6 +3,28 @@ A Nix store is a collection of *store objects* with references between them. It supports operations to manipulate that collection. +The following concept map is a graphical outline of this chapter. +Arrows indicate suggested reading order. + +``` + ,----------------------[ store ] + | | + v v + [ store object ] [ operations ]----------------------, + | | | + v v | + [ files and processes ] [ build ]--------->[ derivation ] | + | | | | | + v v v v | + [ file system object ] [ store path ]--->[ reference scanning ]--->[ closure ] | + | ^ | | + v | v v + [ digest ]--------------' [ garbage collection ] + / \ + v v +[ input addressing ] [ content addressing ] +``` + ## Store Object {#store-object} A store object can hold From d8b2f9f772a7fb33db17f98605f221f5254c51b4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:21:12 +0200 Subject: [PATCH 141/191] make concept map more compact invert arrows to/from derivation: - we need closures to form derivations - we need derivations to perform builds --- doc/manual/src/architecture/store/store.md | 35 ++++++++++++---------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index ed250ce0c..fba2f90fd 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -7,22 +7,25 @@ The following concept map is a graphical outline of this chapter. Arrows indicate suggested reading order. ``` - ,----------------------[ store ] - | | - v v - [ store object ] [ operations ]----------------------, - | | | - v v | - [ files and processes ] [ build ]--------->[ derivation ] | - | | | | | - v v v v | - [ file system object ] [ store path ]--->[ reference scanning ]--->[ closure ] | - | ^ | | - v | v v - [ digest ]--------------' [ garbage collection ] - / \ - v v -[ input addressing ] [ content addressing ] + ,----------[ store ]---------, + | | + | v + | [ operations ] + | / \ + v v v + [ store object ] [ garbage collection ] [ build ] + | ^ ^ | + v | | | + [ files and processes ] | [ derivation ]--' | + / \ | ^ | + v v | | | +[ file system object ] [ store path ] '--[ closure ] | + | ^ \ | | + v | v v | + [ digest ]-----' [ reference scanning ]<------' + / \ + v v +[ input addressing ] [ content addressing ] ``` ## Store Object {#store-object} From 475a332025fd9bc399206f79298d9897dde45380 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:25:19 +0200 Subject: [PATCH 142/191] make concept map even more compact --- doc/manual/src/architecture/store/store.md | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index fba2f90fd..c4eea33fa 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -11,18 +11,18 @@ Arrows indicate suggested reading order. | | | v | [ operations ] - | / \ - v v v - [ store object ] [ garbage collection ] [ build ] - | ^ ^ | - v | | | - [ files and processes ] | [ derivation ]--' | - / \ | ^ | - v v | | | -[ file system object ] [ store path ] '--[ closure ] | - | ^ \ | | - v | v v | - [ digest ]-----' [ reference scanning ]<------' + | | | + v v | + [ store object ] [ garbage collection ] | + | ^ | + v | v + [ files and processes ] [ closure ] [ build ] + / \ | | ^ | + v v | v | | +[ file system object ] [ store path ] | [ derivation ] | + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ] <----' / \ v v [ input addressing ] [ content addressing ] From a28d6878b263270c7db271c414bef2c688bb872d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:27:08 +0200 Subject: [PATCH 143/191] concept map: put closure as it is in the chapter this is not as compact any more, but it more closely resembles the chapter structure, and clearly shows that the closure property is the key idea on which most of Nix operates. --- doc/manual/src/architecture/store/store.md | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index c4eea33fa..0f43d5a62 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -7,22 +7,19 @@ The following concept map is a graphical outline of this chapter. Arrows indicate suggested reading order. ``` - ,----------[ store ]---------, - | | - | v - | [ operations ] - | | | - v v | - [ store object ] [ garbage collection ] | - | ^ | - v | v - [ files and processes ] [ closure ] [ build ] - / \ | | ^ | - v v | v | | -[ file system object ] [ store path ] | [ derivation ] | - | ^ | | | - v | v v | - [ digest ]----' [ reference scanning ] <----' + ,--------------[ store ]----------------, + | | | + v v v + [ store object ] [ closure ]--, [ operations ] + | | | | | | + v | | v v | + [ files and processes ] | | [ garbage collection ] | + / \ | V v + v v | [ derivation ]--->[ building ] +[ file system object ] [ store path ] | | + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ] <-----------' / \ v v [ input addressing ] [ content addressing ] From c345345deeb188830e62e9b2e957cf1d09d1c6b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:33:31 +0200 Subject: [PATCH 144/191] concept map: align hights this has the weird but nice emergent property that terms at the same height are roughly at the same level of abstraction. --- doc/manual/src/architecture/store/store.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 0f43d5a62..cba875092 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -14,12 +14,12 @@ Arrows indicate suggested reading order. | | | | | | v | | v v | [ files and processes ] | | [ garbage collection ] | - / \ | V v - v v | [ derivation ]--->[ building ] -[ file system object ] [ store path ] | | + / \ | | | + v v | v v +[ file system object ] [ store path ] | [ derivation ]--->[ building ] | ^ | | | v | v v | - [ digest ]----' [ reference scanning ] <-----------' + [ digest ]----' [ reference scanning ]<------------' / \ v v [ input addressing ] [ content addressing ] From def80d5777dcd5d4b8a10e54be24d745b5d7562b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:50:24 +0200 Subject: [PATCH 145/191] add subsections to table of contents --- doc/manual/src/SUMMARY.md.in | 8 +++++++- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 997d75444..5581bad3a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,7 +17,13 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Path](architecture/store/path.md) + - [Store Object](architecture/store/store.md#store-object) + - [Reference](architecture/store/store.md#reference) + - [Operations](architecture/store/store.md#operations) + - [Closure](architecture/store/store.md#closure) + - [Files and Processes](architecture/store/store.md#files-and-processes) + - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Store Path](architecture/store/path.md) - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) - [Content Addressing](architecture/store/path.md#content-addressing) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cba875092..e1d983d99 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -36,13 +36,13 @@ Nix makes no distinction if store objects are build inputs, build results, or bu Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -## Reference +## Reference {#reference} A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -## Operations +## Operations {#operations} A Nix store can *add*, *retrieve*, and *delete* store objects. @@ -178,7 +178,7 @@ Examples: To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. -## A [Rosetta stone][rosetta-stone] for build system terminology +## A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. From fe4c0b8d75b9380ab64a752a7f1da680b1163731 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 12 Jul 2022 12:28:09 +0100 Subject: [PATCH 146/191] fix typo Co-authored-by: Thomas --- doc/manual/src/architecture/store/fso.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index 958b8f62a..c51996e41 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -42,7 +42,7 @@ Examples: └── nix_node -> /nix/store/f20...-nodejs-10.24. A bare file or symlink can be a root file system object. -Examles: +Examples: /nix/store/-hello-2.10.tar.gz From de5dea45f7443dd14068e6289e8844816f445c35 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 09:37:16 +0100 Subject: [PATCH 147/191] use correct Nix entity Nix itself does care a lot about what type of store object you have. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index e1d983d99..5ac512621 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -32,7 +32,7 @@ A store object can hold - arbitrary *data* - *references* to other store objects. -Nix makes no distinction if store objects are build inputs, build results, or build tasks. +A Nix store makes no distinction if store objects are build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. From 5a5a95696f4728010486ee01b375b861acc2fddd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 10:42:21 +0100 Subject: [PATCH 148/191] note customized base32 Nix omits E O U T characters for some reason. --- doc/manual/src/architecture/store/path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 3f2a4bbff..9f47b877f 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -16,7 +16,7 @@ It is rendered to a file system path as the concatenation of - [store directory](#store-directory) - path-separator (`/`) - - [digest](#digest) rendered in [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) - hyphen (`-`) - name From bac86231add178e4def3e3fe0bc88ace83d86c21 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 10:59:09 +0100 Subject: [PATCH 149/191] use "build plan" consistently --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 3d17074cc..d9fb2e7e5 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -36,7 +36,7 @@ The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a derivation are just data. +As far as we are concerned here, the inputs and results of a build plan are just data. ::: Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. From 9cabba1fc35c4989b0e280c12a845adf0e300dab Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:17:46 +0100 Subject: [PATCH 150/191] mention hard links Co-authored-by: Thomas --- doc/manual/src/architecture/store/fso.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index c51996e41..2ec365eb6 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -61,4 +61,9 @@ Examples: /nix/store/-bar └── bar -> /nix/store/abc...-foo +Nix file system objects do not support [hard links][hardlink]: +each file system object which is not the root has exactly one parent and one name. +However, as store objects are immutable, an underlying file system can use hard links for optimization. + [symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link +[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link From 29c062537d37fc10997767edd3edb6af41f1fb00 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:42:16 +0100 Subject: [PATCH 151/191] hashes: truncate -> reduce, mention SHA-256 the longer SHA-256 hashes are not truncated, but in fact processed. Co-authored-by: Thomas --- doc/manual/src/architecture/store/path.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 9f47b877f..64bc4a522 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -62,12 +62,13 @@ Store objects are therefore said to be either [input-addressed](#input-addressin ::: {.note} **Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -This is no longer true, but longer hashes and other information are still truncated to 20 bytes for compatibility. +Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. ::: [digest]: https://en.m.wiktionary.org/wiki/digest#Noun [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function [sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 +[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Reference scanning From 0228eb8ae76d048b16caffff9628aec347ff1485 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:53:59 +0100 Subject: [PATCH 152/191] add Java example on manual dependency declaration --- doc/manual/src/architecture/store/path.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 64bc4a522..7d793749b 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -83,6 +83,8 @@ Doing it at build time and persisting references in the store object avoids repe ::: {.note} In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. + +For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. ::: ## Input Addressing {#input-addressing} From db6faf44a9befeaabbbd1bb025b3f5a7d743dd16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 23:23:43 +0200 Subject: [PATCH 153/191] clarify what store objects can be the store of course makes a distinction, but that is not relevant here --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5ac512621..920d483d5 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -32,7 +32,7 @@ A store object can hold - arbitrary *data* - *references* to other store objects. -A Nix store makes no distinction if store objects are build inputs, build results, or build tasks. +Store objects can be build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. From 00a7eaed6759c284444c161c69a3a585579a7d5f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:30:27 +0200 Subject: [PATCH 154/191] add file system object to table of contents --- doc/manual/src/SUMMARY.md.in | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5581bad3a..d589156ec 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -27,6 +27,7 @@ - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) - [Content Addressing](architecture/store/path.md#content-addressing) + - [File System Object](architecture/store/fso.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) From b7309ce624d17a2dbe97f63700d59830c35b074a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:31:24 +0200 Subject: [PATCH 155/191] move architecture to the end --- doc/manual/src/SUMMARY.md.in | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index d589156ec..9c2ae2ad3 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,19 +15,6 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Architecture](architecture/architecture.md) - - [Store](architecture/store/store.md) - - [Store Object](architecture/store/store.md#store-object) - - [Reference](architecture/store/store.md#reference) - - [Operations](architecture/store/store.md#operations) - - [Closure](architecture/store/store.md#closure) - - [Files and Processes](architecture/store/store.md#files-and-processes) - - [Build system terminology](architecture/store/store.md#build-system-terminology) - - [Store Path](architecture/store/path.md) - - [Digest](architecture/store/path.md#digest) - - [Input Addressing](architecture/store/path.md#input-addressing) - - [Content Addressing](architecture/store/path.md#content-addressing) - - [File System Object](architecture/store/fso.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) @@ -79,6 +66,19 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) +- [Architecture](architecture/architecture.md) + - [Store](architecture/store/store.md) + - [Store Object](architecture/store/store.md#store-object) + - [Reference](architecture/store/store.md#reference) + - [Operations](architecture/store/store.md#operations) + - [Closure](architecture/store/store.md#closure) + - [Files and Processes](architecture/store/store.md#files-and-processes) + - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Store Path](architecture/store/path.md) + - [Digest](architecture/store/path.md#digest) + - [Input Addressing](architecture/store/path.md#input-addressing) + - [Content Addressing](architecture/store/path.md#content-addressing) + - [File System Object](architecture/store/fso.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) From 3df1ee2ba573a999aebb87cc9c5cac21df158120 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:32:35 +0200 Subject: [PATCH 156/191] clarify what explicitly declaring certain dependencies means --- doc/manual/src/architecture/store/path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 7d793749b..3b14b2f3c 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -81,7 +81,7 @@ This way, scanning files captures run time dependencies without the user having Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. ::: {.note} -In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. From 3c2de2da01704ad2b43182b3b9080dc541e7efc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 4 Aug 2022 06:23:36 -0500 Subject: [PATCH 157/191] Update doc/manual/src/installation/installing-binary.md Co-authored-by: Valentin Gagarin --- doc/manual/src/installation/installing-binary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index a2f284d5a..d94e78a91 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -149,7 +149,7 @@ and `/etc/zshrc` which you may remove. 4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or + `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic mounting of the Nix Store volume. From 84a26882f86e68df3f2d0f66ff627e0dbd6365f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 4 Aug 2022 06:23:45 -0500 Subject: [PATCH 158/191] Update doc/manual/src/installation/installing-binary.md Co-authored-by: Valentin Gagarin --- doc/manual/src/installation/installing-binary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index d94e78a91..ed0f65177 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -176,7 +176,7 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. - If this command indicates that it couldn't remove the volume, you should + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. Look for a "Nix Store" volume in the output of the following command: From 8cec32e7f5e073946ef7bbf5ec1839e40b203bbc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 13:50:44 +0200 Subject: [PATCH 159/191] fix directory tree renderings --- doc/manual/src/architecture/store/fso.md | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index 2ec365eb6..e0eb69f60 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -19,27 +19,27 @@ Examples: - a directory with contents - /nix/store/-hello-2.10 - ├── bin - │   └── hello - └── share - ├── info - │   └── hello.info - └── man - └── man1 - └── hello.1.gz + /nix/store/-hello-2.10 + ├── bin + │   └── hello + └── share + ├── info + │   └── hello.info + └── man + └── man1 + └── hello.1.gz - a directory with relative symlink and other contents - /nix/store/-go-1.16.9 - ├── bin -> share/go/bin - ├── nix-support/ - └── share/ + /nix/store/-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ - a directory with absolute symlink - /nix/store/d3k...-nodejs - └── nix_node -> /nix/store/f20...-nodejs-10.24. + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. A bare file or symlink can be a root file system object. Examples: @@ -53,13 +53,13 @@ Examples: - an arbitrarily symlinked file may change or not exist at all - /nix/store/-foo - └── foo -> /home/foo + /nix/store/-foo + └── foo -> /home/foo - if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted - /nix/store/-bar - └── bar -> /nix/store/abc...-foo + /nix/store/-bar + └── bar -> /nix/store/abc...-foo Nix file system objects do not support [hard links][hardlink]: each file system object which is not the root has exactly one parent and one name. From cc3a5f4ba2cb6859e0f324a5e74a42668b96aff0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 14:04:02 +0200 Subject: [PATCH 160/191] use correct mdBook syntax for callouts --- doc/manual/src/architecture/architecture.md | 7 +++---- doc/manual/src/architecture/store/path.md | 21 +++++++++------------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index d9fb2e7e5..96371056b 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -34,10 +34,9 @@ It is used to compose expressions which ultimately evaluate to self-contained *b The command line and Nix language are what users interact with most. -::: {.note} -The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a build plan are just data. -::: +> **Note** +> The Nix language itself does not have a notion of *packages* or *configurations*. +> As far as we are concerned here, the inputs and results of a build plan are just data. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data. diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 3b14b2f3c..5a18df67a 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -34,9 +34,8 @@ If the store has a [file system representation](./store.md#files-and-processes), This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. -::: {.note} -The store directory defaults to `/nix/store`, but is in principle arbitrary. -::: +> **Note** +> The store directory defaults to `/nix/store`, but is in principle arbitrary. It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. @@ -60,10 +59,9 @@ In a [store path](#store-path), the [digest][digest] is the output of a [cryptog Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). -::: {.note} -**Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. -::: +> **Historical Note** +> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. [digest]: https://en.m.wiktionary.org/wiki/digest#Noun [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function @@ -80,12 +78,11 @@ Nix uses the [closure](store.md#closure) of build inputs to derive the list of a This way, scanning files captures run time dependencies without the user having to declare them explicitly. Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. -::: {.note} -In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +> **Note** +> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. - -For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. -::: +> +> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. ## Input Addressing {#input-addressing} From b63174227704c2a5fc11e815aaf3b4a6fd6f4576 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 14:31:33 +0200 Subject: [PATCH 161/191] fix page rendering apparently it is not possible to link to page anchors with `mdBook`[1] [1]: https://github.com/rust-lang/mdBook/issues/167 --- doc/manual/src/SUMMARY.md.in | 8 +-- doc/manual/src/architecture/store/path.md | 2 +- doc/manual/src/architecture/store/store.md | 61 ------------------- .../src/architecture/store/store/closure.md | 29 +++++++++ .../architecture/store/store/terminology.md | 32 ++++++++++ 5 files changed, 64 insertions(+), 68 deletions(-) create mode 100644 doc/manual/src/architecture/store/store/closure.md create mode 100644 doc/manual/src/architecture/store/store/terminology.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9c2ae2ad3..a95c385e4 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -68,12 +68,8 @@ - [nix.conf](command-ref/conf-file.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Object](architecture/store/store.md#store-object) - - [Reference](architecture/store/store.md#reference) - - [Operations](architecture/store/store.md#operations) - - [Closure](architecture/store/store.md#closure) - - [Files and Processes](architecture/store/store.md#files-and-processes) - - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Closure](architecture/store/store/closure.md) + - [Build system terminology](architecture/store/build-system-terminology.md) - [Store Path](architecture/store/path.md) - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 5a18df67a..039d8b1c6 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -39,7 +39,7 @@ This means a store path is not just derived from the referenced store object its It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. -Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. +Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries. Therefore one can only copy store objects to a different store if diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 920d483d5..9f580f9da 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -89,35 +89,6 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] - -## Closure {#closure} - -Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. - -The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. - -Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - -- A newly added store object cannot have references, unless it is a build task. - -- Build results must only refer to store objects in the closure of the build inputs. - - Building a store object will add appropriate references, according to the build task. - -- Store objects being copied must refer to objects already in the destination store. - - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete store objects which are not reachable from any reference still in use. - - - -[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity -[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) -[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object -[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type -[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier - ## Files and Processes {#files-and-processes} Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. @@ -178,35 +149,3 @@ Examples: To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. -## A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} - -The Nix store's design is comparable to other build systems. -Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. - -The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. - -| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | -| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | -| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | -| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | -| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | -| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | -| build | build | build | application of `Build` | evaluation | -| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | - -All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). - -[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone -[bazel]: https://bazel.build/start/bazel-intro -[bazel-artifact]: https://bazel.build/reference/glossary#artifact -[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html -[bazel-action]: https://bazel.build/reference/glossary#action -[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph -[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph -[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache -[thunk]: https://en.m.wikipedia.org/wiki/Thunk -[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph -[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming -[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf -[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf -[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf diff --git a/doc/manual/src/architecture/store/store/closure.md b/doc/manual/src/architecture/store/store/closure.md new file mode 100644 index 000000000..065b95ffc --- /dev/null +++ b/doc/manual/src/architecture/store/store/closure.md @@ -0,0 +1,29 @@ +# Closure + +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. + +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: + +- A newly added store object cannot have references, unless it is a build task. + +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to the build task. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + + +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + + diff --git a/doc/manual/src/architecture/store/store/terminology.md b/doc/manual/src/architecture/store/store/terminology.md new file mode 100644 index 000000000..3dd7aa756 --- /dev/null +++ b/doc/manual/src/architecture/store/store/terminology.md @@ -0,0 +1,32 @@ +# A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} + +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. + +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | + +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf From b74a3f51c2ecb200ceb877bf5dda528664ef7e10 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:01:27 -0400 Subject: [PATCH 162/191] Fix gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba8e95191..0c1b89ace 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ perl/Makefile.config /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/expressions/builtins.md +/doc/manual/src/language/builtins.md # /scripts/ /scripts/nix-profile.sh From b430a6743c59aebc08968da3cf01e476edbc36c2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:01:55 -0400 Subject: [PATCH 163/191] Remove sections within from SUMMARY --- doc/manual/src/SUMMARY.md.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 812a3d732..a47d39f31 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -62,11 +62,8 @@ - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - [Closure](architecture/store/store/closure.md) - - [Build system terminology](architecture/store/build-system-terminology.md) + - [Build system terminology](architecture/store/store/build-system-terminology.md) - [Store Path](architecture/store/path.md) - - [Digest](architecture/store/path.md#digest) - - [Input Addressing](architecture/store/path.md#input-addressing) - - [Content Addressing](architecture/store/path.md#content-addressing) - [File System Object](architecture/store/fso.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) From 016d7a8f78bee2181d1d60bcf9a8a59a4f18e91f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:19 -0400 Subject: [PATCH 164/191] Fix rosetta stone file name --- .../store/store/{terminology.md => build-system-terminology.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename doc/manual/src/architecture/store/store/{terminology.md => build-system-terminology.md} (97%) diff --git a/doc/manual/src/architecture/store/store/terminology.md b/doc/manual/src/architecture/store/store/build-system-terminology.md similarity index 97% rename from doc/manual/src/architecture/store/store/terminology.md rename to doc/manual/src/architecture/store/store/build-system-terminology.md index 3dd7aa756..eefbaa630 100644 --- a/doc/manual/src/architecture/store/store/terminology.md +++ b/doc/manual/src/architecture/store/store/build-system-terminology.md @@ -1,4 +1,4 @@ -# A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} +# A [Rosetta stone][rosetta-stone] for build system terminology The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. From 6f6498f59c9263d3d4bdc6d2c3776e6abad2e516 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:38 -0400 Subject: [PATCH 165/191] Remove header fragments which is not needd --- doc/manual/src/architecture/store/path.md | 8 ++++---- doc/manual/src/architecture/store/store.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 039d8b1c6..663f04f46 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -26,7 +26,7 @@ Example: |--------| |------------------------------| |----------| store directory digest name -## Store Directory {#store-directory} +## Store Directory Every [store](./store.md) has a store directory. @@ -53,7 +53,7 @@ One cannot copy a store object to a store with a different store directory. Instead, it has to be rebuilt, together with all its dependencies. It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. -# Digest {#digest} +# Digest In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. @@ -84,7 +84,7 @@ This depends on the specifics of the software to build and run. > > For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. -## Input Addressing {#input-addressing} +## Input Addressing Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. @@ -95,7 +95,7 @@ Nix has a custom serialisation format called Nix Archive (NAR) Store object references of this sort can *not* be validated from the content of the store object. Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. -## Content Addressing {#content-addressing} +## Content Addressing Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. If one knows content addressing was used, one can recalculate the reference and thus verify the store object. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 9f580f9da..08b6701d5 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -25,7 +25,7 @@ Arrows indicate suggested reading order. [ input addressing ] [ content addressing ] ``` -## Store Object {#store-object} +## Store Object A store object can hold @@ -36,13 +36,13 @@ Store objects can be build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -## Reference {#reference} +## Reference A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -## Operations {#operations} +## Operations A Nix store can *add*, *retrieve*, and *delete* store objects. @@ -89,7 +89,7 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -## Files and Processes {#files-and-processes} +## Files and Processes Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. From 39d32ac4c63f4aa3784d114b19c0eca83e306ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:58 -0400 Subject: [PATCH 166/191] Add disclaimer that arch section is WIP and links may rot So we can iterate without worrying so much. --- doc/manual/src/architecture/architecture.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 96371056b..41deb07af 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -1,5 +1,7 @@ # Architecture +*(This chapter is unstable and a work in progress. Incoming links may rot.)* + This chapter describes how Nix works. It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. From 4eb566603a1f6f08eae074d48e5c860e1d5d561b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Aug 2022 11:01:40 +0200 Subject: [PATCH 167/191] Comment out the architecture section This needs more review and probably should be a separate book. --- doc/manual/src/SUMMARY.md.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a47d39f31..8fbb59716 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -59,12 +59,14 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) + - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) From cb6794a0d983eb364601f26fc32ead98ed67bfb4 Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Sun, 7 Aug 2022 10:13:11 -0400 Subject: [PATCH 168/191] Do not spam logs if the owned-homedir check results in a noop --- src/libutil/util.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..e11cb9c60 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -577,6 +577,7 @@ Path getHome() { static Path homeDir = []() { + std::optional unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); if (homeDir) { // Only use $HOME if doesn't exist or is owned by the current user. @@ -588,8 +589,7 @@ Path getHome() homeDir.reset(); } } else if (st.st_uid != geteuid()) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); - homeDir.reset(); + unownedUserHomeDir.swap(homeDir); } } if (!homeDir) { @@ -600,6 +600,9 @@ Path getHome() || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } } return *homeDir; }(); From 96b4339d86a22175da22900acecd6b0fb4d5e12b Mon Sep 17 00:00:00 2001 From: Tom Franklin Date: Mon, 8 Aug 2022 00:42:18 +0100 Subject: [PATCH 169/191] Only fail if selinux is in enforcing --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..e6864eaaf 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -640,7 +640,7 @@ place_channel_configuration() { check_selinux() { if command -v getenforce > /dev/null 2>&1; then - if ! [ "$(getenforce)" = "Disabled" ]; then + if [ "$(getenforce)" = "Enforcing" ]; then failure < Date: Mon, 8 Aug 2022 14:34:22 +0200 Subject: [PATCH 170/191] Remove the explicit `c++fs` linkage on darwin Doesn't seem needed on a recent-enough clang anymore (and even seems to break stuff) --- src/libutil/local.mk | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 13e8d426a..f880c0fc5 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,7 +11,3 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif - -ifdef HOST_DARWIN - libutil_LDFLAGS += -lc++fs -endif From 64c3adbe1ad14e043b772c3e981d511922cb06e5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 8 Aug 2022 15:45:23 -0400 Subject: [PATCH 171/191] install-multi-user: abstract is_root, is_os_linux, is_os_darwin --- scripts/install-multi-user.sh | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..472d25842 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -59,6 +59,30 @@ headless() { fi } +is_root() { + if [ "$EUID" -eq 0 ]; then + return 0 + else + return 1 + fi +} + +is_os_linux() { + if [ "$(uname -s)" = "Linux" ]; then + return 0 + else + return 1 + fi +} + +is_os_darwin() { + if [ "$(uname -s)" = "Darwin" ]; then + return 0 + else + return 1 + fi +} + contact_us() { echo "You can open an issue at https://github.com/nixos/nix/issues" echo "" @@ -423,7 +447,7 @@ EOF fi done - if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then + if is_os_linux && [ ! -e /run/systemd/system ]; then warning < Date: Mon, 8 Aug 2022 16:01:59 -0400 Subject: [PATCH 172/191] Strip whitespace in installing-binary.md --- doc/manual/src/installation/installing-binary.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 9fb9c80c3..18fc8fff5 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -13,7 +13,7 @@ for your platform: - multi-user on macOS > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > + > > - It took some time to support this cleanly. You may see posts, > examples, and tutorials using obsolete workarounds. > - Supporting it cleanly made macOS installs too complex to qualify @@ -75,7 +75,7 @@ should run this under your usual user account, *not* as root. The script will invoke `sudo` as needed. > **Note** -> +> > If you need Nix to use a different group ID or user ID set, you will > have to download the tarball manually and [edit the install > script](#installing-from-a-binary-tarball). @@ -167,7 +167,7 @@ and `/etc/zshrc` which you may remove. removed next. 7. Remove the Nix Store volume: - + ```console sudo diskutil apfs deleteVolume /nix ``` @@ -176,7 +176,7 @@ and `/etc/zshrc` which you may remove. store. > **Note** -> +> > After you complete the steps here, you will still have an empty `/nix` > directory. This is an expected sign of a successful uninstall. The empty > `/nix` directory will disappear the next time you reboot. From 7bb1e913b33499d3ce74929749977774bcc35aed Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 8 Aug 2022 15:46:17 -0400 Subject: [PATCH 173/191] Don't prompt about using sudo if we're already root --- .../src/installation/installing-binary.md | 6 ++--- scripts/install-multi-user.sh | 25 ++++++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 18fc8fff5..a46735196 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -31,8 +31,8 @@ $ sh <(curl -L https://nixos.org/nix/install) --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You should run this under your usual user -account, *not* as root. The script will invoke `sudo` to create `/nix` +is owned by the invoking user. You can run this under your usual user +account or root. The script will invoke `sudo` to create `/nix` if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: @@ -71,7 +71,7 @@ $ sh <(curl -L https://nixos.org/nix/install) --daemon The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. You -should run this under your usual user account, *not* as root. The script +can run this under your usual user account or root. The script will invoke `sudo` as needed. > **Note** diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 472d25842..e7bdc1227 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -337,10 +337,15 @@ __sudo() { _sudo() { local expl="$1" shift - if ! headless; then + if ! headless || is_root; then __sudo "$expl" "$*" >&2 fi - sudo "$@" + + if is_root; then + env "$@" + else + sudo "$@" + fi } @@ -891,17 +896,6 @@ EOF main() { - # TODO: I've moved this out of validate_starting_assumptions so we - # can fail faster in this case. Sourcing install-darwin... now runs - # `touch /` to detect Read-only root, but it could update times on - # pre-Catalina macOS if run as root user. - if is_root; then - failure < Date: Wed, 10 Aug 2022 18:49:29 -0500 Subject: [PATCH 174/191] docfix: bundlers --- src/nix/bundle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 2bb70711f..a18161a3c 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -44,7 +44,7 @@ flake output attributes: * `bundlers..default` -If an attribute *name* is given, `nix run` tries the following flake +If an attribute *name* is given, `nix bundle` tries the following flake output attributes: * `bundlers..` From 3d4489b623deaceef720da7d884a41452f928db6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 15:57:25 +0200 Subject: [PATCH 175/191] Show when we're evaluating a flake --- src/libcmd/installables.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 59162c4df..e097f23b3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -616,6 +616,8 @@ InstallableFlake::InstallableFlake( std::tuple InstallableFlake::toDerivation() { + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + auto attr = getCursor(*state); auto attrPath = attr->getAttrPathStr(); From e62160579f40a0425061c2223e0a303d42736ea2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Aug 2022 14:58:08 +0200 Subject: [PATCH 176/191] nix flake metadata: Don't show "Inputs" if there are no inputs --- src/nix/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e01bc6d10..3967f1102 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -212,7 +212,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + if (!lockedFlake.lockFile.root->inputs.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set> visited; From c3769c68465bae971ab6bb48cfcdea85b61ea83a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 15:56:08 +0200 Subject: [PATCH 177/191] ProgressBar: Delay before showing a new activity Some activities are numerous but usually very short (e.g. copying a source file to the store) which would cause a lot of flickering. So only show activities that have been running for at least 10 ms. --- src/libmain/progress-bar.cc | 43 ++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f4306ab91..5183f212f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -8,6 +8,7 @@ #include #include #include +#include namespace nix { @@ -48,6 +49,7 @@ private: bool visible = true; ActivityId parent; std::optional name; + std::chrono::time_point startTime; }; struct ActivitiesByType @@ -91,10 +93,11 @@ public: state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); + auto nextWakeup = std::chrono::milliseconds::max(); while (state->active) { if (!state->haveUpdate) - state.wait(updateCV); - draw(*state); + state.wait_for(updateCV, nextWakeup); + nextWakeup = draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } }); @@ -118,7 +121,8 @@ public: updateThread.join(); } - bool isVerbose() override { + bool isVerbose() override + { return printBuildLogs; } @@ -159,11 +163,13 @@ public: if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); - state->activities.emplace_back(ActInfo()); + state->activities.emplace_back(ActInfo { + .s = s, + .type = type, + .parent = parent, + .startTime = std::chrono::steady_clock::now() + }); auto i = std::prev(state->activities.end()); - i->s = s; - i->type = type; - i->parent = parent; state->its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i); @@ -327,10 +333,12 @@ public: updateCV.notify_one(); } - void draw(State & state) + std::chrono::milliseconds draw(State & state) { + auto nextWakeup = std::chrono::milliseconds::max(); + state.haveUpdate = false; - if (!state.active) return; + if (!state.active) return nextWakeup; std::string line; @@ -341,12 +349,25 @@ public: line += "]"; } + auto now = std::chrono::steady_clock::now(); + if (!state.activities.empty()) { if (!status.empty()) line += " "; auto i = state.activities.rbegin(); - while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + while (i != state.activities.rend()) { + if (i->visible && (!i->s.empty() || !i->lastLine.empty())) { + /* Don't show activities until some time has + passed, to avoid displaying very short + activities. */ + auto delay = std::chrono::milliseconds(10); + if (i->startTime + delay < now) + break; + else + nextWakeup = std::min(nextWakeup, std::chrono::duration_cast(delay - (now - i->startTime))); + } ++i; + } if (i != state.activities.rend()) { line += i->s; @@ -366,6 +387,8 @@ public: if (width <= 0) width = std::numeric_limits::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + + return nextWakeup; } std::string getStatus(State & state) From 53e7b7e8ac44704199a868c0f6850ac53a0b8ad1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 12:28:02 +0200 Subject: [PATCH 178/191] Remove warnLargeDump() This message was unhelpful (#1184) and probably misleading since memory is O(1) in most cases now. --- src/libstore/remote-store.cc | 2 -- src/libutil/serialise.cc | 20 -------------------- src/libutil/serialise.hh | 4 +--- 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..eaaf9669f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -580,7 +580,6 @@ ref RemoteStore::addCAToStore( try { conn->to.written = 0; - conn->to.warn = true; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); @@ -591,7 +590,6 @@ ref RemoteStore::addCAToStore( dumpString(contents, conn->to); } } - conn->to.warn = false; conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8ff904583..2c3597775 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -48,24 +48,9 @@ FdSink::~FdSink() } -size_t threshold = 256 * 1024 * 1024; - -static void warnLargeDump() -{ - warn("dumping very large path (> 256 MiB); this may run out of memory"); -} - - void FdSink::write(std::string_view data) { written += data.size(); - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } try { writeFull(fd, data); } catch (SysError & e) { @@ -448,11 +433,6 @@ Error readError(Source & source) void StringSink::operator () (std::string_view data) { - static bool warned = false; - if (!warned && s.size() > threshold) { - warnLargeDump(); - warned = true; - } s.append(data); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 13da26c6a..84847835a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -97,19 +97,17 @@ protected: struct FdSink : BufferedSink { int fd; - bool warn = false; size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink && s) + FdSink & operator=(FdSink && s) { flush(); fd = s.fd; s.fd = -1; - warn = s.warn; written = s.written; return *this; } From 81e42e0d3f0345c28f3d19841c89c4b1975c37a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jul 2022 16:41:26 +0200 Subject: [PATCH 179/191] Fix onError --- tests/common.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.sh.in b/tests/common.sh.in index 79da10199..73c2d2309 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -193,7 +193,7 @@ fi onError() { set +x echo "$0: test failed at:" >&2 - for ((i = 1; i < 16; i++)); do + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 done From c80a74b7d52f0cf5fc5c1c7497e059ebf301f789 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 16:59:02 +0200 Subject: [PATCH 180/191] Don't pass --force to 'git add' Fixes #5810. --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d01aaa7a..c1a21e764 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, From 6547dcde2a37179f98cc9a8702bc15a2fd1e6a4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:41:19 +0200 Subject: [PATCH 181/191] Use plain mktemp This fixes the case where $TMPDIR doesn't end in a slash. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a9f3e74dc..7c8e159b5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,7 +349,7 @@ _sudo() { } -readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" } From 823e1017d809219e7e00b24ef4ccb0a8b568449c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:47:01 +0200 Subject: [PATCH 182/191] Ensure that $TMPDIR exists if defined --- scripts/install-multi-user.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7c8e159b5..1431857d5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,6 +349,11 @@ _sudo() { } +# Ensure that $TMPDIR exists if defined. +if [[ -v TMPDIR ]]; then + mkdir -m 0700 -p "$TMPDIR" +fi + readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" From 8188b1d0abc2eba6497b5dc47f7e848cbacb7677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:59:04 +0200 Subject: [PATCH 183/191] json: write null on abnormal placeholder destruction Avoids leaving dangling attributes like { "foo": } in case of exceptions. --- src/libutil/json.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index abe0e6e74..2f9e97ff5 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + if (first) { + assert(std::uncaught_exceptions()); + if (state->stack != 0) + write(nullptr); + } } } From 7535ee345da6c0aea1dd81e7de7725b37d8bf8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:33:46 +0200 Subject: [PATCH 184/191] nix-env: don't output incomplete JSON --- src/nix-env/nix-env.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..fdd66220a 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -940,12 +940,12 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin JSONObject metaObj = pkgObj.object("meta"); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); + metaObj.attr(j, nullptr); } else { + auto placeholder = metaObj.placeholder(j); PathSet context; printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } From 7d934f7880d460cba1ab908fd35b3d43e97a4749 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 19 Aug 2022 11:26:26 +0200 Subject: [PATCH 185/191] don't read outputs into memory for output rewriting RewritingSink can handle being fed input where a reference crosses a chunk boundary. we don't need to load the whole source into memory, and in fact *not* loading the whole source lets nix build FODs that do not fit into memory (eg fetchurl'ing data files larger than system memory). --- src/libstore/build/local-derivation-goal.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6843173a7..18b682e13 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2374,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() if (*scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - rsink2(sink.s); + dumpPath(actualPath, rsink2); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; From 0d2bf7acf994ba331d6f72c746721b354931be76 Mon Sep 17 00:00:00 2001 From: Solene Rapenne Date: Fri, 19 Aug 2022 12:40:22 +0200 Subject: [PATCH 186/191] add a nix.conf option to set a download speed limit --- src/libstore/filetransfer.cc | 3 +++ src/libstore/globals.hh | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8454ad7d2..252403cb5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..ca8fc6d5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -746,6 +746,13 @@ public: /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maxium transfer rate in kilobytes per second you want + nix to use for download. + )"}; + Setting netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( From 0bf52b73f4cb61bc12c95a015a7be45f7174ca01 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 19 Aug 2022 15:03:37 -0500 Subject: [PATCH 187/191] install: only create TMPDIR if missing --- scripts/install-multi-user.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 1431857d5..01dbf0c0e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -348,10 +348,9 @@ _sudo() { fi } - # Ensure that $TMPDIR exists if defined. -if [[ -v TMPDIR ]]; then - mkdir -m 0700 -p "$TMPDIR" +if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then + mkdir -m 0700 -p "${TMPDIR:-}" fi readonly SCRATCH=$(mktemp -d) From 7d800909e94c482a2093bc95a2f3dca565c148b2 Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Sat, 20 Aug 2022 03:48:42 +0200 Subject: [PATCH 188/191] Fix default profile path for root in nix profile documentation --- src/nix/profile.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.md b/src/nix/profile.md index 8dade051d..be3c5ba1a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`/nix/var/nix/profiles/default` if Nix is invoked by the `root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. You can specify another profile location using `--profile` *path*. From caad87e6dbfbc62fafb4a055b45d7f2eb3d11efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Sat, 20 Aug 2022 18:21:36 +0200 Subject: [PATCH 189/191] Better documentation wording Co-authored-by: Anderson Torres --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca8fc6d5f..1ff7d2d0a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -749,7 +749,7 @@ public: Setting downloadSpeed { this, 0, "download-speed", R"( - Specify the maxium transfer rate in kilobytes per second you want + Specify the maximum transfer rate in kilobytes per second you want nix to use for download. )"}; From c21b1a7e67cc28b6e95a563daa786f385bc716b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:14:14 +0200 Subject: [PATCH 190/191] Spelling --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ff7d2d0a..e9d721e59 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -750,7 +750,7 @@ public: this, 0, "download-speed", R"( Specify the maximum transfer rate in kilobytes per second you want - nix to use for download. + Nix to use for downloads. )"}; Setting netrcFile{ From 0d2163c6dcf03463fa91ec6d0d96c928ad907366 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:27:36 +0200 Subject: [PATCH 191/191] nix repl: Stop the progress bar The repl was broken since c3769c68465bae971ab6bb48cfcdea85b61ea83a. In general, the progress bar is incompatible with the repl. --- src/libcmd/repl.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 23df40337..150bd42ac 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -35,6 +35,7 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" +#include "progress-bar.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -252,6 +253,10 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif + /* Stop the progress bar because it interferes with the display of + the repl. */ + stopProgressBar(); + std::string input; while (true) { @@ -1037,9 +1042,10 @@ void runRepl( struct CmdRepl : InstallablesCommand { - CmdRepl(){ + CmdRepl() { evalSettings.pureEval = false; } + void prepare() { if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { @@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand } installables = InstallablesCommand::load(); } + std::vector files; + Strings getDefaultFlakeAttrPaths() override { return {""}; } - virtual bool useDefaultInstallables() override + + bool useDefaultInstallables() override { return file.has_value() or expr.has_value(); }