From e30d1daf26bfd4a4647bb8c55c7643f7308e4f88 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 16 Jun 2021 19:06:41 +0200 Subject: [PATCH 001/371] installer: Jeeze -> Oh no Even if it doesn't offend me or probably most people, this word can be considered mildly blasphemous. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index e1046c19c..b06847ffe 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -329,7 +329,7 @@ finish_fail() { finish_cleanup failure < Date: Fri, 3 Jun 2022 17:01:16 +0200 Subject: [PATCH 002/371] Make nix copy parallel again FILLME --- src/libstore/store-api.cc | 203 ++++++++++++++++++++------------------ src/libstore/store-api.hh | 7 ++ 2 files changed, 115 insertions(+), 95 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8861274a2..008451666 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -258,6 +258,86 @@ StorePath Store::addToStore( return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } +void Store::addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; + + using PathWithInfo = std::pair>; + + std::map infosMap; + StorePathSet storePathsToAdd; + for (auto & thingToAdd : pathsToCopy) { + infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); + storePathsToAdd.insert(thingToAdd.first.path); + } + + auto showProgress = [&]() { + act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); + }; + + ThreadPool pool; + + processGraph(pool, + storePathsToAdd, + + [&](const StorePath & path) { + auto & [info, source] = *infosMap.at(path); + /* auto storePathForDst = info.storePath; */ + /* if (info->ca && info->references.empty()) { */ + /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ + /* if (dstStore.storeDir == srcStore.storeDir) */ + /* assert(storePathForDst == storePath); */ + /* if (storePathForDst != storePath) */ + /* debug("replaced path '%s' to '%s' for substituter '%s'", */ + /* srcStore.printStorePath(storePath), */ + /* dstStore.printStorePath(storePathForDst), */ + /* dstStore.getUri()); */ + /* } */ + /* pathsMap.insert_or_assign(storePath, storePathForDst); */ + + if (isValidPath(info.path)) { + nrDone++; + showProgress(); + return StorePathSet(); + } + + bytesExpected += info.narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info.references; + }, + + [&](const StorePath & path) { + checkInterrupt(); + + auto & [info, source] = *infosMap.at(path); + + if (!isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + addToStore(info, *source, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what()); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} void Store::addMultipleToStore( Source & source, @@ -998,106 +1078,39 @@ std::map copyPaths( Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); - auto sorted = srcStore.topoSortPaths(missing); - std::reverse(sorted.begin(), sorted.end()); + /* auto sorted = srcStore.topoSortPaths(missing); */ + /* std::reverse(sorted.begin(), sorted.end()); */ - auto source = sinkToSource([&](Sink & sink) { - sink << sorted.size(); - for (auto & storePath : sorted) { - auto srcUri = srcStore.getUri(); - auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); - Activity act(*logger, lvlInfo, actCopyPath, - makeCopyPathMessage(srcUri, dstUri, storePathS), - {storePathS, srcUri, dstUri}); - PushActivity pact(act.id); + /* auto source = sinkToSource([&](Sink & sink) { */ + /* sink << sorted.size(); */ + /* for (auto & storePath : sorted) { */ + /* auto srcUri = srcStore.getUri(); */ + /* auto dstUri = dstStore.getUri(); */ + /* auto storePathS = srcStore.printStorePath(storePath); */ + /* Activity act(*logger, lvlInfo, actCopyPath, */ + /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ + /* {storePathS, srcUri, dstUri}); */ + /* PushActivity pact(act.id); */ - auto info = srcStore.queryPathInfo(storePath); - info->write(sink, srcStore, 16); - srcStore.narFromPath(storePath, sink); - } - }); + /* auto info = srcStore.queryPathInfo(storePath); */ + /* info->write(sink, srcStore, 16); */ + /* srcStore.narFromPath(storePath, sink); */ + /* } */ + /* }); */ - dstStore.addMultipleToStore(*source, repair, checkSigs); + std::vector>> pathsToCopy; + + for (auto & missingPath : missing) { + auto info = srcStore.queryPathInfo(missingPath); + auto source = sinkToSource([&](Sink & sink) { + srcStore.narFromPath(missingPath, sink); + }); + pathsToCopy.push_back(std::pair{*info, std::move(source)}); + } + + dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); #if 0 - std::atomic nrDone{0}; - std::atomic nrFailed{0}; - std::atomic bytesExpected{0}; - std::atomic nrRunning{0}; - - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; - - ThreadPool pool; - - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), - - [&](const StorePath & storePath) { - auto info = srcStore.queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (dstStore.isValidPath(storePath)) { - nrDone++; - showProgress(); - return StorePathSet(); - } - - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); - - return info->references; - }, - - [&](const StorePath & storePath) { - checkInterrupt(); - - auto info = srcStore.queryPathInfo(storePath); - - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (!dstStore.isValidPath(storePathForDst)) { - MaintainCount mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); - showProgress(); - return; - } - } - - nrDone++; - showProgress(); - }); #endif return pathsMap; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0c8a4db56..d934979cf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "nar-info.hh" #include "realisation.hh" #include "path.hh" #include "derived-path.hh" @@ -364,6 +365,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs + ); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. From cb0553ecd0122f693653c1ac82beb26d127ce3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 14:03:46 +0200 Subject: [PATCH 003/371] Restore the "low-latency" ssh copying --- src/libstore/remote-store.cc | 17 +++++++++++++ src/libstore/remote-store.hh | 8 ++++++ src/libstore/store-api.cc | 49 ++++++++++++++++-------------------- src/libstore/store-api.hh | 6 ++++- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..ad2e5c18a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -673,6 +673,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } +void RemoteStore::addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto source = sinkToSource([&](Sink & sink) { + sink << pathsToCopy.size(); + for (auto & [pathInfo, pathSource] : pathsToCopy) { + pathInfo.write(sink, *this, 16); + pathSource->drainInto(sink); + } + }); + + addMultipleToStore(*source, repair, checkSigs); +} + void RemoteStore::addMultipleToStore( Source & source, RepairFlag repair, diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8493be6fc..5a599997e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -88,6 +88,14 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + + + StorePath addTextToStore( std::string_view name, std::string_view s, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 008451666..eeec6c6f7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -259,7 +259,7 @@ StorePath Store::addToStore( } void Store::addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) @@ -1072,37 +1072,33 @@ std::map copyPaths( for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + // In the general case, `addMultipleToStore` requires a sorted list of + // store paths to add, so sort them right now + auto sortedMissing = srcStore.topoSortPaths(missing); + std::reverse(sortedMissing.begin(), sortedMissing.end()); + std::map pathsMap; for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + Store::PathsSource pathsToCopy; - /* auto sorted = srcStore.topoSortPaths(missing); */ - /* std::reverse(sorted.begin(), sorted.end()); */ - - /* auto source = sinkToSource([&](Sink & sink) { */ - /* sink << sorted.size(); */ - /* for (auto & storePath : sorted) { */ - /* auto srcUri = srcStore.getUri(); */ - /* auto dstUri = dstStore.getUri(); */ - /* auto storePathS = srcStore.printStorePath(storePath); */ - /* Activity act(*logger, lvlInfo, actCopyPath, */ - /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ - /* {storePathS, srcUri, dstUri}); */ - /* PushActivity pact(act.id); */ - - /* auto info = srcStore.queryPathInfo(storePath); */ - /* info->write(sink, srcStore, 16); */ - /* srcStore.narFromPath(storePath, sink); */ - /* } */ - /* }); */ - - std::vector>> pathsToCopy; - - for (auto & missingPath : missing) { + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); auto source = sinkToSource([&](Sink & sink) { + + // We can reasonably assume that the copy will happen whenever we + // read the path, so log something about that at that point + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(missingPath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + srcStore.narFromPath(missingPath, sink); }); pathsToCopy.push_back(std::pair{*info, std::move(source)}); @@ -1110,9 +1106,6 @@ std::map copyPaths( dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); - #if 0 - #endif - return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d934979cf..c0a61115b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,13 +360,17 @@ public: virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; + // A list of paths infos along with a source providing the content of the + // associated store path + using PathsSource = std::vector>>; + /* Import multiple paths into the store. */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); virtual void addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs From 480c2b6699e6afc6a746ba8a72319f197c3556ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:13:11 +0200 Subject: [PATCH 004/371] Rewrite the CA paths when moving them between store Bring back the possibility to copy CA paths with no reference (like the outputs of FO derivations or stuff imported at eval time) between stores that have a different prefix. --- src/libstore/store-api.cc | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index eeec6c6f7..61a12e84a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -289,18 +289,6 @@ void Store::addMultipleToStore( [&](const StorePath & path) { auto & [info, source] = *infosMap.at(path); - /* auto storePathForDst = info.storePath; */ - /* if (info->ca && info->references.empty()) { */ - /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ - /* if (dstStore.storeDir == srcStore.storeDir) */ - /* assert(storePathForDst == storePath); */ - /* if (storePathForDst != storePath) */ - /* debug("replaced path '%s' to '%s' for substituter '%s'", */ - /* srcStore.printStorePath(storePath), */ - /* dstStore.printStorePath(storePathForDst), */ - /* dstStore.getUri()); */ - /* } */ - /* pathsMap.insert_or_assign(storePath, storePathForDst); */ if (isValidPath(info.path)) { nrDone++; @@ -1085,10 +1073,32 @@ std::map copyPaths( Store::PathsSource pathsToCopy; + auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { + auto storePathForSrc = currentPathInfo.path; + auto storePathForDst = storePathForSrc; + if (currentPathInfo.ca && currentPathInfo.references.empty()) { + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + if (dstStore.storeDir == srcStore.storeDir) + assert(storePathForDst == storePathForSrc); + if (storePathForDst != storePathForSrc) + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePathForSrc), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); + } + return storePathForDst; + }; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); - auto source = sinkToSource([&](Sink & sink) { + auto storePathForDst = computeStorePathForDst(*info); + pathsMap.insert_or_assign(missingPath, storePathForDst); + + ValidPathInfo infoForDst = *info; + infoForDst.path = storePathForDst; + + auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we // read the path, so log something about that at that point auto srcUri = srcStore.getUri(); @@ -1101,7 +1111,7 @@ std::map copyPaths( srcStore.narFromPath(missingPath, sink); }); - pathsToCopy.push_back(std::pair{*info, std::move(source)}); + pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); From 34d90fbe224e7b626366f98ea1bc9e370d9cb534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:25:52 +0200 Subject: [PATCH 005/371] Mention the parallel copy in the release notes --- doc/manual/src/release-notes/rl-next.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 52e3b6240..6c2c4689c 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,3 +2,7 @@ * Nix can now be built with LTO by passing `--enable-lto` to `configure`. LTO is currently only supported when building with GCC. + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. From edfcc8256ee232736e335d6cc315f98f6f40d1f3 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sat, 11 Jun 2022 13:30:51 -0500 Subject: [PATCH 006/371] doc: add install test info to hacking.md --- doc/manual/src/contributing/hacking.md | 64 +++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 59ce5cac7..9a371afa7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -83,7 +83,7 @@ by: $ nix develop ``` -## Testing +## Testing Nix Nix comes with three different flavors of tests: unit, functional and integration. @@ -108,3 +108,65 @@ These tests include everything that needs to interact with external services or Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` + +## Testing the install scripts + +Testing the install scripts has traditionally been tedious, but you can now do this much more easily via the GitHub Actions CI runs (at least for platforms that Github Actions supports). + +If you've already pushed to a fork of Nix on GitHub before, you may have noticed that the CI workflows in your fork list skipped "installer" and "installer_test" jobs. Once your Nix fork is set up correctly, pushing to it will also run these jobs. +- The `installer` job will generate installers for these platforms: x86_64-linux, armv6l-linux, armv7l-linux, x86_64-darwin. While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. +- the `installer_test` job will try to use this installer and run a trivial Nix command on `ubuntu-latest` and `macos-latest`. + +### One-time setup +1. Have a GitHub account with a fork of the Nix repo. +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new cachix auth token and copy the generated value. +4. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN` + - Paste the copied value of the Cachix cache auth token. + +### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer url: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the install_url +5. To generate an install command, plug this install_url and your github username into this template: + + ```console + sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve + ``` + + From fe76b9f4b461c6a25f49a80a2476b7d60537c455 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 16 Jun 2022 09:37:52 -0500 Subject: [PATCH 007/371] doc: fix some darwin uninstall gaps --- doc/manual/src/installation/installing-binary.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 9fb9c80c3..762402948 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -148,7 +148,8 @@ and `/etc/zshrc` which you may remove. This will remove all the build users that no longer serve a purpose. 4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like this, + volume on `/nix`, which looks like + `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic mounting of the Nix Store volume. @@ -175,6 +176,16 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. + If this command indicates that it couldn't remove the volume, your Nix Store + volume may not be mounted. Run the following to double-check: + + ```console + diskutil list + ``` + + If you see a Nix Store volume, re-run the diskutil deleteVolume command, but + replace `/nix` with the volume's `diskXsY` identifier. + > **Note** > > After you complete the steps here, you will still have an empty `/nix` From 649c9d9b4c0b2327b11c44cda7b5738efa245c08 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 16 Jun 2022 09:47:15 -0500 Subject: [PATCH 008/371] doc: acknowledge post-rsync reality Before #5150 the copy-to-store phase of the install was idempotent, but the recursive cp isn't. This is probably baiting a few people into trying corrective installs that will fail. --- doc/manual/src/installation/installing-binary.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 762402948..a2f284d5a 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -176,15 +176,17 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. - If this command indicates that it couldn't remove the volume, your Nix Store - volume may not be mounted. Run the following to double-check: + If this command indicates that it couldn't remove the volume, you should + make sure you don't have an _unmounted_ Nix Store volume. Look for a + "Nix Store" volume in the output of the following command: ```console diskutil list ``` - If you see a Nix Store volume, re-run the diskutil deleteVolume command, but - replace `/nix` with the volume's `diskXsY` identifier. + If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil + deleteVolume command, but replace `/nix` with the store volume's `diskXsY` + identifier. > **Note** > @@ -202,8 +204,7 @@ and `/etc/zshrc` which you may remove. We believe we have ironed out how to cleanly support the read-only root -on modern macOS. New installs will do this automatically, and you can -also re-run a new installer to convert your existing setup. +on modern macOS. New installs will do this automatically. This section previously detailed the situation, options, and trade-offs, but it now only outlines what the installer does. You don't need to know From 32effccb51783a36ce88607a7a404083e84ab3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 11 Jul 2022 15:16:19 +0200 Subject: [PATCH 009/371] Add some tests for the CLI completion --- tests/completions.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 49 insertions(+) create mode 100644 tests/completions.sh diff --git a/tests/completions.sh b/tests/completions.sh new file mode 100644 index 000000000..4d6beccf8 --- /dev/null +++ b/tests/completions.sh @@ -0,0 +1,48 @@ +source common.sh + +cd "$TEST_ROOT" + +mkdir -p dep && pushd dep +cat < flake.nix +{ + outputs = i: { }; +} +EOF +popd +mkdir -p foo && pushd foo +cat < flake.nix +{ + inputs.a.url = "path:$(realpath ../dep)"; + + outputs = i: { + sampleOutput = 1; + }; +} +EOF + +popd + +# Test the completion of a subcommand +[[ $(printf "normal\nbuild\t\n") == $(NIX_GET_COMPLETIONS=1 nix buil) ]] +[[ $(printf "normal\nmetadata\t\n") == $(NIX_GET_COMPLETIONS=2 nix flake metad) ]] + +# Filename completion +[[ $(printf "filenames\n./foo\t\n") == $(NIX_GET_COMPLETIONS=2 nix build ./f) ]] +[[ $(printf "filenames\n") == $(NIX_GET_COMPLETIONS=2 nix build ./nonexistent) ]] + +# Input override completion +[[ $(printf "normal\na\t\n") == $(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '') ]] + +# Cli flag completion +NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" + +# Config option completion +## With `--option` +NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-import-from-derivation" +## As a cli flag – not working atm +# NIX_GET_COMPLETIONS=2 nix build --allow-import-from | grep -- "allow-import-from-derivation" + + +# Attr path completions +[[ $(printf "attrs\n./foo#sampleOutput\t\n") == $(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam) ]] +[[ $(printf "attrs\noutputs\t\n") == $(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp) ]] diff --git a/tests/local.mk b/tests/local.mk index ae15c70f9..e0579f503 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -102,6 +102,7 @@ nix_tests = \ suggestions.sh \ store-ping.sh \ fetchClosure.sh \ + completion.sh \ impure-derivations.sh ifeq ($(HAVE_LIBCPUID), 1) From 260fb837de849a10571f19f81adbd6091a28c815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Jul 2022 09:19:05 +0200 Subject: [PATCH 010/371] Fix the name of the completions test --- tests/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local.mk b/tests/local.mk index e0579f503..82e47729b 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -102,7 +102,7 @@ nix_tests = \ suggestions.sh \ store-ping.sh \ fetchClosure.sh \ - completion.sh \ + completions.sh \ impure-derivations.sh ifeq ($(HAVE_LIBCPUID), 1) From 07e14d3ef09e7852fce3a3850cd50eea032e3753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Jul 2022 09:19:15 +0200 Subject: [PATCH 011/371] Harden the comparisons in the completion test - Don't use `printf` for the expected result, but just use bash's `$' '` litteral strings - Quote the `nix` call result - Invert the order in the comparisons (just because it feels more natural) --- tests/completions.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/completions.sh b/tests/completions.sh index 4d6beccf8..b7bb31aea 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -23,15 +23,15 @@ EOF popd # Test the completion of a subcommand -[[ $(printf "normal\nbuild\t\n") == $(NIX_GET_COMPLETIONS=1 nix buil) ]] -[[ $(printf "normal\nmetadata\t\n") == $(NIX_GET_COMPLETIONS=2 nix flake metad) ]] +[[ "$(NIX_GET_COMPLETIONS=1 nix buil)" == $'normal\nbuild\t' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix flake metad)" == $'normal\nmetadata\t' ]] # Filename completion -[[ $(printf "filenames\n./foo\t\n") == $(NIX_GET_COMPLETIONS=2 nix build ./f) ]] -[[ $(printf "filenames\n") == $(NIX_GET_COMPLETIONS=2 nix build ./nonexistent) ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix build ./f)" == $'filenames\n./foo\t' ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix build ./nonexistent)" == $'filenames' ]] # Input override completion -[[ $(printf "normal\na\t\n") == $(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '') ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" @@ -44,5 +44,5 @@ NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-impo # Attr path completions -[[ $(printf "attrs\n./foo#sampleOutput\t\n") == $(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam) ]] -[[ $(printf "attrs\noutputs\t\n") == $(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp) ]] +[[ "$(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam)" == $'attrs\n./foo#sampleOutput\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp)" == $'attrs\noutputs\t' ]] From 21c443d4fd0dc4e28f4af085aef711d5ce30c5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Jul 2022 09:37:57 +0200 Subject: [PATCH 012/371] Test the tilde expansion for the flake completion Also add a disabled test for when the `--override-input` flag comes *before* the flake ref --- tests/completions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/completions.sh b/tests/completions.sh index b7bb31aea..a510e28b0 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -32,6 +32,10 @@ popd # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] +## With tilde expansion +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +## Out of order – not working atm. Should have been fixed by #6693 but apparently not +# [[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' ./foo)" == $'normal\na\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" From d34a333e2ea6f7c3a8f86b821edd0542661d036a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Jul 2022 10:25:28 +0200 Subject: [PATCH 013/371] =?UTF-8?q?Fix=20the=20=E2=80=9Cout=20of=20order?= =?UTF-8?q?=E2=80=9D=20completion=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--override-input` id snarky because it takes two arguments, so it doesn't play well when completed in the middle of the CLI (since the argument just after gets interpreted as its second argument). So use `--update-input` instead --- tests/completions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/completions.sh b/tests/completions.sh index a510e28b0..75b567146 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -34,8 +34,8 @@ popd [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] -## Out of order – not working atm. Should have been fixed by #6693 but apparently not -# [[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' ./foo)" == $'normal\na\t' ]] +## Out of order +[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" From b052e7e71ddf85921580415039a5a86a98ce042c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Jul 2022 10:29:47 +0200 Subject: [PATCH 014/371] Add some more completion tests - Test another command than `build` - Test with two input flakes --- tests/completions.sh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/completions.sh b/tests/completions.sh index 75b567146..522aa1c86 100644 --- a/tests/completions.sh +++ b/tests/completions.sh @@ -2,25 +2,32 @@ source common.sh cd "$TEST_ROOT" -mkdir -p dep && pushd dep -cat < flake.nix +mkdir -p dep +cat < dep/flake.nix { outputs = i: { }; } EOF -popd -mkdir -p foo && pushd foo -cat < flake.nix +mkdir -p foo +cat < foo/flake.nix { - inputs.a.url = "path:$(realpath ../dep)"; + inputs.a.url = "path:$(realpath dep)"; outputs = i: { sampleOutput = 1; }; } EOF +mkdir -p bar +cat < bar/flake.nix +{ + inputs.b.url = "path:$(realpath dep)"; -popd + outputs = i: { + sampleOutput = 1; + }; +} +EOF # Test the completion of a subcommand [[ "$(NIX_GET_COMPLETIONS=1 nix buil)" == $'normal\nbuild\t' ]] @@ -32,10 +39,14 @@ popd # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +## With multiple input flakes +[[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] ## Out of order [[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" @@ -46,7 +57,6 @@ NIX_GET_COMPLETIONS=3 nix build --option allow-import-from | grep -- "allow-impo ## As a cli flag – not working atm # NIX_GET_COMPLETIONS=2 nix build --allow-import-from | grep -- "allow-import-from-derivation" - # Attr path completions [[ "$(NIX_GET_COMPLETIONS=2 nix eval ./foo\#sam)" == $'attrs\n./foo#sampleOutput\t' ]] [[ "$(NIX_GET_COMPLETIONS=4 nix eval --file ./foo/flake.nix outp)" == $'attrs\noutputs\t' ]] From 99208bb8ccd7387c09907e7b17d0091f2f767b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 14 Jul 2022 17:36:31 -0500 Subject: [PATCH 015/371] curl: patch for netrc regression in Nix --- flake.lock | 6 +++--- flake.nix | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 01e4f506a..a66c9cb1b 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1653988320, - "narHash": "sha256-ZaqFFsSDipZ6KVqriwM34T739+KLYJvNmCWzErjAg7c=", + "lastModified": 1657693803, + "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2fa57ed190fd6c7c746319444f34b5917666e5c1", + "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1ee040441..1b26460e7 100644 --- a/flake.nix +++ b/flake.nix @@ -108,7 +108,7 @@ ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; buildDeps = - [ curl + [ (curl.override { patchNetrcRegression = true; }) bzip2 xz brotli editline openssl sqlite libarchive @@ -363,7 +363,7 @@ buildInputs = [ nix - curl + (curl.override { patchNetrcRegression = true; }) bzip2 xz pkgs.perl From 04386f7d69d9c370eb4367ca41d89ac5990ac02e Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Thu, 14 Jul 2022 23:11:02 -0700 Subject: [PATCH 016/371] nix develop: do not assume that saved vars are set This fixes https://github.com/NixOS/nix/issues/6809 --- src/nix/develop.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6d9ad9942..ba7ba7c25 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -288,8 +288,10 @@ struct Common : InstallableCommand, MixProfile out << "unset shellHook\n"; - for (auto & var : savedVars) + for (auto & var : savedVars) { + out << fmt("%s=${%s:-}\n", var, var); out << fmt("nix_saved_%s=\"$%s\"\n", var, var); + } buildEnvironment.toBash(out, ignoreVars); From 3bcd7a5474f065a6e94a60b3042a0d42ed0184ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 Jul 2022 12:32:29 +0200 Subject: [PATCH 017/371] Disable auto-chroot if $NIX_STATE_DIR is set Issue #6732. --- src/libstore/store-api.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 45c53f23e..91dbd991e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1321,7 +1321,12 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); #if __linux__ - else if (!pathExists(stateDir) && params.empty() && getuid() != 0 && !getEnv("NIX_STORE_DIR").has_value()) { + else if (!pathExists(stateDir) + && params.empty() + && getuid() != 0 + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ From b88fb50e218cd3099cbceace48f7cfdf50a8f11f Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Wed, 6 Jul 2022 11:28:23 -0400 Subject: [PATCH 018/371] fix(libstore): allow Nix to access all Rosetta 2 paths on MacOS Fixes: #5884 --- src/libstore/sandbox-defaults.sb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 56b35c3fe..d9d710559 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -98,7 +98,9 @@ (allow file* (literal "/private/var/select/sh")) -; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. +; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin (and vice versa). (allow file-read* (subpath "/Library/Apple/usr/libexec/oah") - (subpath "/System/Library/Apple/usr/libexec/oah")) + (subpath "/System/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") + (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) From 8ea3a911aa81d41efdff231f4b42b11d8861a000 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 14:39:22 -0700 Subject: [PATCH 019/371] local-derivation-goal.cc: improve error messages when sandboxing fails The failure modes for nix's sandboxing setup are pretty complicated. When nix is unable to set up the sandbox, let's provide more detail about what went wrong. Specifically: * Make sure the error message includes the word "sandbox" so the user knows that the failure was related to sandboxing. * If `--option sandbox-fallback false` was provided, and removing it would have allowed further attempts to make progress, let the user know. --- src/libstore/build/local-derivation-goal.cc | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d1ec91ed5..86a59e427 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -850,13 +850,23 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - + if (child == -1) + switch(errno) { + case EPERM: + case EINVAL: { + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); + /* Mention sandbox-fallback in the error message so the user + knows that having it disabled contributed to the + unrecoverability of this failure */ + throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + } + default: + throw SysError("creating sandboxed builder process using clone()"); + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); From 90830b1074cd09b58adde859fb1741a33390412f Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:28:13 -0700 Subject: [PATCH 020/371] local-derivation-goal.cc: warn if failing due to max_user_namespaces==0 This commit uses `warn()` to notify the user if sandbox setup fails with errno==EPERM and /proc/sys/user/max_user_namespaces is missing or zero, since that is at least part of the reason why sandbox setup failed. Note that `echo -n 0 > /proc/sys/user/max_user_namespaces` or equivalent at boot time has been the recommended mitigation for several Linux LPE vulnerabilities over the past few years. Many users have applied this mitigation and then forgotten that they have done so. --- src/libstore/build/local-derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 86a59e427..674b2eaa3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -859,6 +859,8 @@ void LocalDerivationGoal::startBuilder() to true (the default). */ if (settings.sandboxFallback) _exit(1); + if (!userNamespacesEnabled && errno==EPERM) + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 8d35f387dcfa61c59f898de88ef45f3f97817267 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:37:27 -0700 Subject: [PATCH 021/371] local-derivation-goal.cc: warn if failing and /proc/self/ns/user missing This commit causes nix to `warn()` if sandbox setup has failed and `/proc/self/ns/user` does not exist. This is usually a sign that the kernel was compiled without `CONFIG_USER_NS=y`, which is required for sandboxing. --- src/libstore/build/local-derivation-goal.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 674b2eaa3..3aa85e264 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,6 +861,9 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + Path procSelfNsUser = "/proc/self/ns/user"; + if (!pathExists(procSelfNsUser)) + warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 6fc56318bf32f715de8634c199c0fb812f813a8c Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:23:27 -0700 Subject: [PATCH 022/371] local-derivation-goal.cc: add comment re: CLONE_NEWUSER local-derivation-goal.cc contains a comment stating that "Some distros patch Linux to not allow unprivileged user namespaces." Let's give a pointer to a common version of this patch for those who want more details about this failure mode. --- src/libstore/build/local-derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3aa85e264..1c7618045 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -845,6 +845,7 @@ void LocalDerivationGoal::startBuilder() /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. + * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 */ usingUserNamespace = false; flags &= ~CLONE_NEWUSER; From c8c6203c2c6912a52c5f08881c453a04e7fc3f58 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:27:22 -0700 Subject: [PATCH 023/371] local-derivation-goal.cc: detect unprivileged_userns_clone failure mode The workaround for "Some distros patch Linux" mentioned in local-derivation-goal.cc will not help in the `--option sandbox-fallback false` case. To provide the user more helpful guidance on how to get the sandbox working, let's check to see if the `/proc` node created by the aforementioned patch is present and configured in a way that will cause us problems. If so, give the user a suggestion for how to troubleshoot the problem. --- src/libstore/build/local-derivation-goal.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 1c7618045..047c5c8ea 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -862,6 +862,13 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + if (userNamespacesEnabled) { + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + } + } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); From 5f51539f88227285866843f1383fd47d80fd5918 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:30:52 -0700 Subject: [PATCH 024/371] change warn() to notice() --- src/libstore/build/local-derivation-goal.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 047c5c8ea..595149f0a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,17 +861,17 @@ void LocalDerivationGoal::startBuilder() if (settings.sandboxFallback) _exit(1); if (!userNamespacesEnabled && errno==EPERM) - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); } } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) - warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From 99fcc91f67ece5a9646065665395f496d6a0cb84 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:33:12 -0700 Subject: [PATCH 025/371] as requested by @thufschmitt https://github.com/NixOS/nix/pull/6814#discussion_r924275777 --- src/libstore/build/local-derivation-goal.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 595149f0a..43df41e34 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -855,11 +855,6 @@ void LocalDerivationGoal::startBuilder() switch(errno) { case EPERM: case EINVAL: { - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -872,6 +867,11 @@ void LocalDerivationGoal::startBuilder() Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ From a9e75eca00f7efe361545c6e77ecb65244230dc3 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:33 -0700 Subject: [PATCH 026/371] error.hh: add additional constructor with explicit errno argument --- src/libutil/error.hh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a53e9802e..3d1479c54 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -204,13 +204,19 @@ public: int errNo; template - SysError(const Args & ... args) + SysError(int errNo_, const Args & ... args) : Error("") { - errNo = errno; + errNo = errNo_; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } + + template + SysError(const Args & ... args) + : SysError(errno, args ...) + { + } }; } From 36e1383b6b32abd414c8402dd66f7ef78f93f4e1 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:52 -0700 Subject: [PATCH 027/371] local-derivation-goal.cc: save global errno to the stack before performing tests which might clobber it --- src/libstore/build/local-derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 43df41e34..79a241ae0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -851,10 +851,11 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - if (child == -1) + if (child == -1) { switch(errno) { case EPERM: case EINVAL: { + int errno_ = errno; if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -875,11 +876,12 @@ void LocalDerivationGoal::startBuilder() /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ - throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); } default: throw SysError("creating sandboxed builder process using clone()"); } + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); From 56f6f3725f4fbeeb7900ae95bd71d559695b3dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 19 Jul 2022 19:46:00 +0200 Subject: [PATCH 028/371] Don't ultimately trust the signed paths Like the old implem did (and like you'd want it to be anyways) --- src/libstore/store-api.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 61a12e84a..06d03bff2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -305,7 +305,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info, source] = *infosMap.at(path); + auto & [info_, source] = *infosMap.at(path); + auto info = info_; + info.ultimate = false; if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); From 1af5d798a4d10a07e995d420a759f2fe752b583a Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Fri, 15 Jul 2022 00:14:29 -0400 Subject: [PATCH 029/371] libstore/globals.cc: Automatically set cores based on cgroup CPU limit By default, Nix sets the "cores" setting to the number of CPUs which are physically present on the machine. If cgroups are used to limit the CPU and memory consumption of a large Nix build, the OOM killer may be invoked. For example, consider a GitLab CI pipeline which builds a large software package. The GitLab runner spawns a container whose CPU is limited to 4 cores and whose memory is limited to 16 GiB. If the underlying machine has 64 cores, Nix will invoke the build with -j64. In many cases, that level of parallelism will invoke the OOM killer and the build will completely fail. This change sets the default value of "cores" to be ceil(cpu_quota / cpu_period), with a fallback to std::thread::hardware_concurrency() if cgroups v2 is not detected. --- src/libstore/globals.cc | 50 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0f2ca4b15..48df839fa 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,6 +11,11 @@ #include #include +#if __linux__ +#include +#include +#endif + #include @@ -114,7 +119,50 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - return std::max(1U, std::thread::hardware_concurrency()); + unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + + #if __linux__ + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return concurrency; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + concurrency = std::ceil(std::stoi(quota) / std::stof(period)); + } + } + #endif + + return concurrency; } StringSet Settings::getDefaultSystemFeatures() From 722de8ddcc875c7e8e9a228f9d88454bae31fd40 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 19 Jul 2022 02:09:46 -0400 Subject: [PATCH 030/371] libstore/globals.cc: Move cgroup detection to libutil --- src/libstore/globals.cc | 54 +++++------------------------------------ src/libutil/util.cc | 51 ++++++++++++++++++++++++++++++++++++++ src/libutil/util.hh | 3 +++ 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 48df839fa..d724897bb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,11 +11,6 @@ #include #include -#if __linux__ -#include -#include -#endif - #include @@ -119,50 +114,13 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int maxCPU = getMaxCPU(); - #if __linux__ - FILE *fp = fopen("/proc/mounts", "r"); - if (!fp) - return concurrency; - - Strings cgPathParts; - - struct mntent *ent; - while ((ent = getmntent(fp))) { - std::string mountType, mountPath; - - mountType = ent->mnt_type; - mountPath = ent->mnt_dir; - - if (mountType == "cgroup2") { - cgPathParts.push_back(mountPath); - break; - } - } - - fclose(fp); - - if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { - std::string currentCgroup = readFile("/proc/self/cgroup"); - Strings cgValues = tokenizeString(currentCgroup, ":"); - cgPathParts.push_back(trim(cgValues.back(), "\n")); - cgPathParts.push_back("cpu.max"); - std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); - - if (pathExists(fullCgPath)) { - std::string cpuMax = readFile(fullCgPath); - std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); - std::string quota = cpuMaxParts[0]; - std::string period = trim(cpuMaxParts[1], "\n"); - - if (quota != "max") - concurrency = std::ceil(std::stoi(quota) / std::stof(period)); - } - } - #endif - - return concurrency; + if (maxCPU > 0) + return maxCPU; + else + return concurrency; } StringSet Settings::getDefaultSystemFeatures() diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 28df30fef..be6fe091f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -35,6 +35,9 @@ #ifdef __linux__ #include #include + +#include +#include #endif @@ -788,7 +791,55 @@ void drainFD(int fd, Sink & sink, bool block) } } +////////////////////////////////////////////////////////////////////// +unsigned int getMaxCPU() +{ + #if __linux__ + try { + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return 0; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } + } + } catch (Error &) { ignoreException(); } + #endif + + return 0; +} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d3ed15b0b..29227ecc6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -182,6 +182,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); +/* If cgroups are active, attempt to calculate the number of CPUs available. + If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +unsigned int getMaxCPU(); /* Automatic cleanup of resources. */ From 64404220f54a36d3457433580ab8d78cf016572d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Wed, 20 Jul 2022 14:53:03 +0200 Subject: [PATCH 031/371] nix shell: document how to invoke multiple commands from the command line --- src/nix/shell.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/shell.md b/src/nix/shell.md index 90b81fb2f..161fdeb8d 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -23,6 +23,12 @@ R""( Hi everybody! ``` +* Run multiple commands in a shell environment: + + ```console + # nix shell nixpkgs#gnumake -c /bin/sh -c "cd src && make" + ``` + * Run GNU Hello in a chroot store: ```console From 92bae33ca5db60e729ce07156ebf1c06cf865cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Thu, 21 Jul 2022 14:25:07 +0200 Subject: [PATCH 032/371] nix shell: example shouldn't use an absolute path for the shell Co-authored-by: Eelco Dolstra --- src/nix/shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index 161fdeb8d..9fa1031f5 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -26,7 +26,7 @@ R""( * Run multiple commands in a shell environment: ```console - # nix shell nixpkgs#gnumake -c /bin/sh -c "cd src && make" + # nix shell nixpkgs#gnumake -c sh -c "cd src && make" ``` * Run GNU Hello in a chroot store: From 228028fc1aaad20a217387fbe9d4aa2d8698a048 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Thu, 28 Jul 2022 03:36:39 -0400 Subject: [PATCH 033/371] docker.nix: Allow Nix configuration to be customized --- docker.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index ddf6feff5..8e6aa227f 100644 --- a/docker.nix +++ b/docker.nix @@ -6,6 +6,7 @@ , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] , maxLayers ? 100 +, nixConf ? {} }: let defaultPkgs = with pkgs; [ @@ -123,12 +124,17 @@ let (lib.attrValues (lib.mapAttrs groupToGroup groups)) ); - nixConf = { + defaultNixConf = { sandbox = "false"; build-users-group = "nixbld"; - trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; }; - nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: + let + vStr = if builtins.isList v then lib.concatStringsSep " " v else v; + in + "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n"; baseSystem = let From be4654c344f0745d4c2eefb33a878bd1f23f6b40 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 15:55:41 +0200 Subject: [PATCH 034/371] manual: fix section title in table of contents --- doc/manual/src/SUMMARY.md.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9728728aa..5df4e2d75 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,7 +33,7 @@ - [Arguments and Variables](expressions/arguments-variables.md) - [Building and Testing](expressions/simple-building-testing.md) - [Generic Builder Syntax](expressions/generic-builder.md) - - [Writing Nix Expressions](expressions/expression-language.md) + - [Nix Expression Language](expressions/expression-language.md) - [Values](expressions/language-values.md) - [Language Constructs](expressions/language-constructs.md) - [Operators](expressions/language-operators.md) From 85cdaebcd69c4f6abd15b77888f0093be9c5ada4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 16:10:22 +0200 Subject: [PATCH 035/371] manual: set -> attribute set reword description to have shorter sentences. --- doc/manual/src/expressions/language-values.md | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 75ae9f2eb..abbe1fd35 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -172,25 +172,27 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. -## Sets +## Attribute Sets -Sets are really the core of the language, since ultimately the Nix -language is all about creating derivations, which are really just sets -of attributes to be passed to build scripts. +Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). -Sets are just a list of name/value pairs (called *attributes*) enclosed -in curly brackets, where each value is an arbitrary expression -terminated by a semicolon. For example: +Names and values are separated by an equal sign (`=`). +Each value is an arbitrary expression terminated by a semicolon (`;`). + +Attributes can appear in any order. +An attribute name may only occur once. + +Example: ```nix -{ x = 123; +{ + x = 123; text = "Hello"; y = f { bla = 456; }; } ``` -This defines a set with attributes named `x`, `text`, `y`. The order of -the attributes is irrelevant. An attribute name may only occur once. +This defines a set with attributes named `x`, `text`, `y`. Attributes can be selected from a set using the `.` operator. For instance, From 3063e5b94c1cc1f64cea6af792513c1a04e12155 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:09:53 +0200 Subject: [PATCH 036/371] manual: use subheadings for primitive types this gives us HTML anchors for each of them --- doc/manual/src/expressions/language-values.md | 288 +++++++++--------- 1 file changed, 149 insertions(+), 139 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index abbe1fd35..fa5743222 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -4,151 +4,161 @@ Nix has the following basic data types: - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the result of an expression into a string by - enclosing it in `${...}`, a feature known as *antiquotation*. The - enclosed expression must evaluate to something that can be coerced - into a string (meaning that it must be a string, a path, or a - derivation). For instance, rather than writing - - ```nix - "--with-freetype2-library=" + freetype + "/lib" - ``` - - (where `freetype` is a derivation), you can instead write the more - natural - - ```nix - "--with-freetype2-library=${freetype}/lib" - ``` - - The latter is automatically translated to the former. A more - complicated example (from the Nix expression for - [Qt](http://www.trolltech.com/products/qt)): - - ```nix - configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} - "; - ``` - - Note that Nix expressions and strings can be arbitrarily nested; in - this case the outer string contains various antiquotations that - themselves contain strings (e.g., `"-thread"`), some of which in - turn contain expressions (e.g., `${mesa}`). - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix +### Strings + +*Strings* can be written in three ways. + +The most common way is to enclose the string between double quotes, +e.g., `"foo bar"`. Strings can span multiple lines. The special +characters `"` and `\` and the character sequence `${` must be +escaped by prefixing them with a backslash (`\`). Newlines, carriage +returns and tabs can be written as `\n`, `\r` and `\t`, +respectively. + +You can include the result of an expression into a string by +enclosing it in `${...}`, a feature known as *antiquotation*. The +enclosed expression must evaluate to something that can be coerced +into a string (meaning that it must be a string, a path, or a +derivation). For instance, rather than writing + +```nix +"--with-freetype2-library=" + freetype + "/lib" +``` + +(where `freetype` is a derivation), you can instead write the more +natural + +```nix +"--with-freetype2-library=${freetype}/lib" +``` + +The latter is automatically translated to the former. A more +complicated example (from the Nix expression for +[Qt](http://www.trolltech.com/products/qt)): + +```nix +configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} +"; +``` + +Note that Nix expressions and strings can be arbitrarily nested; in +this case the outer string contains various antiquotations that +themselves contain strings (e.g., `"-thread"`), some of which in +turn contain expressions (e.g., `${mesa}`). + +The second way to write string literals is as an *indented string*, +which is enclosed between pairs of *double single-quotes*, like so: + +```nix +'' + This is the first line. + This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +Note that the whitespace and newline following the opening `''` is +ignored if there is no non-whitespace text on the initial line. + +Antiquotation (`${expr}`) is supported in indented strings. + +Since `${` and `''` have special meaning in indented strings, you +need a way to quote them. `$` can be escaped by prefixing it with +`''` (that is, two single quotes), i.e., `''$`. `''` can be escaped +by prefixing it with `'`, i.e., `'''`. `$` removes any special +meaning from the following `$`. Linefeed, carriage-return and tab +characters can be written as `''\n`, `''\r`, `''\t`, and `''\` +escapes any other character. + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { + ... + postInstall = '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Antiquotation (`${expr}`) is supported in indented strings. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... +} +``` - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - Numbers are type-compatible: pure integer operations will always - return integers, whereas any operation involving at least one - floating point number will have a floating point number as a result. +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. - - *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at - least one slash to be recognised as such. For instance, `builder.sh` - is not a path: it's parsed as an expression that selects the - attribute `sh` from the variable `builder`. If the file name is - relative, i.e., if it does not begin with a slash, it is made - absolute at parse time relative to the directory of the Nix - expression that contained it. For instance, if a Nix expression in - `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path - is `/foo/xyzzy/fnord.nix`. - - If the first component of a path is a `~`, it is interpreted as if - the rest of the path were relative to the user's home directory. - e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user - whose home directory is `/home/edolstra`. - - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. +### Numbers - Antiquotation is supported in any paths except those in angle brackets. - `./${foo}-${bar}.nix` is a more convenient way of writing - `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At - least one slash must appear *before* any antiquotations for this to be - recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division - operation. `./a.${foo}/b.${bar}` is a path. +Numbers, which can be *integers* (like `123`) or *floating point* +(like `123.43` or `.27e13`). - - *Booleans* with values `true` and `false`. +Numbers are type-compatible: pure integer operations will always +return integers, whereas any operation involving at least one +floating point number will have a floating point number as a result. - - The null value, denoted as `null`. +### Paths + +*Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at +least one slash to be recognised as such. For instance, `builder.sh` +is not a path: it's parsed as an expression that selects the +attribute `sh` from the variable `builder`. If the file name is +relative, i.e., if it does not begin with a slash, it is made +absolute at parse time relative to the directory of the Nix +expression that contained it. For instance, if a Nix expression in +`/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path +is `/foo/xyzzy/fnord.nix`. + +If the first component of a path is a `~`, it is interpreted as if +the rest of the path were relative to the user's home directory. +e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user +whose home directory is `/home/edolstra`. + +Paths can also be specified between angle brackets, e.g. +``. This means that the directories listed in the +environment variable `NIX_PATH` will be searched for the given file +or directory name. + +Antiquotation is supported in any paths except those in angle brackets. +`./${foo}-${bar}.nix` is a more convenient way of writing +`./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At +least one slash must appear *before* any antiquotations for this to be +recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division +operation. `./a.${foo}/b.${bar}` is a path. + +### Booleans + +*Booleans* with values `true` and `false`. + +### Null + +The null value, denoted as `null`. ## Lists From 4ff48854b85f5af6b7448957beae448939c56884 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:11:23 +0200 Subject: [PATCH 037/371] manual: simple values -> primitives "simple" is a loaded term --- doc/manual/src/expressions/language-values.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index fa5743222..19d2f7c0d 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -1,8 +1,6 @@ # Values -## Simple Values - -Nix has the following basic data types: +## Primitives ### Strings From 8f4fab8fab9c321aea62b2953cbe16083781f099 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:25:18 +0200 Subject: [PATCH 038/371] manual: use singular for headings --- doc/manual/src/expressions/language-values.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 19d2f7c0d..6b1047c71 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -2,7 +2,7 @@ ## Primitives -### Strings +### String *Strings* can be written in three ways. @@ -112,7 +112,7 @@ is*, without quotes. For instance, the string `"http://example.org/foo.tar.bz2"` can also be written as `http://example.org/foo.tar.bz2`. -### Numbers +### Number Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). @@ -121,7 +121,7 @@ Numbers are type-compatible: pure integer operations will always return integers, whereas any operation involving at least one floating point number will have a floating point number as a result. -### Paths +### Path *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at least one slash to be recognised as such. For instance, `builder.sh` @@ -150,7 +150,7 @@ least one slash must appear *before* any antiquotations for this to be recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. -### Booleans +### Boolean *Booleans* with values `true` and `false`. @@ -158,7 +158,7 @@ operation. `./a.${foo}/b.${bar}` is a path. The null value, denoted as `null`. -## Lists +## List Lists are formed by enclosing a whitespace-separated list of values between square brackets. For example, @@ -180,7 +180,7 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. -## Attribute Sets +## Attribute Set Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). From 41a3b315fd24b83d7e570e1dae8d384317fb89e6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 17:25:25 +0200 Subject: [PATCH 039/371] manual: values -> data types --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/expressions/language-values.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5df4e2d75..c8cb72fc0 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -34,7 +34,7 @@ - [Building and Testing](expressions/simple-building-testing.md) - [Generic Builder Syntax](expressions/generic-builder.md) - [Nix Expression Language](expressions/expression-language.md) - - [Values](expressions/language-values.md) + - [Data Types](expressions/language-values.md) - [Language Constructs](expressions/language-constructs.md) - [Operators](expressions/language-operators.md) - [Derivations](expressions/derivations.md) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 6b1047c71..e4e3bf181 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -1,4 +1,4 @@ -# Values +# Data Types ## Primitives From 27138f1ec69c7a6f09ab467de317909780e4ae7a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 23:30:07 +0200 Subject: [PATCH 040/371] manual: use singular in body, too Co-authored-by: Cole Helbling --- doc/manual/src/expressions/language-values.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index e4e3bf181..4aa354ba6 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -182,7 +182,7 @@ Note that lists are only lazy in values, and they are strict in length. ## Attribute Set -Attribute sets are collections of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). From 297f6b5d56204c5ece34c88ba09c50d4e12ccf89 Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Thu, 14 Jul 2022 09:45:48 -0700 Subject: [PATCH 041/371] Fix link to hacking doc Right now, https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html redirects to https://hydra.nixos.org/build/183877779/download/1/manual/contributing/hacking.html, which gives me a "500 Internal Server Error". Not super useful =( Feel free to ignore if someone's working to fix the 500 I was running into. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80d6f128c..8a02c4c75 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Information on additional installation methods is available on the [Nix download ## Building And Developing -See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual/contributing/hacking.html) in our manual for instruction on how to +See our [Hacking guide](https://nixos.org/manual/nix/stable/contributing/hacking.html) in our manual for instruction on how to build nix from source with nix-build or how to get a development environment. ## Additional Resources From 1467a98d4c2014e512825c87e9056de79408421a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Mon, 1 Aug 2022 11:37:21 +0200 Subject: [PATCH 042/371] derivation-goal.cc: remove bmCheck custom return branch on buildDone Once a derivation goal has been completed, we check whether or not this goal was meant to be repeated to check its output. An early return branch was preventing the worker to reach that repeat code branch, hence breaking the --check command (#2619). It seems like this early return branch is an artifact of a passed refactoring. As far as I can tell, buildDone's main branch also cleanup the tmp directory before returning. --- src/libstore/build/derivation-goal.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..3c0b14029 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -914,12 +914,6 @@ void DerivationGoal::buildDone() outputPaths ); - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built, std::move(builtOutputs)); - return; - } - cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ From f675ba53310f1bfbe6e9867184b3a5177532b370 Mon Sep 17 00:00:00 2001 From: K900 Date: Mon, 1 Aug 2022 13:50:35 +0300 Subject: [PATCH 043/371] doc/distributed-builds: don't use deprecated alias `nix ping-store` -> `nix store ping`. --- doc/manual/src/advanced-topics/distributed-builds.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index b0d7fbf1a..fefd10100 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix ping-store --store ssh://mac +$ nix store ping --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a From 8119390abcbb25e849acc50d0af0b37d85adfb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:08 +0100 Subject: [PATCH 044/371] Move some fs-related functions to their own file Unclutter `util.cc` a bit --- src/libutil/filesystem.cc | 44 +++++++++++++++++++++++++++++++++++++++ src/libutil/util.cc | 38 --------------------------------- 2 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 src/libutil/filesystem.cc diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc new file mode 100644 index 000000000..e67383e6f --- /dev/null +++ b/src/libutil/filesystem.cc @@ -0,0 +1,44 @@ +#include + +#include "util.hh" +#include "types.hh" + +namespace nix { + +void createSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } +} + +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp, mtime); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmp, link); + + break; + } +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..3f3695f0d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -681,44 +681,6 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - - -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); - - break; - } -} - - void readFull(int fd, char * buf, size_t count) { while (count) { From c2de0a232c1cfddb1f1385ffd23dd43a2099458e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:46 +0100 Subject: [PATCH 045/371] =?UTF-8?q?Create=20a=20wrapper=20around=20stdlib?= =?UTF-8?q?=E2=80=99s=20`rename`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Directly takes some c++ strings, and gently throws an exception on error (rather than having to inline this logic everywhere) --- src/libstore/build/derivation-goal.cc | 3 +-- src/libstore/build/local-derivation-goal.cc | 8 +++----- src/libstore/builtins/unpack-channel.cc | 3 +-- src/libstore/gc.cc | 4 +--- src/libstore/local-binary-cache-store.cc | 3 +-- src/libstore/local-store.cc | 6 ++---- src/libstore/optimise-store.cc | 6 ++++-- src/libutil/filesystem.cc | 9 +++++++-- src/libutil/util.hh | 2 ++ 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..93f923f18 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 79a241ae0..2435377e2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); + moveFile((chrootRootDir + p), p); } return diskFull; @@ -2625,8 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); + moveFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 426d58a53..a8417d7ff 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) - throw SysError("renaming channel directory"); + moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d58ed78b1..3682f81cc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError("cannot rename '%1%' to '%2%'", - tempLink , link); + moveFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ba4416f6d..ff7403f9d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,8 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - if (rename(tmp.c_str(), path2.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path2); + moveFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eba3b0fa5..7f708b243 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - if (rename(tmpFile.c_str(), logPath.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); + moveFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8af9b1dde..20b9c7611 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { + try { + moveFile(tempLink, path); + } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { @@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); + throw; } stats.filesLinked++; diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index e67383e6f..33a8d81a6 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -34,11 +34,16 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); + moveFile(tmp, link); break; } } +void moveFile(const Path & oldName, const Path & newName) +{ + if (::rename(oldName.c_str(), newName.c_str())) + throw SysError("renaming '%1%' to '%2%'", oldName, newName); +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29227ecc6..564d36e79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,6 +168,8 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ From 6f89fb60088c4bc1513a005f0350c2bc13068892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 16:13:29 +0100 Subject: [PATCH 046/371] rename: Fallback to a copy if the filesystems mismatch In `nix::rename`, if the call to `rename` fails with `EXDEV` (failure because the source and the destination are in a different filesystems) switch to copying and removing the source. To avoid having to re-implement the copy manually, I switched the function to use the c++17 `filesystem` library (which has a `copy` function that should do what we want). Fix #6262 --- src/libutil/filesystem.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 33a8d81a6..cf99b848a 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,8 +1,11 @@ #include +#include #include "util.hh" #include "types.hh" +namespace fs = std::filesystem; + namespace nix { void createSymlink(const Path & target, const Path & link, @@ -42,8 +45,16 @@ void replaceSymlink(const Path & target, const Path & link, void moveFile(const Path & oldName, const Path & newName) { - if (::rename(oldName.c_str(), newName.c_str())) - throw SysError("renaming '%1%' to '%2%'", oldName, newName); + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + try { + fs::rename(oldPath, newPath); + } catch (fs::filesystem_error & e) { + if (e.code().value() == EXDEV) { + fs::copy(oldName, newName, fs::copy_options::copy_symlinks); + fs::remove_all(oldName); + } + } } } From c5db1821a94406eaff86341220bc301ba1dac82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Mar 2022 14:25:56 +0100 Subject: [PATCH 047/371] Re-implement the recursive directory copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recursive copy from the stl doesn’t exactly do what we need because 1. It doesn’t delete things as we go 2. It doesn’t keep the mtime, which change the nars So re-implement it ourselves. A bit dull, but that way we have what we want --- src/libutil/filesystem.cc | 52 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index cf99b848a..198db2832 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -43,6 +43,53 @@ void replaceSymlink(const Path & target, const Path & link, } } +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + void moveFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); @@ -51,8 +98,9 @@ void moveFile(const Path & oldName, const Path & newName) fs::rename(oldPath, newPath); } catch (fs::filesystem_error & e) { if (e.code().value() == EXDEV) { - fs::copy(oldName, newName, fs::copy_options::copy_symlinks); - fs::remove_all(oldName); + fs::remove(newPath); + warn("Copying %s to %s", oldName, newName); + copy(fs::directory_entry(oldPath), newPath, true); } } } From a4f0fd633c9ec865d385b34bdc51923c204e7ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 11:52:39 +0200 Subject: [PATCH 048/371] Link against c++fs on darwin Required by the old clang version --- src/libutil/local.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..13e8d426a 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,3 +11,7 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif + +ifdef HOST_DARWIN + libutil_LDFLAGS += -lc++fs +endif From d71d9e9fbfc972514b0b940181ab7f9e766f41f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:10:36 +0200 Subject: [PATCH 049/371] moveFile -> renameFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `move` tends to have this `mv` connotation of “I will copy it for you if needs be” --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +++--- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- src/libutil/filesystem.cc | 4 ++-- src/libutil/util.hh | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 93f923f18..459fdae79 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,7 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2435377e2..6843173a7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,7 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -310,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - moveFile((chrootRootDir + p), p); + renameFile((chrootRootDir + p), p); } return diskFull; @@ -2624,7 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - moveFile(path, dst); + renameFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index a8417d7ff..ba04bb16c 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,7 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); + renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3682f81cc..4c1a82279 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,7 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - moveFile(tempLink, link); + renameFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ff7403f9d..f20b1fa02 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,7 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - moveFile(tmp, path2); + renameFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7f708b243..2d076f404 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - moveFile(tempPath, realPath); + renameFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1941,7 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - moveFile(tmpFile, logPath); + renameFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 20b9c7611..4d2781180 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -230,7 +230,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ try { - moveFile(tempLink, path); + renameFile(tempLink, path); } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 198db2832..fee40d09e 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -37,7 +37,7 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - moveFile(tmp, link); + renameFile(tmp, link); break; } @@ -90,7 +90,7 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) } } -void moveFile(const Path & oldName, const Path & newName) +void renameFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 564d36e79..186513cea 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,7 +168,7 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); -void moveFile(const Path & src, const Path & dst); +void renameFile(const Path & src, const Path & dst); /* Wrappers arount read()/write() that read/write exactly the From 90f968073338d6ba276994bc281fa5efa5306e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:19:42 +0200 Subject: [PATCH 050/371] Only use `renameFile` where needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In most places the fallback to copying isn’t needed and can actually be bad, so we’d rather not transparently fallback --- src/libstore/local-store.cc | 2 +- src/libutil/filesystem.cc | 14 +++++++++----- src/libutil/util.hh | 9 +++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d076f404..a272e4301 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - renameFile(tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index fee40d09e..9cc18507b 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -54,7 +54,6 @@ void setWriteTime(const fs::path & p, const struct stat & st) .tv_sec = st.st_mtime, .tv_usec = 0, }; - warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); if (lutimes(p.c_str(), times) != 0) throw SysError("changing modification time of '%s'", p); } @@ -92,14 +91,19 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) void renameFile(const Path & oldName, const Path & newName) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ try { - fs::rename(oldPath, newPath); + renameFile(oldName, newName); } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); if (e.code().value() == EXDEV) { fs::remove(newPath); - warn("Copying %s to %s", oldName, newName); + warn("Can’t rename %s as %s, copying instead", oldName, newName); copy(fs::directory_entry(oldPath), newPath, true); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 186513cea..cd83f250f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -170,6 +170,15 @@ void replaceSymlink(const Path & target, const Path & link, void renameFile(const Path & src, const Path & dst); +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ From 1ba5b3e001c3da3c2e2a4dc7d475da1b20f53638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:48:31 +0200 Subject: [PATCH 051/371] Make `moveFile` more atomic Rather than directly copying the source to its dest, copy it first to a temporary location, and eventually move that temporary. That way, the move is at least atomic from the point-of-view of the destination --- src/libutil/filesystem.cc | 62 ++++++++++++++++++++++++++++++++++++++- src/libutil/util.cc | 55 ---------------------------------- 2 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 9cc18507b..403389e60 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,6 +1,7 @@ #include #include +#include "finally.hh" #include "util.hh" #include "types.hh" @@ -8,6 +9,59 @@ namespace fs = std::filesystem; namespace nix { +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + void createSymlink(const Path & target, const Path & link, std::optional mtime) { @@ -101,10 +155,16 @@ void moveFile(const Path & oldName, const Path & newName) } catch (fs::filesystem_error & e) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), newPath, true); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); } } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3f3695f0d..2a1bb770a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -508,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed) } -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); From d1cda07ce47064bda2c609a0290c867295ddd0a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 25 May 2022 11:37:01 +0200 Subject: [PATCH 052/371] Don't use -load_all on darwin That flag breaks `-lc++fs` (introducing a duplicate symbol for some reason). Besides, it was apparently needed for bzip2, but we're not using bzip2 anymore. --- configure.ac | 9 --------- 1 file changed, 9 deletions(-) diff --git a/configure.ac b/configure.ac index f0210ab78..64fa12fc7 100644 --- a/configure.ac +++ b/configure.ac @@ -296,15 +296,6 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) -# This is needed if bzip2 is a static library, and the Nix libraries -# are dynamic. -case "${host_os}" in - darwin*) - LDFLAGS="-all_load $LDFLAGS" - ;; -esac - - AC_ARG_WITH(sandbox-shell, AS_HELP_STRING([--with-sandbox-shell=PATH],[path of a statically-linked shell to use as /bin/sh in sandboxes]), sandbox_shell=$withval) AC_SUBST(sandbox_shell) From ceed4d41426f6e2dc74473d5c137a8f061c49061 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 3 Aug 2022 11:23:40 +0200 Subject: [PATCH 053/371] encode primitive as list with anchors to make it consistent with builtins and configuration options --- doc/manual/src/expressions/language-values.md | 248 +++++++++--------- 1 file changed, 124 insertions(+), 124 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 4aa354ba6..a53b2cb22 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -2,161 +2,161 @@ ## Primitives -### String +- String -*Strings* can be written in three ways. + *Strings* can be written in three ways. -The most common way is to enclose the string between double quotes, -e.g., `"foo bar"`. Strings can span multiple lines. The special -characters `"` and `\` and the character sequence `${` must be -escaped by prefixing them with a backslash (`\`). Newlines, carriage -returns and tabs can be written as `\n`, `\r` and `\t`, -respectively. + The most common way is to enclose the string between double quotes, + e.g., `"foo bar"`. Strings can span multiple lines. The special + characters `"` and `\` and the character sequence `${` must be + escaped by prefixing them with a backslash (`\`). Newlines, carriage + returns and tabs can be written as `\n`, `\r` and `\t`, + respectively. -You can include the result of an expression into a string by -enclosing it in `${...}`, a feature known as *antiquotation*. The -enclosed expression must evaluate to something that can be coerced -into a string (meaning that it must be a string, a path, or a -derivation). For instance, rather than writing + You can include the result of an expression into a string by + enclosing it in `${...}`, a feature known as *antiquotation*. The + enclosed expression must evaluate to something that can be coerced + into a string (meaning that it must be a string, a path, or a + derivation). For instance, rather than writing -```nix -"--with-freetype2-library=" + freetype + "/lib" -``` + ```nix + "--with-freetype2-library=" + freetype + "/lib" + ``` -(where `freetype` is a derivation), you can instead write the more -natural + (where `freetype` is a derivation), you can instead write the more + natural -```nix -"--with-freetype2-library=${freetype}/lib" -``` + ```nix + "--with-freetype2-library=${freetype}/lib" + ``` -The latter is automatically translated to the former. A more -complicated example (from the Nix expression for -[Qt](http://www.trolltech.com/products/qt)): + The latter is automatically translated to the former. A more + complicated example (from the Nix expression for + [Qt](http://www.trolltech.com/products/qt)): -```nix -configureFlags = " - -system-zlib -system-libpng -system-libjpeg - ${if openglSupport then "-dlopen-opengl - -L${mesa}/lib -I${mesa}/include - -L${libXmu}/lib -I${libXmu}/include" else ""} - ${if threadSupport then "-thread" else "-no-thread"} -"; -``` + ```nix + configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} + "; + ``` -Note that Nix expressions and strings can be arbitrarily nested; in -this case the outer string contains various antiquotations that -themselves contain strings (e.g., `"-thread"`), some of which in -turn contain expressions (e.g., `${mesa}`). + Note that Nix expressions and strings can be arbitrarily nested; in + this case the outer string contains various antiquotations that + themselves contain strings (e.g., `"-thread"`), some of which in + turn contain expressions (e.g., `${mesa}`). -The second way to write string literals is as an *indented string*, -which is enclosed between pairs of *double single-quotes*, like so: + The second way to write string literals is as an *indented string*, + which is enclosed between pairs of *double single-quotes*, like so: -```nix -'' - This is the first line. - This is the second line. - This is the third line. -'' -``` + ```nix + '' + This is the first line. + This is the second line. + This is the third line. + '' + ``` -This kind of string literal intelligently strips indentation from -the start of each line. To be precise, it strips from each line a -number of spaces equal to the minimal indentation of the string as a -whole (disregarding the indentation of empty lines). For instance, -the first and second line are indented two spaces, while the third -line is indented four spaces. Thus, two spaces are stripped from -each line, so the resulting string is + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as a + whole (disregarding the indentation of empty lines). For instance, + the first and second line are indented two spaces, while the third + line is indented four spaces. Thus, two spaces are stripped from + each line, so the resulting string is -```nix -"This is the first line.\nThis is the second line.\n This is the third line.\n" -``` + ```nix + "This is the first line.\nThis is the second line.\n This is the third line.\n" + ``` -Note that the whitespace and newline following the opening `''` is -ignored if there is no non-whitespace text on the initial line. + Note that the whitespace and newline following the opening `''` is + ignored if there is no non-whitespace text on the initial line. -Antiquotation (`${expr}`) is supported in indented strings. + Antiquotation (`${expr}`) is supported in indented strings. -Since `${` and `''` have special meaning in indented strings, you -need a way to quote them. `$` can be escaped by prefixing it with -`''` (that is, two single quotes), i.e., `''$`. `''` can be escaped -by prefixing it with `'`, i.e., `'''`. `$` removes any special -meaning from the following `$`. Linefeed, carriage-return and tab -characters can be written as `''\n`, `''\r`, `''\t`, and `''\` -escapes any other character. + Since `${` and `''` have special meaning in indented strings, you + need a way to quote them. `$` can be escaped by prefixing it with + `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped + by prefixing it with `'`, i.e., `'''`. `$` removes any special + meaning from the following `$`. Linefeed, carriage-return and tab + characters can be written as `''\n`, `''\r`, `''\t`, and `''\` + escapes any other character. -Indented strings are primarily useful in that they allow multi-line -string literals to follow the indentation of the enclosing Nix -expression, and that less escaping is typically necessary for -strings representing languages such as shell scripts and -configuration files because `''` is much less common than `"`. -Example: + Indented strings are primarily useful in that they allow multi-line + string literals to follow the indentation of the enclosing Nix + expression, and that less escaping is typically necessary for + strings representing languages such as shell scripts and + configuration files because `''` is much less common than `"`. + Example: -```nix -stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... -} -``` + ```nix + stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... + } + ``` -Finally, as a convenience, *URIs* as defined in appendix B of -[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as -is*, without quotes. For instance, the string -`"http://example.org/foo.tar.bz2"` can also be written as -`http://example.org/foo.tar.bz2`. + Finally, as a convenience, *URIs* as defined in appendix B of + [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as + is*, without quotes. For instance, the string + `"http://example.org/foo.tar.bz2"` can also be written as + `http://example.org/foo.tar.bz2`. -### Number +- Number -Numbers, which can be *integers* (like `123`) or *floating point* -(like `123.43` or `.27e13`). + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). -Numbers are type-compatible: pure integer operations will always -return integers, whereas any operation involving at least one -floating point number will have a floating point number as a result. + Numbers are type-compatible: pure integer operations will always + return integers, whereas any operation involving at least one + floating point number will have a floating point number as a result. -### Path +- Path -*Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at -least one slash to be recognised as such. For instance, `builder.sh` -is not a path: it's parsed as an expression that selects the -attribute `sh` from the variable `builder`. If the file name is -relative, i.e., if it does not begin with a slash, it is made -absolute at parse time relative to the directory of the Nix -expression that contained it. For instance, if a Nix expression in -`/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path -is `/foo/xyzzy/fnord.nix`. + *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at + least one slash to be recognised as such. For instance, `builder.sh` + is not a path: it's parsed as an expression that selects the + attribute `sh` from the variable `builder`. If the file name is + relative, i.e., if it does not begin with a slash, it is made + absolute at parse time relative to the directory of the Nix + expression that contained it. For instance, if a Nix expression in + `/foo/bar/bla.nix` refers to `../xyzzy/fnord.nix`, the absolute path + is `/foo/xyzzy/fnord.nix`. -If the first component of a path is a `~`, it is interpreted as if -the rest of the path were relative to the user's home directory. -e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user -whose home directory is `/home/edolstra`. + If the first component of a path is a `~`, it is interpreted as if + the rest of the path were relative to the user's home directory. + e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user + whose home directory is `/home/edolstra`. -Paths can also be specified between angle brackets, e.g. -``. This means that the directories listed in the -environment variable `NIX_PATH` will be searched for the given file -or directory name. + Paths can also be specified between angle brackets, e.g. + ``. This means that the directories listed in the + environment variable `NIX_PATH` will be searched for the given file + or directory name. -Antiquotation is supported in any paths except those in angle brackets. -`./${foo}-${bar}.nix` is a more convenient way of writing -`./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At -least one slash must appear *before* any antiquotations for this to be -recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division -operation. `./a.${foo}/b.${bar}` is a path. + Antiquotation is supported in any paths except those in angle brackets. + `./${foo}-${bar}.nix` is a more convenient way of writing + `./. + "/" + foo + "-" + bar + ".nix"` or `./. + "/${foo}-${bar}.nix"`. At + least one slash must appear *before* any antiquotations for this to be + recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division + operation. `./a.${foo}/b.${bar}` is a path. -### Boolean +- Boolean -*Booleans* with values `true` and `false`. + *Booleans* with values `true` and `false`. -### Null +- Null -The null value, denoted as `null`. + The null value, denoted as `null`. ## List From b47b6a418d656ee5408b7773dd9b76770ca0d2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Wed, 3 Aug 2022 12:05:41 +0200 Subject: [PATCH 054/371] tests/check.sh: add nix3-build check test --- tests/check.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/check.sh b/tests/check.sh index ab48ff865..495202781 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -40,6 +40,14 @@ nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix deterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log +if grep -q 'checking is not possible' $TEST_ROOT/log; then false; fi +# Repeat is set to 1, ie. nix should build deterministic twice. +if [ "$(grep "checking outputs" $TEST_ROOT/log | wc -l)" -ne 2 ]; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log @@ -50,6 +58,12 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] checkBuildTempDirRemoved $TEST_ROOT/log +nix build -f check.nix nondeterministic --rebuild --repeat 1 \ + --argstr checkBuildId $checkBuildId --keep-failed --no-link \ + 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log From c55bea420402fcf995688ef6a12cfa413bc5e35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 3 Aug 2022 14:16:00 +0200 Subject: [PATCH 055/371] Fix the html id of the list headers --- doc/manual/src/expressions/language-values.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index a53b2cb22..f09400d02 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -2,7 +2,7 @@ ## Primitives -- String +- String *Strings* can be written in three ways. @@ -112,7 +112,7 @@ `"http://example.org/foo.tar.bz2"` can also be written as `http://example.org/foo.tar.bz2`. -- Number +- Number Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). @@ -121,7 +121,7 @@ return integers, whereas any operation involving at least one floating point number will have a floating point number as a result. -- Path +- Path *Paths*, e.g., `/bin/sh` or `./builder.sh`. A path must contain at least one slash to be recognised as such. For instance, `builder.sh` @@ -150,11 +150,11 @@ recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. -- Boolean +- Boolean *Booleans* with values `true` and `false`. -- Null +- Null The null value, denoted as `null`. From ccbd906c86cb10e5c7b49753e057fec4ab46233c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Aug 2022 17:45:11 +0200 Subject: [PATCH 056/371] Fix NIX_COUNT_CALLS=1 Also, make the JSON writer support std::string_view. Fixes #6857. --- src/libexpr/eval.cc | 12 ++++++------ src/libutil/json.cc | 21 +++++++++------------ src/libutil/json.hh | 11 +++++------ src/libutil/tests/json.cc | 4 ++-- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f485e2fed..e3716f217 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2501,18 +2501,18 @@ void EvalState::printStats() } { auto list = topObj.list("functions"); - for (auto & i : functionCalls) { + for (auto & [fun, count] : functionCalls) { auto obj = list.object(); - if (i.first->name) - obj.attr("name", (const std::string &) i.first->name); + if (fun->name) + obj.attr("name", (std::string_view) symbols[fun->name]); else obj.attr("name", nullptr); - if (auto pos = positions[i.first->pos]) { - obj.attr("file", (const std::string &) pos.file); + if (auto pos = positions[fun->pos]) { + obj.attr("file", (std::string_view) pos.file); obj.attr("line", pos.line); obj.attr("column", pos.column); } - obj.attr("count", i.second); + obj.attr("count", count); } } { diff --git a/src/libutil/json.cc b/src/libutil/json.cc index b0a5d7e75..abe0e6e74 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -6,7 +6,8 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end) +template<> +void toJSON(std::ostream & str, const std::string_view & s) { constexpr size_t BUF_SIZE = 4096; char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts @@ -21,7 +22,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) }; put('"'); - for (auto i = start; i != end; i++) { + for (auto i = s.begin(); i != s.end(); i++) { if (bufPos >= BUF_SIZE) flush(); if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } else if (*i == '\n') { put('\\'); put('n'); } @@ -44,7 +45,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) void toJSON(std::ostream & str, const char * s) { - if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); + if (!s) str << "null"; else toJSON(str, std::string_view(s)); } template<> void toJSON(std::ostream & str, const int & n) { str << n; } @@ -55,11 +56,7 @@ template<> void toJSON(std::ostream & str, const long long & n) { str template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } - -template<> void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} +template<> void toJSON(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); } template<> void toJSON(std::ostream & str, const bool & b) { @@ -154,7 +151,7 @@ JSONObject::~JSONObject() } } -void JSONObject::attr(const std::string & s) +void JSONObject::attr(std::string_view s) { comma(); toJSON(state->str, s); @@ -162,19 +159,19 @@ void JSONObject::attr(const std::string & s) if (state->indent) state->str << ' '; } -JSONList JSONObject::list(const std::string & name) +JSONList JSONObject::list(std::string_view name) { attr(name); return JSONList(state); } -JSONObject JSONObject::object(const std::string & name) +JSONObject JSONObject::object(std::string_view name) { attr(name); return JSONObject(state); } -JSONPlaceholder JSONObject::placeholder(const std::string & name) +JSONPlaceholder JSONObject::placeholder(std::string_view name) { attr(name); return JSONPlaceholder(state); diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 83213ca66..3790b1a2e 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -6,7 +6,6 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end); void toJSON(std::ostream & str, const char * s); template @@ -107,7 +106,7 @@ private: open(); } - void attr(const std::string & s); + void attr(std::string_view s); public: @@ -128,18 +127,18 @@ public: ~JSONObject(); template - JSONObject & attr(const std::string & name, const T & v) + JSONObject & attr(std::string_view name, const T & v) { attr(name); toJSON(state->str, v); return *this; } - JSONList list(const std::string & name); + JSONList list(std::string_view name); - JSONObject object(const std::string & name); + JSONObject object(std::string_view name); - JSONPlaceholder placeholder(const std::string & name); + JSONPlaceholder placeholder(std::string_view name); }; class JSONPlaceholder : JSONWriter diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc index dea73f53a..156286999 100644 --- a/src/libutil/tests/json.cc +++ b/src/libutil/tests/json.cc @@ -102,8 +102,8 @@ namespace nix { TEST(toJSON, substringEscape) { std::stringstream out; - const char *s = "foo\t"; - toJSON(out, s+3, s + strlen(s)); + std::string_view s = "foo\t"; + toJSON(out, s.substr(3)); ASSERT_EQ(out.str(), "\"\\t\""); } From c9f446ede1b09a9daff59e479601f49382071a22 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 4 Aug 2022 09:45:28 +0200 Subject: [PATCH 057/371] flakeref: fix comment --- src/libexpr/flake/flakeref.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a9182f4bf..fe4f67193 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -28,7 +28,7 @@ typedef std::string FlakeId; * object that fetcher generates (usually via * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). * - * The actual fetch not have been performed yet (i.e. a FlakeRef may + * The actual fetch may not have been performed yet (i.e. a FlakeRef may * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ From 53833dfb40ddfc978f47f5c180a73bc7e0400320 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 4 Aug 2022 09:45:29 +0200 Subject: [PATCH 058/371] libexpr/flake: remove `FIXME` Line 593 checks that all overrides (i.e. all elements of `lockFlags.inputOverrides`) are members of `overridesUsed`. --- src/libexpr/flake/flake.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index cc9be1336..105e76bc6 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -341,7 +341,6 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); - // FIXME: check whether all overrides are used. std::map overrides; std::set overridesUsed, updatesUsed; From 4c8441be0a59cead901acdfc3285d835c400c615 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Thu, 4 Aug 2022 09:45:30 +0200 Subject: [PATCH 059/371] docs/flake-update: fix example --- src/nix/flake-update.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 03b50e38e..2ee8a707d 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -6,7 +6,7 @@ R""( lock file: ```console - # nix flake update + # nix flake update --commit-lock-file * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' … From 499ed265088948e823c83cc95d1097a6362d205b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 11:36:32 +0200 Subject: [PATCH 060/371] manual: remove "Writing Nix Expressions" chapter it is out of date, all over the place in level of detail, is really about `nixpkgs`, and in general instructions should not be part of a reference manual. also: - update redirects and internal links - use "Nix language" consistently --- doc/manual/local.mk | 8 +- doc/manual/redirects.js | 207 +++++++++--------- doc/manual/src/SUMMARY.md.in | 23 +- doc/manual/src/command-ref/nix-env.md | 2 +- doc/manual/src/command-ref/nix-instantiate.md | 4 +- doc/manual/src/command-ref/nix-store.md | 2 +- doc/manual/src/command-ref/opt-common.md | 4 +- .../src/expressions/arguments-variables.md | 80 ------- doc/manual/src/expressions/build-script.md | 70 ------ .../src/expressions/expression-syntax.md | 93 -------- doc/manual/src/expressions/generic-builder.md | 66 ------ .../expressions/simple-building-testing.md | 61 ------ .../src/expressions/simple-expression.md | 23 -- .../expressions/writing-nix-expressions.md | 12 - doc/manual/src/glossary.md | 2 +- .../advanced-attributes.md | 0 .../builtin-constants.md | 0 .../builtins-prefix.md | 0 .../builtins-suffix.md | 0 .../constructs.md} | 0 .../{expressions => language}/derivations.md | 0 .../index.md} | 5 +- .../operators.md} | 2 +- .../language-values.md => language/values.md} | 0 .../package-management/package-management.md | 3 +- 25 files changed, 124 insertions(+), 543 deletions(-) delete mode 100644 doc/manual/src/expressions/arguments-variables.md delete mode 100644 doc/manual/src/expressions/build-script.md delete mode 100644 doc/manual/src/expressions/expression-syntax.md delete mode 100644 doc/manual/src/expressions/generic-builder.md delete mode 100644 doc/manual/src/expressions/simple-building-testing.md delete mode 100644 doc/manual/src/expressions/simple-expression.md delete mode 100644 doc/manual/src/expressions/writing-nix-expressions.md rename doc/manual/src/{expressions => language}/advanced-attributes.md (100%) rename doc/manual/src/{expressions => language}/builtin-constants.md (100%) rename doc/manual/src/{expressions => language}/builtins-prefix.md (100%) rename doc/manual/src/{expressions => language}/builtins-suffix.md (100%) rename doc/manual/src/{expressions/language-constructs.md => language/constructs.md} (100%) rename doc/manual/src/{expressions => language}/derivations.md (100%) rename doc/manual/src/{expressions/expression-language.md => language/index.md} (84%) rename doc/manual/src/{expressions/language-operators.md => language/operators.md} (99%) rename doc/manual/src/{expressions/language-values.md => language/values.md} (100%) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 371ed6f21..02a88e4fb 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -61,10 +61,10 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix - @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp +$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp - @cat doc/manual/src/expressions/builtins-suffix.md >> $@.tmp + @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix @@ -97,7 +97,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(call rwildcard, $(d)/src, *.md) $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 19f928c7e..167e221b8 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -132,113 +132,106 @@ var redirects = { "#sec-common-options": "command-ref/opt-common.html", "#ch-utilities": "command-ref/utilities.html", "#chap-hacking": "contributing/hacking.html", - "#adv-attr-allowSubstitutes": "expressions/advanced-attributes.html#adv-attr-allowSubstitutes", - "#adv-attr-allowedReferences": "expressions/advanced-attributes.html#adv-attr-allowedReferences", - "#adv-attr-allowedRequisites": "expressions/advanced-attributes.html#adv-attr-allowedRequisites", - "#adv-attr-disallowedReferences": "expressions/advanced-attributes.html#adv-attr-disallowedReferences", - "#adv-attr-disallowedRequisites": "expressions/advanced-attributes.html#adv-attr-disallowedRequisites", - "#adv-attr-exportReferencesGraph": "expressions/advanced-attributes.html#adv-attr-exportReferencesGraph", - "#adv-attr-impureEnvVars": "expressions/advanced-attributes.html#adv-attr-impureEnvVars", - "#adv-attr-outputHash": "expressions/advanced-attributes.html#adv-attr-outputHash", - "#adv-attr-outputHashAlgo": "expressions/advanced-attributes.html#adv-attr-outputHashAlgo", - "#adv-attr-outputHashMode": "expressions/advanced-attributes.html#adv-attr-outputHashMode", - "#adv-attr-passAsFile": "expressions/advanced-attributes.html#adv-attr-passAsFile", - "#adv-attr-preferLocalBuild": "expressions/advanced-attributes.html#adv-attr-preferLocalBuild", - "#fixed-output-drvs": "expressions/advanced-attributes.html#adv-attr-outputHash", - "#sec-advanced-attributes": "expressions/advanced-attributes.html", - "#sec-arguments": "expressions/arguments-variables.html", - "#sec-build-script": "expressions/build-script.html", - "#builtin-abort": "expressions/builtins.html#builtins-abort", - "#builtin-add": "expressions/builtins.html#builtins-add", - "#builtin-all": "expressions/builtins.html#builtins-all", - "#builtin-any": "expressions/builtins.html#builtins-any", - "#builtin-attrNames": "expressions/builtins.html#builtins-attrNames", - "#builtin-attrValues": "expressions/builtins.html#builtins-attrValues", - "#builtin-baseNameOf": "expressions/builtins.html#builtins-baseNameOf", - "#builtin-bitAnd": "expressions/builtins.html#builtins-bitAnd", - "#builtin-bitOr": "expressions/builtins.html#builtins-bitOr", - "#builtin-bitXor": "expressions/builtins.html#builtins-bitXor", - "#builtin-builtins": "expressions/builtins.html#builtins-builtins", - "#builtin-compareVersions": "expressions/builtins.html#builtins-compareVersions", - "#builtin-concatLists": "expressions/builtins.html#builtins-concatLists", - "#builtin-concatStringsSep": "expressions/builtins.html#builtins-concatStringsSep", - "#builtin-currentSystem": "expressions/builtins.html#builtins-currentSystem", - "#builtin-deepSeq": "expressions/builtins.html#builtins-deepSeq", - "#builtin-derivation": "expressions/builtins.html#builtins-derivation", - "#builtin-dirOf": "expressions/builtins.html#builtins-dirOf", - "#builtin-div": "expressions/builtins.html#builtins-div", - "#builtin-elem": "expressions/builtins.html#builtins-elem", - "#builtin-elemAt": "expressions/builtins.html#builtins-elemAt", - "#builtin-fetchGit": "expressions/builtins.html#builtins-fetchGit", - "#builtin-fetchTarball": "expressions/builtins.html#builtins-fetchTarball", - "#builtin-fetchurl": "expressions/builtins.html#builtins-fetchurl", - "#builtin-filterSource": "expressions/builtins.html#builtins-filterSource", - "#builtin-foldl-prime": "expressions/builtins.html#builtins-foldl-prime", - "#builtin-fromJSON": "expressions/builtins.html#builtins-fromJSON", - "#builtin-functionArgs": "expressions/builtins.html#builtins-functionArgs", - "#builtin-genList": "expressions/builtins.html#builtins-genList", - "#builtin-getAttr": "expressions/builtins.html#builtins-getAttr", - "#builtin-getEnv": "expressions/builtins.html#builtins-getEnv", - "#builtin-hasAttr": "expressions/builtins.html#builtins-hasAttr", - "#builtin-hashFile": "expressions/builtins.html#builtins-hashFile", - "#builtin-hashString": "expressions/builtins.html#builtins-hashString", - "#builtin-head": "expressions/builtins.html#builtins-head", - "#builtin-import": "expressions/builtins.html#builtins-import", - "#builtin-intersectAttrs": "expressions/builtins.html#builtins-intersectAttrs", - "#builtin-isAttrs": "expressions/builtins.html#builtins-isAttrs", - "#builtin-isBool": "expressions/builtins.html#builtins-isBool", - "#builtin-isFloat": "expressions/builtins.html#builtins-isFloat", - "#builtin-isFunction": "expressions/builtins.html#builtins-isFunction", - "#builtin-isInt": "expressions/builtins.html#builtins-isInt", - "#builtin-isList": "expressions/builtins.html#builtins-isList", - "#builtin-isNull": "expressions/builtins.html#builtins-isNull", - "#builtin-isString": "expressions/builtins.html#builtins-isString", - "#builtin-length": "expressions/builtins.html#builtins-length", - "#builtin-lessThan": "expressions/builtins.html#builtins-lessThan", - "#builtin-listToAttrs": "expressions/builtins.html#builtins-listToAttrs", - "#builtin-map": "expressions/builtins.html#builtins-map", - "#builtin-match": "expressions/builtins.html#builtins-match", - "#builtin-mul": "expressions/builtins.html#builtins-mul", - "#builtin-parseDrvName": "expressions/builtins.html#builtins-parseDrvName", - "#builtin-path": "expressions/builtins.html#builtins-path", - "#builtin-pathExists": "expressions/builtins.html#builtins-pathExists", - "#builtin-placeholder": "expressions/builtins.html#builtins-placeholder", - "#builtin-readDir": "expressions/builtins.html#builtins-readDir", - "#builtin-readFile": "expressions/builtins.html#builtins-readFile", - "#builtin-removeAttrs": "expressions/builtins.html#builtins-removeAttrs", - "#builtin-replaceStrings": "expressions/builtins.html#builtins-replaceStrings", - "#builtin-seq": "expressions/builtins.html#builtins-seq", - "#builtin-sort": "expressions/builtins.html#builtins-sort", - "#builtin-split": "expressions/builtins.html#builtins-split", - "#builtin-splitVersion": "expressions/builtins.html#builtins-splitVersion", - "#builtin-stringLength": "expressions/builtins.html#builtins-stringLength", - "#builtin-sub": "expressions/builtins.html#builtins-sub", - "#builtin-substring": "expressions/builtins.html#builtins-substring", - "#builtin-tail": "expressions/builtins.html#builtins-tail", - "#builtin-throw": "expressions/builtins.html#builtins-throw", - "#builtin-toFile": "expressions/builtins.html#builtins-toFile", - "#builtin-toJSON": "expressions/builtins.html#builtins-toJSON", - "#builtin-toPath": "expressions/builtins.html#builtins-toPath", - "#builtin-toString": "expressions/builtins.html#builtins-toString", - "#builtin-toXML": "expressions/builtins.html#builtins-toXML", - "#builtin-trace": "expressions/builtins.html#builtins-trace", - "#builtin-tryEval": "expressions/builtins.html#builtins-tryEval", - "#builtin-typeOf": "expressions/builtins.html#builtins-typeOf", - "#ssec-builtins": "expressions/builtins.html", - "#attr-system": "expressions/derivations.html#attr-system", - "#ssec-derivation": "expressions/derivations.html", - "#ch-expression-language": "expressions/expression-language.html", - "#sec-expression-syntax": "expressions/expression-syntax.html", - "#sec-generic-builder": "expressions/generic-builder.html", - "#sec-constructs": "expressions/language-constructs.html", - "#sect-let-expressions": "expressions/language-constructs.html#let-expressions", - "#ss-functions": "expressions/language-constructs.html#functions", - "#sec-language-operators": "expressions/language-operators.html", - "#table-operators": "expressions/language-operators.html", - "#ssec-values": "expressions/language-values.html", - "#sec-building-simple": "expressions/simple-building-testing.html", - "#ch-simple-expression": "expressions/simple-expression.html", - "#chap-writing-nix-expressions": "expressions/writing-nix-expressions.html", + "#adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", + "#adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", + "#adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", + "#adv-attr-disallowedReferences": "language/advanced-attributes.html#adv-attr-disallowedReferences", + "#adv-attr-disallowedRequisites": "language/advanced-attributes.html#adv-attr-disallowedRequisites", + "#adv-attr-exportReferencesGraph": "language/advanced-attributes.html#adv-attr-exportReferencesGraph", + "#adv-attr-impureEnvVars": "language/advanced-attributes.html#adv-attr-impureEnvVars", + "#adv-attr-outputHash": "language/advanced-attributes.html#adv-attr-outputHash", + "#adv-attr-outputHashAlgo": "language/advanced-attributes.html#adv-attr-outputHashAlgo", + "#adv-attr-outputHashMode": "language/advanced-attributes.html#adv-attr-outputHashMode", + "#adv-attr-passAsFile": "language/advanced-attributes.html#adv-attr-passAsFile", + "#adv-attr-preferLocalBuild": "language/advanced-attributes.html#adv-attr-preferLocalBuild", + "#fixed-output-drvs": "language/advanced-attributes.html#adv-attr-outputHash", + "#sec-advanced-attributes": "language/advanced-attributes.html", + "#builtin-abort": "language/builtins.html#builtins-abort", + "#builtin-add": "language/builtins.html#builtins-add", + "#builtin-all": "language/builtins.html#builtins-all", + "#builtin-any": "language/builtins.html#builtins-any", + "#builtin-attrNames": "language/builtins.html#builtins-attrNames", + "#builtin-attrValues": "language/builtins.html#builtins-attrValues", + "#builtin-baseNameOf": "language/builtins.html#builtins-baseNameOf", + "#builtin-bitAnd": "language/builtins.html#builtins-bitAnd", + "#builtin-bitOr": "language/builtins.html#builtins-bitOr", + "#builtin-bitXor": "language/builtins.html#builtins-bitXor", + "#builtin-builtins": "language/builtins.html#builtins-builtins", + "#builtin-compareVersions": "language/builtins.html#builtins-compareVersions", + "#builtin-concatLists": "language/builtins.html#builtins-concatLists", + "#builtin-concatStringsSep": "language/builtins.html#builtins-concatStringsSep", + "#builtin-currentSystem": "language/builtins.html#builtins-currentSystem", + "#builtin-deepSeq": "language/builtins.html#builtins-deepSeq", + "#builtin-derivation": "language/builtins.html#builtins-derivation", + "#builtin-dirOf": "language/builtins.html#builtins-dirOf", + "#builtin-div": "language/builtins.html#builtins-div", + "#builtin-elem": "language/builtins.html#builtins-elem", + "#builtin-elemAt": "language/builtins.html#builtins-elemAt", + "#builtin-fetchGit": "language/builtins.html#builtins-fetchGit", + "#builtin-fetchTarball": "language/builtins.html#builtins-fetchTarball", + "#builtin-fetchurl": "language/builtins.html#builtins-fetchurl", + "#builtin-filterSource": "language/builtins.html#builtins-filterSource", + "#builtin-foldl-prime": "language/builtins.html#builtins-foldl-prime", + "#builtin-fromJSON": "language/builtins.html#builtins-fromJSON", + "#builtin-functionArgs": "language/builtins.html#builtins-functionArgs", + "#builtin-genList": "language/builtins.html#builtins-genList", + "#builtin-getAttr": "language/builtins.html#builtins-getAttr", + "#builtin-getEnv": "language/builtins.html#builtins-getEnv", + "#builtin-hasAttr": "language/builtins.html#builtins-hasAttr", + "#builtin-hashFile": "language/builtins.html#builtins-hashFile", + "#builtin-hashString": "language/builtins.html#builtins-hashString", + "#builtin-head": "language/builtins.html#builtins-head", + "#builtin-import": "language/builtins.html#builtins-import", + "#builtin-intersectAttrs": "language/builtins.html#builtins-intersectAttrs", + "#builtin-isAttrs": "language/builtins.html#builtins-isAttrs", + "#builtin-isBool": "language/builtins.html#builtins-isBool", + "#builtin-isFloat": "language/builtins.html#builtins-isFloat", + "#builtin-isFunction": "language/builtins.html#builtins-isFunction", + "#builtin-isInt": "language/builtins.html#builtins-isInt", + "#builtin-isList": "language/builtins.html#builtins-isList", + "#builtin-isNull": "language/builtins.html#builtins-isNull", + "#builtin-isString": "language/builtins.html#builtins-isString", + "#builtin-length": "language/builtins.html#builtins-length", + "#builtin-lessThan": "language/builtins.html#builtins-lessThan", + "#builtin-listToAttrs": "language/builtins.html#builtins-listToAttrs", + "#builtin-map": "language/builtins.html#builtins-map", + "#builtin-match": "language/builtins.html#builtins-match", + "#builtin-mul": "language/builtins.html#builtins-mul", + "#builtin-parseDrvName": "language/builtins.html#builtins-parseDrvName", + "#builtin-path": "language/builtins.html#builtins-path", + "#builtin-pathExists": "language/builtins.html#builtins-pathExists", + "#builtin-placeholder": "language/builtins.html#builtins-placeholder", + "#builtin-readDir": "language/builtins.html#builtins-readDir", + "#builtin-readFile": "language/builtins.html#builtins-readFile", + "#builtin-removeAttrs": "language/builtins.html#builtins-removeAttrs", + "#builtin-replaceStrings": "language/builtins.html#builtins-replaceStrings", + "#builtin-seq": "language/builtins.html#builtins-seq", + "#builtin-sort": "language/builtins.html#builtins-sort", + "#builtin-split": "language/builtins.html#builtins-split", + "#builtin-splitVersion": "language/builtins.html#builtins-splitVersion", + "#builtin-stringLength": "language/builtins.html#builtins-stringLength", + "#builtin-sub": "language/builtins.html#builtins-sub", + "#builtin-substring": "language/builtins.html#builtins-substring", + "#builtin-tail": "language/builtins.html#builtins-tail", + "#builtin-throw": "language/builtins.html#builtins-throw", + "#builtin-toFile": "language/builtins.html#builtins-toFile", + "#builtin-toJSON": "language/builtins.html#builtins-toJSON", + "#builtin-toPath": "language/builtins.html#builtins-toPath", + "#builtin-toString": "language/builtins.html#builtins-toString", + "#builtin-toXML": "language/builtins.html#builtins-toXML", + "#builtin-trace": "language/builtins.html#builtins-trace", + "#builtin-tryEval": "language/builtins.html#builtins-tryEval", + "#builtin-typeOf": "language/builtins.html#builtins-typeOf", + "#ssec-builtins": "language/builtins.html", + "#attr-system": "language/derivations.html#attr-system", + "#ssec-derivation": "language/derivations.html", + "#ch-expression-language": "language/index.html", + "#sec-constructs": "language/constructs.html", + "#sect-let-language": "language/constructs.html#let-language", + "#ss-functions": "language/constructs.html#functions", + "#sec-language-operators": "language/operators.html", + "#table-operators": "language/operators.html", + "#ssec-values": "language/values.html", "#gloss-closure": "glossary.html#gloss-closure", "#gloss-derivation": "glossary.html#gloss-derivation", "#gloss-deriver": "glossary.html#gloss-deriver", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c8cb72fc0..084c8f442 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,21 +26,14 @@ - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - [Serving a Nix store via S3](package-management/s3-substituter.md) -- [Writing Nix Expressions](expressions/writing-nix-expressions.md) - - [A Simple Nix Expression](expressions/simple-expression.md) - - [Expression Syntax](expressions/expression-syntax.md) - - [Build Script](expressions/build-script.md) - - [Arguments and Variables](expressions/arguments-variables.md) - - [Building and Testing](expressions/simple-building-testing.md) - - [Generic Builder Syntax](expressions/generic-builder.md) - - [Nix Expression Language](expressions/expression-language.md) - - [Data Types](expressions/language-values.md) - - [Language Constructs](expressions/language-constructs.md) - - [Operators](expressions/language-operators.md) - - [Derivations](expressions/derivations.md) - - [Advanced Attributes](expressions/advanced-attributes.md) - - [Built-in Constants](expressions/builtin-constants.md) - - [Built-in Functions](expressions/builtins.md) +- [Nix Language](language/index.md) + - [Data Types](language/values.md) + - [Language Constructs](language/constructs.md) + - [Operators](language/operators.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Built-in Constants](language/builtin-constants.md) + - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index a372c5eae..a5df35d77 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -198,7 +198,7 @@ a number of possible ways: another. - If `--from-expression` is given, *args* are Nix - [functions](../expressions/language-constructs.md#functions) + [functions](../language/constructs.md#functions) that are called with the active Nix expression as their single argument. The derivations returned by those function calls are installed. This allows derivations to be specified in an diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 2e198daed..8f143729e 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -51,7 +51,7 @@ standard input. - `--strict`\ When used with `--eval`, recursively evaluate list elements and attributes. Normally, such sub-expressions are left unevaluated - (since the Nix expression language is lazy). + (since the Nix language is lazy). > **Warning** > @@ -66,7 +66,7 @@ standard input. When used with `--eval`, print the resulting value as an XML representation of the abstract syntax tree rather than as an ATerm. The schema is the same as that used by the [`toXML` - built-in](../expressions/builtins.md). + built-in](../language/builtins.md). - `--read-write-mode`\ When used with `--eval`, perform evaluation in read/write mode so diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index dc8faba68..ecd838e8d 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -121,7 +121,7 @@ Special exit codes: - `102`\ Hash mismatch, the build output was rejected because it does not match the [`outputHash` attribute of the - derivation](../expressions/advanced-attributes.md). + derivation](../language/advanced-attributes.md). - `104`\ Not deterministic, the build succeeded in check mode but the diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 51d7de18a..e612c416f 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -145,7 +145,7 @@ Most Nix commands accept the following command-line options: expression evaluator will automatically try to call functions that it encounters. It can automatically call functions for which every argument has a [default - value](../expressions/language-constructs.md#functions) (e.g., + value](../language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a @@ -164,7 +164,7 @@ Most Nix commands accept the following command-line options: So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](../expressions/builtins.md) for + value [`builtins.currentSystem`](../language/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the diff --git a/doc/manual/src/expressions/arguments-variables.md b/doc/manual/src/expressions/arguments-variables.md deleted file mode 100644 index 12198c879..000000000 --- a/doc/manual/src/expressions/arguments-variables.md +++ /dev/null @@ -1,80 +0,0 @@ -# Arguments and Variables - -The [Nix expression for GNU Hello](expression-syntax.md) is a -function; it is missing some arguments that have to be filled in -somewhere. In the Nix Packages collection this is done in the file -`pkgs/top-level/all-packages.nix`, where all Nix expressions for -packages are imported and called with the appropriate arguments. Here -are some fragments of `all-packages.nix`, with annotations of what -they mean: - -```nix -... - -rec { ① - - hello = import ../applications/misc/hello/ex-1 ② { ③ - inherit fetchurl stdenv perl; - }; - - perl = import ../development/interpreters/perl { ④ - inherit fetchurl stdenv; - }; - - fetchurl = import ../build-support/fetchurl { - inherit stdenv; ... - }; - - stdenv = ...; - -} -``` - -1. This file defines a set of attributes, all of which are concrete - derivations (i.e., not functions). In fact, we define a *mutually - recursive* set of attributes. That is, the attributes can refer to - each other. This is precisely what we want since we want to “plug” - the various packages into each other. - -2. Here we *import* the Nix expression for GNU Hello. The import - operation just loads and returns the specified Nix expression. In - fact, we could just have put the contents of the Nix expression - for GNU Hello in `all-packages.nix` at this point. That would be - completely equivalent, but it would make `all-packages.nix` rather - bulky. - - Note that we refer to `../applications/misc/hello/ex-1`, not - `../applications/misc/hello/ex-1/default.nix`. When you try to - import a directory, Nix automatically appends `/default.nix` to the - file name. - -3. This is where the actual composition takes place. Here we *call* the - function imported from `../applications/misc/hello/ex-1` with a set - containing the things that the function expects, namely `fetchurl`, - `stdenv`, and `perl`. We use inherit again to use the attributes - defined in the surrounding scope (we could also have written - `fetchurl = fetchurl;`, etc.). - - The result of this function call is an actual derivation that can be - built by Nix (since when we fill in the arguments of the function, - what we get is its body, which is the call to `stdenv.mkDerivation` - in the [Nix expression for GNU Hello](expression-syntax.md)). - - > **Note** - > - > Nixpkgs has a convenience function `callPackage` that imports and - > calls a function, filling in any missing arguments by passing the - > corresponding attribute from the Nixpkgs set, like this: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { }; - > ``` - > - > If necessary, you can set or override arguments: - > - > ```nix - > hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; }; - > ``` - -4. Likewise, we have to instantiate Perl, `fetchurl`, and the standard - environment. diff --git a/doc/manual/src/expressions/build-script.md b/doc/manual/src/expressions/build-script.md deleted file mode 100644 index b1eacae88..000000000 --- a/doc/manual/src/expressions/build-script.md +++ /dev/null @@ -1,70 +0,0 @@ -# Build Script - -Here is the builder referenced from Hello's Nix expression (stored in -`pkgs/applications/misc/hello/ex-1/builder.sh`): - -```bash -source $stdenv/setup ① - -PATH=$perl/bin:$PATH ② - -tar xvfz $src ③ -cd hello-* -./configure --prefix=$out ④ -make ⑤ -make install -``` - -The builder can actually be made a lot shorter by using the *generic -builder* functions provided by `stdenv`, but here we write out the build -steps to elucidate what a builder does. It performs the following steps: - -1. When Nix runs a builder, it initially completely clears the - environment (except for the attributes declared in the derivation). - This is done to prevent undeclared inputs from being used in the - build process. If for example the `PATH` contained `/usr/bin`, then - you might accidentally use `/usr/bin/gcc`. - - So the first step is to set up the environment. This is done by - calling the `setup` script of the standard environment. The - environment variable `stdenv` points to the location of the - standard environment being used. (It wasn't specified explicitly - as an attribute in Hello's Nix expression, but `mkDerivation` adds - it automatically.) - -2. Since Hello needs Perl, we have to make sure that Perl is in the - `PATH`. The `perl` environment variable points to the location of - the Perl package (since it was passed in as an attribute to the - derivation), so `$perl/bin` is the directory containing the Perl - interpreter. - -3. Now we have to unpack the sources. The `src` attribute was bound to - the result of fetching the Hello source tarball from the network, so - the `src` environment variable points to the location in the Nix - store to which the tarball was downloaded. After unpacking, we `cd` - to the resulting source directory. - - The whole build is performed in a temporary directory created in - `/tmp`, by the way. This directory is removed after the builder - finishes, so there is no need to clean up the sources afterwards. - Also, the temporary directory is always newly created, so you don't - have to worry about files from previous builds interfering with the - current build. - -4. GNU Hello is a typical Autoconf-based package, so we first have to - run its `configure` script. In Nix every package is stored in a - separate location in the Nix store, for instance - `/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1`. Nix - computes this path by cryptographically hashing all attributes of - the derivation. The path is passed to the builder through the `out` - environment variable. So here we give `configure` the parameter - `--prefix=$out` to cause Hello to be installed in the expected - location. - -5. Finally we build Hello (`make`) and install it into the location - specified by `out` (`make install`). - -If you are wondering about the absence of error checking on the result -of various commands called in the builder: this is because the shell -script is evaluated with Bash's `-e` option, which causes the script to -be aborted if any command fails without an error check. diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md deleted file mode 100644 index 6b93e692c..000000000 --- a/doc/manual/src/expressions/expression-syntax.md +++ /dev/null @@ -1,93 +0,0 @@ -# Expression Syntax - -Here is a Nix expression for GNU Hello: - -```nix -{ stdenv, fetchurl, perl }: ① - -stdenv.mkDerivation { ② - name = "hello-2.1.1"; ③ - builder = ./builder.sh; ④ - src = fetchurl { ⑤ - url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; - sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; - }; - inherit perl; ⑥ -} -``` - -This file is actually already in the Nix Packages collection in -`pkgs/applications/misc/hello/ex-1/default.nix`. It is customary to -place each package in a separate directory and call the single Nix -expression in that directory `default.nix`. The file has the following -elements (referenced from the figure by number): - -1. This states that the expression is a *function* that expects to be - called with three arguments: `stdenv`, `fetchurl`, and `perl`. They - are needed to build Hello, but we don't know how to build them here; - that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages; it provides a - “standard” environment consisting of the things you would expect - in a basic Unix environment: a C/C++ compiler (GCC, to be precise), - the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, - etc. `fetchurl` is a function that downloads files. `perl` is the - Perl interpreter. - - Nix functions generally have the form `{ x, y, ..., z }: e` where - `x`, `y`, etc. are the names of the expected arguments, and where - *e* is the body of the function. So here, the entire remainder of - the file is the body of the function; when given the required - arguments, the body should describe how to build an instance of - the Hello package. - -2. So we have to build a package. Building something from other stuff - is called a *derivation* in Nix (as opposed to sources, which are - built by humans instead of computers). We perform a derivation by - calling `stdenv.mkDerivation`. `mkDerivation` is a function - provided by `stdenv` that builds a package from a set of - *attributes*. A set is just a list of key/value pairs where each - key is a string and each value is an arbitrary Nix - expression. They take the general form `{ name1 = expr1; ... - nameN = exprN; }`. - -3. The attribute `name` specifies the symbolic name and version of - the package. Nix doesn't really care about these things, but they - are used by for instance `nix-env -q` to show a “human-readable” - name for packages. This attribute is required by `mkDerivation`. - -4. The attribute `builder` specifies the builder. This attribute can - sometimes be omitted, in which case `mkDerivation` will fill in a - default builder (which does a `configure; make; make install`, in - essence). Hello is sufficiently simple that the default builder - would suffice, but in this case, we will show an actual builder - for educational purposes. The value `./builder.sh` refers to the - shell script shown in the [next section](build-script.md), - discussed below. - -5. The builder has to know what the sources of the package are. Here, - the attribute `src` is bound to the result of a call to the - `fetchurl` function. Given a URL and a SHA-256 hash of the expected - contents of the file at that URL, this function builds a derivation - that downloads the file and checks its hash. So the sources are a - dependency that like all other dependencies is built before Hello - itself is built. - - Instead of `src` any other name could have been used, and in fact - there can be any number of sources (bound to different attributes). - However, `src` is customary, and it's also expected by the default - builder (which we don't use in this example). - -6. Since the derivation requires Perl, we have to pass the value of the - `perl` function argument to the builder. All attributes in the set - are actually passed as environment variables to the builder, so - declaring an attribute - - ```nix - perl = perl; - ``` - - will do the trick: it binds an attribute `perl` to the function - argument which also happens to be called `perl`. However, it looks a - bit silly, so there is a shorter syntax. The `inherit` keyword - causes the specified attributes to be bound to whatever variables - with the same name happen to be in scope. diff --git a/doc/manual/src/expressions/generic-builder.md b/doc/manual/src/expressions/generic-builder.md deleted file mode 100644 index cf26b5f82..000000000 --- a/doc/manual/src/expressions/generic-builder.md +++ /dev/null @@ -1,66 +0,0 @@ -# Generic Builder Syntax - -Recall that the [build script for GNU Hello](build-script.md) looked -something like this: - -```bash -PATH=$perl/bin:$PATH -tar xvfz $src -cd hello-* -./configure --prefix=$out -make -make install -``` - -The builders for almost all Unix packages look like this — set up some -environment variables, unpack the sources, configure, build, and -install. For this reason the standard environment provides some Bash -functions that automate the build process. Here is what a builder using -the generic build facilities looks like: - -```bash -buildInputs="$perl" ① - -source $stdenv/setup ② - -genericBuild ③ -``` - -Here is what each line means: - -1. The `buildInputs` variable tells `setup` to use the indicated - packages as “inputs”. This means that if a package provides a `bin` - subdirectory, it's added to `PATH`; if it has a `include` - subdirectory, it's added to GCC's header search path; and so on. - (This is implemented in a modular way: `setup` tries to source the - file `pkg/nix-support/setup-hook` of all dependencies. These “setup - hooks” can then set up whatever environment variables they want; for - instance, the setup hook for Perl sets the `PERL5LIB` environment - variable to contain the `lib/site_perl` directories of all inputs.) - -2. The function `genericBuild` is defined in the file `$stdenv/setup`. - -3. The final step calls the shell function `genericBuild`, which - performs the steps that were done explicitly in the previous build - script. The generic builder is smart enough to figure out whether - to unpack the sources using `gzip`, `bzip2`, etc. It can be - customised in many ways; see the Nixpkgs manual for details. - -Discerning readers will note that the `buildInputs` could just as well -have been set in the Nix expression, like this: - -```nix - buildInputs = [ perl ]; -``` - -The `perl` attribute can then be removed, and the builder becomes even -shorter: - -```bash -source $stdenv/setup -genericBuild -``` - -In fact, `mkDerivation` provides a default builder that looks exactly -like that, so it is actually possible to omit the builder for Hello -entirely. diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md deleted file mode 100644 index 7f0d8f841..000000000 --- a/doc/manual/src/expressions/simple-building-testing.md +++ /dev/null @@ -1,61 +0,0 @@ -# Building and Testing - -You can now try to build Hello. Of course, you could do `nix-env -f . -iA -hello`, but you may not want to install a possibly broken package just -yet. The best way to test the package is by using the command -`nix-build`, which builds a Nix expression and creates a symlink named -`result` in the current directory: - -```console -$ nix-build -A hello -building path `/nix/store/632d2b22514d...-hello-2.1.1' -hello-2.1.1/ -hello-2.1.1/intl/ -hello-2.1.1/intl/ChangeLog -... - -$ ls -l result -lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1 - -$ ./result/bin/hello -Hello, world! -``` - -The `-A` option selects the `hello` attribute. This is faster than -using the symbolic package name specified by the `name` attribute -(which also happens to be `hello`) and is unambiguous (there can be -multiple packages with the symbolic name `hello`, but there can be -only one attribute in a set named `hello`). - -`nix-build` registers the `./result` symlink as a garbage collection -root, so unless and until you delete the `./result` symlink, the output -of the build will be safely kept on your system. You can use -`nix-build`’s `-o` switch to give the symlink another name. - -Nix has transactional semantics. Once a build finishes successfully, Nix -makes a note of this in its database: it registers that the path denoted -by `out` is now “valid”. If you try to build the derivation again, Nix -will see that the path is already valid and finish immediately. If a -build fails, either because it returns a non-zero exit code, because Nix -or the builder are killed, or because the machine crashes, then the -output paths will not be registered as valid. If you try to build the -derivation again, Nix will remove the output paths if they exist (e.g., -because the builder died half-way through `make -install`) and try again. Note that there is no “negative caching”: Nix -doesn't remember that a build failed, and so a failed build can always -be repeated. This is because Nix cannot distinguish between permanent -failures (e.g., a compiler error due to a syntax error in the source) -and transient failures (e.g., a disk full condition). - -Nix also performs locking. If you run multiple Nix builds -simultaneously, and they try to build the same derivation, the first Nix -instance that gets there will perform the build, while the others block -(or perform other derivations if available) until the build finishes: - -```console -$ nix-build -A hello -waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x' -``` - -So it is always safe to run multiple instances of Nix in parallel (which -isn’t the case with, say, `make`). diff --git a/doc/manual/src/expressions/simple-expression.md b/doc/manual/src/expressions/simple-expression.md deleted file mode 100644 index 857f71b9b..000000000 --- a/doc/manual/src/expressions/simple-expression.md +++ /dev/null @@ -1,23 +0,0 @@ -# A Simple Nix Expression - -This section shows how to add and test the [GNU Hello -package](http://www.gnu.org/software/hello/hello.html) to the Nix -Packages collection. Hello is a program that prints out the text “Hello, -world\!”. - -To add a package to the Nix Packages collection, you generally need to -do three things: - -1. Write a Nix expression for the package. This is a file that - describes all the inputs involved in building the package, such as - dependencies, sources, and so on. - -2. Write a *builder*. This is a shell script that builds the package - from the inputs. (In fact, it can be written in any language, but - typically it's a `bash` shell script.) - -3. Add the package to the file `pkgs/top-level/all-packages.nix`. The - Nix expression written in the first step is a *function*; it - requires other packages in order to build it. In this step you put - it all together, i.e., you call the function with the right - arguments to build the actual package. diff --git a/doc/manual/src/expressions/writing-nix-expressions.md b/doc/manual/src/expressions/writing-nix-expressions.md deleted file mode 100644 index 5664108e7..000000000 --- a/doc/manual/src/expressions/writing-nix-expressions.md +++ /dev/null @@ -1,12 +0,0 @@ -This chapter shows you how to write Nix expressions, which instruct Nix -how to build packages. It starts with a simple example (a Nix expression -for GNU Hello), and then moves on to a more in-depth look at the Nix -expression language. - -> **Note** -> -> This chapter is mostly about the Nix expression language. For more -> extensive information on adding packages to the Nix Packages -> collection (such as functions in the standard environment and coding -> conventions), please consult [its -> manual](http://nixos.org/nixpkgs/manual/). diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 3448b971b..aa0ac78cb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -3,7 +3,7 @@ - [derivation]{#gloss-derivation}\ A description of a build action. The result of a derivation is a store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](expressions/derivations.md). These are + using the [`derivation` primitive](language/derivations.md). These are translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md similarity index 100% rename from doc/manual/src/expressions/advanced-attributes.md rename to doc/manual/src/language/advanced-attributes.md diff --git a/doc/manual/src/expressions/builtin-constants.md b/doc/manual/src/language/builtin-constants.md similarity index 100% rename from doc/manual/src/expressions/builtin-constants.md rename to doc/manual/src/language/builtin-constants.md diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md similarity index 100% rename from doc/manual/src/expressions/builtins-prefix.md rename to doc/manual/src/language/builtins-prefix.md diff --git a/doc/manual/src/expressions/builtins-suffix.md b/doc/manual/src/language/builtins-suffix.md similarity index 100% rename from doc/manual/src/expressions/builtins-suffix.md rename to doc/manual/src/language/builtins-suffix.md diff --git a/doc/manual/src/expressions/language-constructs.md b/doc/manual/src/language/constructs.md similarity index 100% rename from doc/manual/src/expressions/language-constructs.md rename to doc/manual/src/language/constructs.md diff --git a/doc/manual/src/expressions/derivations.md b/doc/manual/src/language/derivations.md similarity index 100% rename from doc/manual/src/expressions/derivations.md rename to doc/manual/src/language/derivations.md diff --git a/doc/manual/src/expressions/expression-language.md b/doc/manual/src/language/index.md similarity index 84% rename from doc/manual/src/expressions/expression-language.md rename to doc/manual/src/language/index.md index 267fcb983..c4b3abf75 100644 --- a/doc/manual/src/expressions/expression-language.md +++ b/doc/manual/src/language/index.md @@ -1,6 +1,6 @@ -# Nix Expression Language +# Nix Language -The Nix expression language is a pure, lazy, functional language. Purity +The Nix language is a pure, lazy, functional language. Purity means that operations in the language don't have side-effects (for instance, there is no variable assignment). Laziness means that arguments to functions are evaluated only when they are needed. @@ -10,3 +10,4 @@ full-featured, general purpose language. Its main job is to describe packages, compositions of packages, and the variability within packages. This section presents the various features of the language. + diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/language/operators.md similarity index 99% rename from doc/manual/src/expressions/language-operators.md rename to doc/manual/src/language/operators.md index 268b44f4c..32398189d 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/language/operators.md @@ -1,6 +1,6 @@ # Operators -The table below lists the operators in the Nix expression language, in +The table below lists the operators in the Nix language, in order of precedence (from strongest to weakest binding). | Name | Syntax | Associativity | Description | Precedence | diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/language/values.md similarity index 100% rename from doc/manual/src/expressions/language-values.md rename to doc/manual/src/language/values.md diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/package-management.md index bd26a09ab..d528112e2 100644 --- a/doc/manual/src/package-management/package-management.md +++ b/doc/manual/src/package-management/package-management.md @@ -1,5 +1,4 @@ This chapter discusses how to do package management with Nix, i.e., how to obtain, install, upgrade, and erase packages. This is the “user’s” perspective of the Nix system — people who want to *create* -packages should consult the [chapter on writing Nix -expressions](../expressions/writing-nix-expressions.md). +packages should consult the chapter on the [Nix language](../language/index.md). From 523359d133ea8f048f69d8e8b3e5f1b674baf17a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 08:38:12 -0500 Subject: [PATCH 061/371] WIP: Document the design of Nix The current docs are all "how to do things" and no "what is Nix" or "why are things the way they are". I see lots of misconception on the wider internet, and I also think we would benefit from a "living document" to answer some questions people currently turn to the thesis for. I think a new section of the manual can address all these issues. --- doc/manual/local.mk | 6 +++- doc/manual/src/SUMMARY.md.in | 5 +++ doc/manual/src/design/design.md | 5 +++ doc/manual/src/design/overview.md | 13 ++++++++ doc/manual/src/design/store/entries.md | 46 ++++++++++++++++++++++++++ doc/manual/src/design/store/paths.md | 30 +++++++++++++++++ doc/manual/src/design/store/store.md | 5 +++ 7 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 doc/manual/src/design/design.md create mode 100644 doc/manual/src/design/overview.md create mode 100644 doc/manual/src/design/store/entries.md create mode 100644 doc/manual/src/design/store/paths.md create mode 100644 doc/manual/src/design/store/store.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 371ed6f21..a26fb8fcf 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,9 @@ ifeq ($(doc_generate),yes) +MANUAL_SRCS := \ + $(call rwildcard, $(d)/src, *.md) \ + $(call rwildcard, $(d)/src, */*.md) + # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ @@ -97,7 +101,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c8cb72fc0..c320bde3f 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,6 +15,11 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) +- [Design and Data Model](design/design.md) + - [Overview](design/overview.md) + - [The Store Layer](design/store/store.md) + - [Store Entries](design/store/entries.md) + - [Store Paths](design/store/paths.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md new file mode 100644 index 000000000..7d4211764 --- /dev/null +++ b/doc/manual/src/design/design.md @@ -0,0 +1,5 @@ +# Design and Data Model + +Most of the manual is about how to use Nix. +This chapter is about what Nix actually is. +The hope is that it can serve as a reference for key concepts, and also shed light on why things are the way they are. diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md new file mode 100644 index 000000000..05f4a70b2 --- /dev/null +++ b/doc/manual/src/design/overview.md @@ -0,0 +1,13 @@ +# Overview + +Nix is broken into layers that operate fairly independently. + +At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. + +Below that is the Nix *expression language*, in which packages and configurations are written. +These are the layers which users interact with most. + +Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. +The store layer may not be as visible, but this is the heart of Nix. + +This chapter will start there and work up towards the more user-facing interfaces described in the rest of the manual. diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md new file mode 100644 index 000000000..c16c98a36 --- /dev/null +++ b/doc/manual/src/design/store/entries.md @@ -0,0 +1,46 @@ +# Store Entries + +File system data in Nix is organized into *store entries*. +A store entry is the combination of + + - some file system data + - references to store entries + +## File system data + +Nix supports the a similar model of the file system as Git. +Namely, every file system object falls into these three cases: + + - File: arbitrary data + + - Directory: mapping of names to child file system objects. + File children additionally have an executable flag. + + - Symlink: may point anywhere. + In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. + +A bare file as the "root" file system object is allowed. +Note that there is no containing directory object to store its executable bit; it's deemed non-executable by default. + +## References + +Store entries can refer to both other store entries and themselves. + +Store references are normally calculated by scanning the file system data for store paths when a new store entry is created, but this isn't mandatory, as store entries are allowed to have references that don't correspond to contained store paths, and contained store paths that don't correspond to references. + +The references themselves need not be store paths per-se (this is an implementation detail of the store). +But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. +That said, all the references of the store entry must agree on a store dir. +Also the store directory of the references must equal that of any store which contains the store entry doing the referencing. + +## Relocatability + +The two final restrictions of the previous section yield an alternative of view of the same information. +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least once reference. + +This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. + +Lastly, this illustrates the purpose of tracking self references. +Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. +\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md new file mode 100644 index 000000000..1e4369625 --- /dev/null +++ b/doc/manual/src/design/store/paths.md @@ -0,0 +1,30 @@ +# Store Paths + +A store path is a pair of a 20-byte digest and a name. + +Historically it is the triple of those two and also the store directory, but the modern implementation's internal representation is just the pair. +This change is because in the vast majority of cases, the store dir is fully determined by the context in which the store path occurs. + +## String representation + +A store path is rendered as the concatenation of + + - the store directory + + - a path-separator (`/`) + + - the digest rendered as Base-32 (20 bytes becomes 32 bytes) + + - a hyphen (`-`) + + - the name + +Let's take the store path from the very beginning of this manual as an example: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + +This parses like so: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + store dir digest name diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md new file mode 100644 index 000000000..c2431f1c5 --- /dev/null +++ b/doc/manual/src/design/store/store.md @@ -0,0 +1,5 @@ +A Nix store is a collection of *store entries* referred to by *store paths*. +Every store also has a "store directory path", which is a path prefix used for various purposes. + +There are many types of stores, but all of them at least respect this model. +Some however offer additional functionality. From a2b3160f2898c2009d458cc27d5e8ea925cf583b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 09:09:42 -0500 Subject: [PATCH 062/371] Briefly describe the digest of a store path --- doc/manual/src/design/store/paths.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 1e4369625..18ba27ecb 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -28,3 +28,24 @@ This parses like so: /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ store dir digest name + +## The digest + +The calculation of the digest is quite complicated for historical reasons. +Some of the details will be saved for later. + +> Historical note. The 20 byte restriction is because originally a digests were SHA-1 hashes. +> This is no longer true, but longer hashes and other information is still boiled down to 20 bytes. + +Store paths are either content-addressed or "input-addressed". + +Content addressing means that the digest ultimately derives from referred store entry's file system data and references, and thus can be verified (if one knows how it was calculated). + +Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. +Store paths of this sort can not be validated from the content of the store entry. +Rather, the store entry might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. +The signature indicates that someone is vouching for the store entry really being the results of a plan with that digest. + +While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. +That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. +In the parlance of referencing schemes, this means that store paths are not "self-describing". From e64633f98f290cb571df72a667a0a267722fc397 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 10:18:39 -0500 Subject: [PATCH 063/371] Flesh out TOC --- doc/manual/src/SUMMARY.md.in | 6 ++++++ doc/manual/src/design/store/building.md | 0 doc/manual/src/design/store/drvs/ca.md | 0 doc/manual/src/design/store/drvs/drvs.md | 0 doc/manual/src/design/store/drvs/ia.md | 0 doc/manual/src/design/store/input-addressing.md | 1 + doc/manual/src/design/store/nar.md | 1 + doc/manual/src/design/store/netry-ca.md | 1 + 8 files changed, 9 insertions(+) create mode 100644 doc/manual/src/design/store/building.md create mode 100644 doc/manual/src/design/store/drvs/ca.md create mode 100644 doc/manual/src/design/store/drvs/drvs.md create mode 100644 doc/manual/src/design/store/drvs/ia.md create mode 100644 doc/manual/src/design/store/input-addressing.md create mode 100644 doc/manual/src/design/store/nar.md create mode 100644 doc/manual/src/design/store/netry-ca.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c320bde3f..8c4dbcb18 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,6 +20,12 @@ - [The Store Layer](design/store/store.md) - [Store Entries](design/store/entries.md) - [Store Paths](design/store/paths.md) + - [Nix Archives](design/store/nar.md) + - [Content-Addressing Store Entries](design/store/entry-ca.md) + - [Derivations](design/store/drvs/drvs.md) + - [Input-Addressing](design/store/drvs/ia.md) + - [Content-Addressing (Experimental)](design/store/drvs/ca.md) + - [Building](design/store/building.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/ca.md b/doc/manual/src/design/store/drvs/ca.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/ia.md b/doc/manual/src/design/store/drvs/ia.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/input-addressing.md b/doc/manual/src/design/store/input-addressing.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/input-addressing.md @@ -0,0 +1 @@ +TODO diff --git a/doc/manual/src/design/store/nar.md b/doc/manual/src/design/store/nar.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/nar.md @@ -0,0 +1 @@ +TODO diff --git a/doc/manual/src/design/store/netry-ca.md b/doc/manual/src/design/store/netry-ca.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/netry-ca.md @@ -0,0 +1 @@ +TODO From a210504bc7b67defe16fda97a6d2a844ecc9bd2b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 11:41:15 -0400 Subject: [PATCH 064/371] Apply suggestions from code review --- doc/manual/src/design/overview.md | 2 +- doc/manual/src/design/store/entries.md | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 05f4a70b2..792cfefbf 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -10,4 +10,4 @@ These are the layers which users interact with most. Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. The store layer may not be as visible, but this is the heart of Nix. -This chapter will start there and work up towards the more user-facing interfaces described in the rest of the manual. +This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index c16c98a36..2eb4bf90c 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -8,8 +8,8 @@ A store entry is the combination of ## File system data -Nix supports the a similar model of the file system as Git. -Namely, every file system object falls into these three cases: +The nix store uses a simple filesystem model, similar to the one Git uses. +In particular, every file system object falls into these three cases: - File: arbitrary data @@ -20,13 +20,25 @@ Namely, every file system object falls into these three cases: In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. -Note that there is no containing directory object to store its executable bit; it's deemed non-executable by default. +Note that it cannot be executable, though. +This is a consequence of the executable flags being part of the child entries of the directory, rather than the child files themselves. +A root file has no parent directory; so there is no child entry about the root file, and therefore no executable flag for it. +Without a flag saying which, whether root files are executable or non-executable by default must be decided by convention, and the choice of Nix (and git) is to make them non-executable. ## References Store entries can refer to both other store entries and themselves. -Store references are normally calculated by scanning the file system data for store paths when a new store entry is created, but this isn't mandatory, as store entries are allowed to have references that don't correspond to contained store paths, and contained store paths that don't correspond to references. +References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store entries. +For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store entries by searching in all their files. +When we get to building in a future section, this process will be described in precise detail. + +However, scanning for references is not mandatory. +Store entries are allowed to have official references that *don't* correspond to store paths contained in their contents, +and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. +Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. + +This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. The references themselves need not be store paths per-se (this is an implementation detail of the store). But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. @@ -35,8 +47,8 @@ Also the store directory of the references must equal that of any store which co ## Relocatability -The two final restrictions of the previous section yield an alternative of view of the same information. -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least once reference. +The two final restrictions of the previous section yield an alternative view of the same information. +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. From e3a0209a9e279a30ea9381743c77b2737e8ed95e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 12:35:01 -0400 Subject: [PATCH 065/371] Move the bits on relocating store entires to the end They are too advanced for up front. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/design/store/building.md | 15 +++++++++++++++ doc/manual/src/design/store/entries.md | 17 ----------------- doc/manual/src/design/store/relocatability.md | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 doc/manual/src/design/store/relocatability.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8c4dbcb18..1e8b35483 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,6 +26,7 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) + - [Advanced Topic: store dir relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index e69de29bb..5adccbc0b 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -0,0 +1,15 @@ +# Building + +## Scanning for references + +Before in the section on [store entries](../entries.md), we talked abstractly about scanning for references. +Now we can make this concrete. + +After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store entries. +There is a few steps of this, but let's start with the simple case of one input-addressed output first. + +\[Overview of things that need to happen.] + +For example, if Nix thinks `/nix/store/asdfasdfasdf-foo` and `/nix/store/qwerqwerqwer-bar` are paths the data might plausibly reference, Nix will scan all the contents of all files recursively for the "hash parts" `asdfasdfasdf`` and `qwerqwerqwer`. + +\[Explain why whitelist.] diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index 2eb4bf90c..74f429e78 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -39,20 +39,3 @@ and they are also allowed to *not* have references that *do* correspond to store Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. - -The references themselves need not be store paths per-se (this is an implementation detail of the store). -But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. -That said, all the references of the store entry must agree on a store dir. -Also the store directory of the references must equal that of any store which contains the store entry doing the referencing. - -## Relocatability - -The two final restrictions of the previous section yield an alternative view of the same information. -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. - -This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. - -Lastly, this illustrates the purpose of tracking self references. -Store entries without self-references or other references are relocatable, while store paths with self-references aren't. -This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. -\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md new file mode 100644 index 000000000..04b581459 --- /dev/null +++ b/doc/manual/src/design/store/relocatability.md @@ -0,0 +1,15 @@ +## Advanced Topic: store entry relocatability + +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store entry to a store with a different mount point. + +Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. + +~~The two final restrictions of the previous section yield an alternative view of the same information.~~ +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. + +This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. + +Lastly, this illustrates the purpose of tracking self references. +Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. +\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] From 678d75baeaca049042330e543f610d630e8c43b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 13:13:55 -0400 Subject: [PATCH 066/371] Start on the derivations section --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/design/store/drvs/drvs.md | 59 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1e8b35483..a0efdb9cd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,7 +26,7 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store dir relocatability](design/store/relocatability.md) + - [Advanced Topic: store entry relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index e69de29bb..cedb05985 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -0,0 +1,59 @@ +# Derivations + +Derivations are recipes to create store entries. + +Derivations are the heart of Nix. +Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. +This is where Nix comes in. + +Derivations produce data by running arbitrary commands, like Make or Ninja rules. +Unlike those systems, derivations do not produce arbitrary files, but only specific store entries. +They cannot modify the store in any way, other than creating those store entries. +This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. + +Based on the above, we can conceptually break derivations down into 3 parts: + +1. What command will be run? + +2. What existing store entries are needed as inputs? + +3. What store entries will be produced as outputs? + +## What command will be run? + +The original core of Nix was very simple about this, in the mold of traditional Unix. +Commands consist of 3 parts: + +1. Path to executable + +2. Arguments (Excecpt for `argv[0]`, which is taken from the path in the usual way) + +3. Environment variables. + +## What existing store entries are needed as inputs? + +The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" +It's a good questions! + +## What store entries will be produced as outputs? + +## Extra extensions + +### `__structuredAttrs` + +Historically speaking, most users of Nix made GNU Bash with a script the command run, regardless of what they were doing. +Bash variable are automatically created from env vars, but bash also supports array and string-keyed map variables in addition to string variables. +People also usually create derivations using language which also support these richer data types. +It was thus desired a way to get this data from the language "planning" the derivation to language to bash, the language evaluated at "run time". + +`__structuredAttrs` does this by smuggling inside the core derivation format a map of named richer data. +At run time, this becomes two things: + +1. A JSON file containing that map. +2. A bash script setting those variables. + +The bash command can be passed a script which will "source" that Nix-created bash script, setting those variables with the richer data. +The outer script can then do whatever it likes with those richer variables as input. + +However, since derivations can already contain arbitary input sources, the vast majority of `__structuredAttrs` can be handled by upper layers. +We might consider implementing `__structuredAttrs` in higher layers in the future, and simplifying the store layer. From f5386d7059ad2c63b32356082d1f6ecacfc1e93b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 13:15:35 -0400 Subject: [PATCH 067/371] Fix stub file's name --- doc/manual/src/design/store/{netry-ca.md => entry-ca.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/manual/src/design/store/{netry-ca.md => entry-ca.md} (100%) diff --git a/doc/manual/src/design/store/netry-ca.md b/doc/manual/src/design/store/entry-ca.md similarity index 100% rename from doc/manual/src/design/store/netry-ca.md rename to doc/manual/src/design/store/entry-ca.md From a04340f9a1ce3c107b21a11395b4a2348fd19571 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 10:55:30 -0400 Subject: [PATCH 068/371] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 792cfefbf..7790e701e 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -7,7 +7,8 @@ At the top is the *command line interface*, i.e. the argument parsing of the var Below that is the Nix *expression language*, in which packages and configurations are written. These are the layers which users interact with most. -Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. +Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. +It is also used for executing those build plans. The store layer may not be as visible, but this is the heart of Nix. This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." From 75c5191a1f8cf70a11098e89ed33edc203f52dab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 11:04:55 -0400 Subject: [PATCH 069/371] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 7790e701e..bd3d935e6 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -1,6 +1,6 @@ # Overview -Nix is broken into layers that operate fairly independently. +Nix consists of layers that operate fairly independently. At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. From cdb0bf3b65172fd0e366d44f17392b4261c3d925 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 11:15:56 -0400 Subject: [PATCH 070/371] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index bd3d935e6..2d3e9226b 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,7 +5,8 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. Below that is the Nix *expression language*, in which packages and configurations are written. -These are the layers which users interact with most. + +The command line and expression language are what users interact with most. Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. It is also used for executing those build plans. From e308602fdfabf0ff1f0cd95baf1d90ba54a02281 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Apr 2022 20:48:03 -0400 Subject: [PATCH 071/371] Update doc/manual/src/design/store/drvs/drvs.md Co-authored-by: Matthieu Coudron --- doc/manual/src/design/store/drvs/drvs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index cedb05985..98e95809b 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -26,7 +26,7 @@ Commands consist of 3 parts: 1. Path to executable -2. Arguments (Excecpt for `argv[0]`, which is taken from the path in the usual way) +2. Arguments (Except for `argv[0]`, which is taken from the path in the usual way) 3. Environment variables. From 4e2d5ae2027a19ab12d6d024cdf5125c3200a7a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 21:53:39 -0400 Subject: [PATCH 072/371] doc: Store entry -> store object This matches the terminology in Eelco's thesis. --- doc/manual/src/SUMMARY.md.in | 6 +++--- doc/manual/src/design/overview.md | 2 +- doc/manual/src/design/store/building.md | 4 ++-- doc/manual/src/design/store/drvs/drvs.md | 14 ++++++------- doc/manual/src/design/store/entries.md | 20 +++++++++---------- doc/manual/src/design/store/paths.md | 8 ++++---- doc/manual/src/design/store/relocatability.md | 10 +++++----- doc/manual/src/design/store/store.md | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a0efdb9cd..5c4cfc920 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,15 +18,15 @@ - [Design and Data Model](design/design.md) - [Overview](design/overview.md) - [The Store Layer](design/store/store.md) - - [Store Entries](design/store/entries.md) + - [Store Objects](design/store/entries.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Entries](design/store/entry-ca.md) + - [Content-Addressing Store Objects](design/store/entry-ca.md) - [Derivations](design/store/drvs/drvs.md) - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store entry relocatability](design/store/relocatability.md) + - [Advanced Topic: store object relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 2d3e9226b..9329f2ad7 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -2,7 +2,7 @@ Nix consists of layers that operate fairly independently. -At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. +At the top is the *command line interface*. Below that is the Nix *expression language*, in which packages and configurations are written. diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index 5adccbc0b..f97b74952 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -2,10 +2,10 @@ ## Scanning for references -Before in the section on [store entries](../entries.md), we talked abstractly about scanning for references. +Before in the section on [store objects](../entries.md), we talked abstractly about scanning for references. Now we can make this concrete. -After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store entries. +After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. There is a few steps of this, but let's start with the simple case of one input-addressed output first. \[Overview of things that need to happen.] diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index 98e95809b..766a7b47f 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -1,23 +1,23 @@ # Derivations -Derivations are recipes to create store entries. +Derivations are recipes to create store objects. Derivations are the heart of Nix. Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. This is where Nix comes in. Derivations produce data by running arbitrary commands, like Make or Ninja rules. -Unlike those systems, derivations do not produce arbitrary files, but only specific store entries. -They cannot modify the store in any way, other than creating those store entries. +Unlike those systems, derivations do not produce arbitrary files, but only specific store objects. +They cannot modify the store in any way, other than creating those store objects. This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. Based on the above, we can conceptually break derivations down into 3 parts: 1. What command will be run? -2. What existing store entries are needed as inputs? +2. What existing store objects are needed as inputs? -3. What store entries will be produced as outputs? +3. What store objects will be produced as outputs? ## What command will be run? @@ -30,12 +30,12 @@ Commands consist of 3 parts: 3. Environment variables. -## What existing store entries are needed as inputs? +## What existing store objects are needed as inputs? The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" It's a good questions! -## What store entries will be produced as outputs? +## What store objects will be produced as outputs? ## Extra extensions diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index 74f429e78..ddfefa166 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -1,10 +1,10 @@ -# Store Entries +# Store Objects -File system data in Nix is organized into *store entries*. -A store entry is the combination of +File system data in Nix is organized into *store objects*. +A store object is the combination of - some file system data - - references to store entries + - references to store objects ## File system data @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: File children additionally have an executable flag. - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. + In particular, Symlinks that do not point within the containing file system data or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. Note that it cannot be executable, though. @@ -27,15 +27,15 @@ Without a flag saying which, whether root files are executable or non-executable ## References -Store entries can refer to both other store entries and themselves. +Store objects can refer to both other store objects and themselves. -References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store entries. -For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store entries by searching in all their files. +References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store objects. +For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. However, scanning for references is not mandatory. -Store entries are allowed to have official references that *don't* correspond to store paths contained in their contents, +Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. -This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. +This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store object, rather than try to recompute them on the fly by scanning their contents. diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 18ba27ecb..d7d2ef960 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -39,12 +39,12 @@ Some of the details will be saved for later. Store paths are either content-addressed or "input-addressed". -Content addressing means that the digest ultimately derives from referred store entry's file system data and references, and thus can be verified (if one knows how it was calculated). +Content addressing means that the digest ultimately derives from referred store object's file system data and references, and thus can be verified (if one knows how it was calculated). Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. -Store paths of this sort can not be validated from the content of the store entry. -Rather, the store entry might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. -The signature indicates that someone is vouching for the store entry really being the results of a plan with that digest. +Store paths of this sort can not be validated from the content of the store object. +Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. +The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md index 04b581459..5cdbe9486 100644 --- a/doc/manual/src/design/store/relocatability.md +++ b/doc/manual/src/design/store/relocatability.md @@ -1,15 +1,15 @@ -## Advanced Topic: store entry relocatability +## Advanced Topic: store object relocatability -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store entry to a store with a different mount point. +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store object to a store with a different mount point. Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. ~~The two final restrictions of the previous section yield an alternative view of the same information.~~ -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. +Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. -This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. +This corresponds to the observation that a store object with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store object without any references, i.e. thus without a store directory, can exist in any store. Lastly, this illustrates the purpose of tracking self references. -Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +Store objects without self-references or other references are relocatable, while store paths with self-references aren't. This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. \[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index c2431f1c5..e91f0f6e4 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,4 +1,4 @@ -A Nix store is a collection of *store entries* referred to by *store paths*. +A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. There are many types of stores, but all of them at least respect this model. From 838ba26fda23649545cc61a274e20076a2e27892 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 21:57:10 -0400 Subject: [PATCH 073/371] Rename files after store entry -> store object rename --- doc/manual/src/SUMMARY.md.in | 4 ++-- doc/manual/src/design/store/building.md | 2 +- doc/manual/src/design/store/{entry-ca.md => object-ca.md} | 0 doc/manual/src/design/store/{entries.md => objects.md} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename doc/manual/src/design/store/{entry-ca.md => object-ca.md} (100%) rename doc/manual/src/design/store/{entries.md => objects.md} (100%) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5c4cfc920..323a569d1 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,10 +18,10 @@ - [Design and Data Model](design/design.md) - [Overview](design/overview.md) - [The Store Layer](design/store/store.md) - - [Store Objects](design/store/entries.md) + - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Objects](design/store/entry-ca.md) + - [Content-Addressing Store Objects](design/store/object-ca.md) - [Derivations](design/store/drvs/drvs.md) - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index f97b74952..f4f2649a3 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -2,7 +2,7 @@ ## Scanning for references -Before in the section on [store objects](../entries.md), we talked abstractly about scanning for references. +Before in the section on [store objects](../objects.md), we talked abstractly about scanning for references. Now we can make this concrete. After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. diff --git a/doc/manual/src/design/store/entry-ca.md b/doc/manual/src/design/store/object-ca.md similarity index 100% rename from doc/manual/src/design/store/entry-ca.md rename to doc/manual/src/design/store/object-ca.md diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/objects.md similarity index 100% rename from doc/manual/src/design/store/entries.md rename to doc/manual/src/design/store/objects.md From 1bbad62c7d5f3b0d18c3fdb5a8e947ae5232139f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:02:21 -0400 Subject: [PATCH 074/371] doc: File system data -> file system object, to match Nix --- doc/manual/src/design/store/objects.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index ddfefa166..8783a1648 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -3,10 +3,10 @@ File system data in Nix is organized into *store objects*. A store object is the combination of - - some file system data + - A (root) file system object - references to store objects -## File system data +## File system objects The nix store uses a simple filesystem model, similar to the one Git uses. In particular, every file system object falls into these three cases: @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: File children additionally have an executable flag. - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing file system data or that of another store object referenced by the containing store object are allowed, but might not function as intended. + In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. Note that it cannot be executable, though. @@ -29,7 +29,7 @@ Without a flag saying which, whether root files are executable or non-executable Store objects can refer to both other store objects and themselves. -References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store objects. +References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. From 5f4d2ac091342c26d1a3f447961932139ed568b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:21:29 -0400 Subject: [PATCH 075/371] Improve store object section In particular, Nix is *not* like Git, so that needs to be fixed. --- doc/manual/src/design/store/objects.md | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 8783a1648..2c2ed2bdb 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -1,34 +1,46 @@ # Store Objects File system data in Nix is organized into *store objects*. -A store object is the combination of +A store object is the pair of - A (root) file system object - - references to store objects + - A set of references to store objects ## File system objects -The nix store uses a simple filesystem model, similar to the one Git uses. +The nix store uses a simple filesystem model. In particular, every file system object falls into these three cases: - - File: arbitrary data + - File: an executable flag, and arbitrary data - Directory: mapping of names to child file system objects. - File children additionally have an executable flag. - Symlink: may point anywhere. + In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. -A bare file as the "root" file system object is allowed. -Note that it cannot be executable, though. -This is a consequence of the executable flags being part of the child entries of the directory, rather than the child files themselves. -A root file has no parent directory; so there is no child entry about the root file, and therefore no executable flag for it. -Without a flag saying which, whether root files are executable or non-executable by default must be decided by convention, and the choice of Nix (and git) is to make them non-executable. +A bare file or symlink as the "root" file system object is allowed. + +### Comparison with Git + +This is close to Git's model, but with one crucial difference: +Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + +So long as the root object is a directory, the representations are isomorphic. +There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. + +However, if the root object is a file, there is loss of fidelity. +Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. + +Git's model matches Unix tradition, but Nix's model is more natural. ## References Store objects can refer to both other store objects and themselves. +Self-reference may seem pointless, but tracking them is in fact useful. +We can best explain why later after more concepts have been established. + References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. From b4df351880bb019f6b3176f739059440f128dc43 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:28:26 -0400 Subject: [PATCH 076/371] Relocability -> relocation in store object title --- doc/manual/src/design/store/relocatability.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md index 5cdbe9486..c7f869135 100644 --- a/doc/manual/src/design/store/relocatability.md +++ b/doc/manual/src/design/store/relocatability.md @@ -1,8 +1,8 @@ -## Advanced Topic: store object relocatability +## Advanced Topic: Store object relocation -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store object to a store with a different mount point. +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissible to relocate a store object to a store with a different mount point. -Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. +Recall from the section on [store paths](./store-paths.md) that concrete store paths look like `/-`. ~~The two final restrictions of the previous section yield an alternative view of the same information.~~ Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. From 55b437b5516dfdcd4358de48e7496b08d7be39c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:56:21 -0400 Subject: [PATCH 077/371] Improve store path section --- doc/manual/src/design/store/objects.md | 2 +- doc/manual/src/design/store/paths.md | 37 +++++++++++++++++++------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 2c2ed2bdb..8431958a5 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. + In particular, symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file or symlink as the "root" file system object is allowed. diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index d7d2ef960..1a00de5cb 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -2,18 +2,15 @@ A store path is a pair of a 20-byte digest and a name. -Historically it is the triple of those two and also the store directory, but the modern implementation's internal representation is just the pair. -This change is because in the vast majority of cases, the store dir is fully determined by the context in which the store path occurs. - ## String representation A store path is rendered as the concatenation of - - the store directory + - a store directory - a path-separator (`/`) - - the digest rendered as Base-32 (20 bytes becomes 32 bytes) + - the digest rendered as Base-32 (20 arbitrary bytes becomes 32 ASCII chars) - a hyphen (`-`) @@ -21,14 +18,32 @@ A store path is rendered as the concatenation of Let's take the store path from the very beginning of this manual as an example: - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 This parses like so: - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ store dir digest name +We then can discard the store dir to recover the conceptual pair that is a store path: + + { + digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", + name: "firefox-33.1", + } + +### Where did the "store directory" come from? + +If you notice, the above references a "store directory", but that is *not* part of the definition of a store path. +We can discard it when parsing, but what about when printing? +We need to get a store directory from *somewhere*. + +The answer is, the store directory is a property of the store that contains the store path. +The explanation for this is simple enough: a store is notionally mounted as a directory at some location, and the store object's root file system likewise mounted at this path within that directory. + +This does, however, mean the string representation of a store path is not derived just from the store path itself, but is in fact "context dependent". + ## The digest The calculation of the digest is quite complicated for historical reasons. @@ -39,10 +54,14 @@ Some of the details will be saved for later. Store paths are either content-addressed or "input-addressed". -Content addressing means that the digest ultimately derives from referred store object's file system data and references, and thus can be verified (if one knows how it was calculated). +Content addressing means that the digest ultimately derives from referred store object's file system objects and references, and thus can be verified. +There is more than one *method* of content-addressing, however. +Still, if one does know the the content addressing schema that was used, +(or guesses, there isn't that many yet!) +one can recalcuate the store path and thus verify the store object. Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. -Store paths of this sort can not be validated from the content of the store object. +Store paths of this sort can *not* be validated from the content of the store object. Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. From b98dc3b19c62abb3b8edc778cee0ac8234c5577d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 23:03:38 -0400 Subject: [PATCH 078/371] store objects, better opining sentances --- doc/manual/src/design/store/objects.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 8431958a5..38c73bc7a 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -1,6 +1,6 @@ # Store Objects -File system data in Nix is organized into *store objects*. +Data in Nix is chiefly organized into *store objects*. A store object is the pair of - A (root) file system object @@ -8,7 +8,7 @@ A store object is the pair of ## File system objects -The nix store uses a simple filesystem model. +The Nix store uses a simple filesystem model. In particular, every file system object falls into these three cases: - File: an executable flag, and arbitrary data From e4eea5e84e78cc57859cca9295da450258e8f55b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 01:44:02 -0400 Subject: [PATCH 079/371] Include abstract syntax based on the thesis for FSOs See https://edolstra.github.io/pubs/phd-thesis.pdf, page 91. --- doc/manual/src/design/store/objects.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 38c73bc7a..0f3d2a499 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -9,6 +9,16 @@ A store object is the pair of ## File system objects The Nix store uses a simple filesystem model. + + data FileSystemObject + = Regular Executable ByteString + | Directory (Map FileName FSO) + | SymLink ByteString + + data Executable + = Executable + | NonExecutable + In particular, every file system object falls into these three cases: - File: an executable flag, and arbitrary data @@ -26,10 +36,21 @@ A bare file or symlink as the "root" file system object is allowed. This is close to Git's model, but with one crucial difference: Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + data GitObject + = Blob ByteString + | Tree (Map FileName (Persission, FSO)) + + data Persission + = Directory -- IFF paired with tree + -- Iff paired with blob, one of: + | RegFile + | ExecutableFile + | Symlink + So long as the root object is a directory, the representations are isomorphic. There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. -However, if the root object is a file, there is loss of fidelity. +However, if the root object is a blob, there is loss of fidelity. Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. Git's model matches Unix tradition, but Nix's model is more natural. From 4e4bbd9e832e810b4e4cf33aced0ec0322cf55b6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 02:02:11 -0400 Subject: [PATCH 080/371] Improve store objects session more --- doc/manual/src/design/store/paths.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 1a00de5cb..cf51eb866 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -47,20 +47,28 @@ This does, however, mean the string representation of a store path is not derive ## The digest The calculation of the digest is quite complicated for historical reasons. -Some of the details will be saved for later. +The details of the algorithms will be discussed later once more concepts have been introduced. +For now, we just concern ourselves with the *key properties* of those algorithms. -> Historical note. The 20 byte restriction is because originally a digests were SHA-1 hashes. -> This is no longer true, but longer hashes and other information is still boiled down to 20 bytes. +::: {.note} +**Historical note** The 20 byte restriction is because originally a digests were SHA-1 hashes. +This is no longer true, but longer hashes and other information are still boiled down to 20 bytes. +::: -Store paths are either content-addressed or "input-addressed". +Store paths are either *content-addressed* or *input-addressed*. -Content addressing means that the digest ultimately derives from referred store object's file system objects and references, and thus can be verified. +::: {.note} +The former is a standard term used elsewhere. +The later is our own creation to evoke a contrast with content addressing. +::: + +Content addressing means that the store path digest ultimately derives from referred store object's contents, namely its file system objects and references. There is more than one *method* of content-addressing, however. -Still, if one does know the the content addressing schema that was used, +Still, if one does know the content addressing schema that was used, (or guesses, there isn't that many yet!) -one can recalcuate the store path and thus verify the store object. +one can recalculate the store path and thus verify the store object. -Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. +Input addressing means that the store path digest derives from how the store path was produced, namely the "inputs" and plan that it was built from. Store paths of this sort can *not* be validated from the content of the store object. Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. From c86c1ec7e31c2725be20e4087845e96878134846 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 02:20:15 -0400 Subject: [PATCH 081/371] Make refernces sneak preview more concise --- doc/manual/src/design/store/objects.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 0f3d2a499..e4f49a170 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -62,9 +62,8 @@ Store objects can refer to both other store objects and themselves. Self-reference may seem pointless, but tracking them is in fact useful. We can best explain why later after more concepts have been established. -References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. -For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. -When we get to building in a future section, this process will be described in precise detail. +References are normally calculated so as to to record the presence of textual references in store object's file systems obejcts. +This process will be described precisely in the section on [building](./building.md), once more concepts are explained, as building is the primary path new store objects with non-trivial references are created. However, scanning for references is not mandatory. Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, From 0737094161cc2b13a6ec06a58a78200cb2116b88 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Apr 2022 01:39:30 -0400 Subject: [PATCH 082/371] Add draft "Rosetta stone" by @fricklerhandwerk and stub commentary The idea and most of the execution are @fricklerhandwerk's. I changed a few things best I could based on @edolstra's corrections, and a Bazel glossary. Valentin Gagarin --- doc/manual/src/SUMMARY.md.in | 3 +- doc/manual/src/design/store/related-work.md | 37 +++++++++++++++++++++ doc/manual/src/design/store/store.md | 22 ++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 doc/manual/src/design/store/related-work.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 323a569d1..9cde44a0d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,7 +26,8 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store object relocatability](design/store/relocatability.md) + - [Advanced Topic: Store object relocatability](design/store/relocatability.md) + - [Advanced Topic: Related work](design/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/related-work.md b/doc/manual/src/design/store/related-work.md new file mode 100644 index 000000000..b64b41988 --- /dev/null +++ b/doc/manual/src/design/store/related-work.md @@ -0,0 +1,37 @@ +# Advanced Topic: Related Work + +## Bazel + +TODO skylark and layering. + +TODO being monadic, if RFC 92. + +## Build Systems à la Carte + +TODO user-choosen keys vs keys chosen automatically? +Purity in face of dynamic tasks (no conflicts, guaranteed). + +TODO Does Nix constitute a different way to be be monadic? +Purity of keys, as mentioned. +Dynamic tasks/keys vs dynamic dependencies. +(Not sure yet.) + +## Lazy evaluation + +We clearly have thunks that produce thunks, but less clearly functions that produce functions. + +Do we have open terms? + +Do we hve thunks vs expressions distinction? +c.f. John Shutt's modern fexprs, when the syntax can "leak". + +## Machine models + +TODO +Derivations as store objects via drv files makes Nix a "Von Neumann" archicture. +Can also imagine a "Harvard" archicture where derivations are stored separately? +Can we in general imagine N heaps for N different sorts of objects? + +TODO +Also, leaning on the notion of "builtin builders" more, having multiple different sorts of user-defined builders too. +The builder is a black box as far as the Nix model is concerned. diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index e91f0f6e4..458c1e2b8 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,5 +1,27 @@ +# Nix Store + A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. There are many types of stores, but all of them at least respect this model. Some however offer additional functionality. + +## A Rosetta stone for the Nix store. + +The design of Nix is comparable to other build systems, even programming languages in general. +Here is a rough [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology. +If you are familiar with one of these columns, this might help the following sections make more sense. + +generic build system | Nix | Bazel | Build Systems à la Carte | lazy programming language +-- | -- | -- | -- | -- +data (build input, build result) | component | file (source, target) | value | value +build plan | derivation graph | action graph | `Tasks` | thunk +build step | derivation | rule | `Task` | thunk +build instructions | builder | (depends on action type) | `Task` | function +build | build | build | `Build` applied to arguments | evaluation +persistence layer | store | file system | `Store` | heap + +(n.b. Bazel terms gotten from https://docs.bazel.build/versions/main/glossary.html.) + +Plenty more could be said elaborating these comparisons. +We will save that for the end of this chapter, in the [Related Work](./related-work.md) section. From 0eae4bfad1bfa94347252b51dec8c87fa73644fb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Apr 2022 14:39:35 +0200 Subject: [PATCH 083/371] reword overview with clear terminology trying to capture alternative terms in one go here, mirroring everyday use: derivation - build plan realise - execute build there will be more of that sort. --- doc/manual/src/design/overview.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 9329f2ad7..6a5555dff 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -2,14 +2,22 @@ Nix consists of layers that operate fairly independently. -At the top is the *command line interface*. +At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the Nix *expression language*, in which packages and configurations are written. +Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +It is used to compose expressions which ultimately evaluate to *derivations* — self-contained *build plans* to derive new data from referenced input data. -The command line and expression language are what users interact with most. +::: {.note} +The Nix language itself does not have a notion of *packages* or *configurations*. +As far as we are concerned here, the result of a derivation is just data. +In practice this amounts to a set of files in a file system. +::: -Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. -It is also used for executing those build plans. -The store layer may not be as visible, but this is the heart of Nix. +The command line and Nix language are what users interact with most. + +Underlying everything is the *Nix store*, a mechanism to keep track of derivations, data, and references between them. +It can also *realise derivations*, that is, *execute build instructions* to produce new data. +It uses the file system as a persistence layer, and a database to keep track of references. + +This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. -This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." From 327ccd3b0787f7803a51aa5124cf0cbab694523c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Apr 2022 15:06:05 +0200 Subject: [PATCH 084/371] only use generic build system terminology we will use a translation table to introduce nix-specific terms --- doc/manual/src/design/overview.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 6a5555dff..c60121a01 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,18 +5,18 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to *derivations* — self-contained *build plans* to derive new data from referenced input data. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the result of a derivation is just data. +As far as we are concerned here, the inputs and results of a derivation are just data. In practice this amounts to a set of files in a file system. ::: The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of derivations, data, and references between them. -It can also *realise derivations*, that is, *execute build instructions* to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. +It can also execute *build instructions* captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. From 804e8bd7473516b087f39161bd34730e26480517 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:43:19 +0200 Subject: [PATCH 085/371] indicate sequence with "then" Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index c60121a01..20a3fe653 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -19,5 +19,5 @@ Underlying everything is the *Nix store*, a mechanism to keep track of build pla It can also execute *build instructions* captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. -This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. +This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 23ee0b24f7c0fd43ccf6f99b87941df4cb071689 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:44:00 +0200 Subject: [PATCH 086/371] correctly use comma for nesting Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 20a3fe653..b1adb143b 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -16,7 +16,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions* captured in the build plans, to produce new data. +It can also execute *build instructions*, captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 51e6bed25e073afac5a7b5e11033769a615d0473 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:54:43 +0200 Subject: [PATCH 087/371] do not mention implementation details Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index b1adb143b..0cead62ab 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -17,7 +17,6 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build plans, to produce new data. -It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 89a7c956ffadbc9c17849051f3c91a1225dcfce2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Apr 2022 14:05:50 -0400 Subject: [PATCH 088/371] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 0cead62ab..15ddb562d 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,7 +5,7 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. From b387d809437b0c787691b814bf65e1647fd18009 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 24 Apr 2022 21:13:45 +0200 Subject: [PATCH 089/371] remove sentence for chapter transition idea: sections could be read in different orders by linking them in different ways (e.g. depth-first or breadth-first). adding hard-coded transitions makes that confusing. --- doc/manual/src/design/overview.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 15ddb562d..b5f8b6aad 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -18,5 +18,3 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build plans, to produce new data. -This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. - From 34ea74c9ec21641399a96386094b021d7a2b30f4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Apr 2022 11:46:15 +0200 Subject: [PATCH 090/371] reword introductory section there should be a meta section for each chapter to give motivation of the presented structure. the structure itself is visible from the table of contents. --- doc/manual/src/design/design.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index 7d4211764..f5135bc9a 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -1,5 +1,6 @@ # Design and Data Model Most of the manual is about how to use Nix. -This chapter is about what Nix actually is. -The hope is that it can serve as a reference for key concepts, and also shed light on why things are the way they are. +This chapter is about the technical principles behind Nix. + +It describes each architectural layer and its components in its own section, starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 75981263912805a77890bcdb63c4aa3bbb0d8e09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 21:17:44 +0200 Subject: [PATCH 091/371] remove separate meta-section, add architecture diagram the diagram is a first approximation and only covers that same section. of course there is much more going on, and other features should at some point also be illustrated. we also have to think about presentation format and technicalities behind it. the manual has to render to `man`, but we may want something more refined for web view. --- doc/manual/src/SUMMARY.md.in | 3 +-- doc/manual/src/design/design.md | 39 ++++++++++++++++++++++++++-- doc/manual/src/design/overview.md | 20 -------------- doc/manual/src/design/store/store.md | 2 +- 4 files changed, 39 insertions(+), 25 deletions(-) delete mode 100644 doc/manual/src/design/overview.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9cde44a0d..ec457126a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,8 +16,7 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Design and Data Model](design/design.md) - - [Overview](design/overview.md) - - [The Store Layer](design/store/store.md) + - [Store](design/store/store.md) - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index f5135bc9a..960a68f41 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -1,6 +1,41 @@ # Design and Data Model -Most of the manual is about how to use Nix. This chapter is about the technical principles behind Nix. -It describes each architectural layer and its components in its own section, starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. +## Architecture + +Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). + +``` + [ commmand line interface ] + | + | evaluates + V + [ configuration language ] + | + | evaluates to + | + reference V build +[ build inputs ] --> [ build plans ] --> [ build results ] + \ | / + \ | persisted to / + \ V / + [ store ] +``` + +At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. + +Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. + +::: {.note} +The Nix language itself does not have a notion of *packages* or *configurations*. +As far as we are concerned here, the inputs and results of a derivation are just data. +In practice this amounts to a set of files in a file system. +::: + +The command line and Nix language are what users interact with most. + +Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. +It can also execute *build instructions*, captured in the build plans, to produce new data. + diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md deleted file mode 100644 index b5f8b6aad..000000000 --- a/doc/manual/src/design/overview.md +++ /dev/null @@ -1,20 +0,0 @@ -# Overview - -Nix consists of layers that operate fairly independently. - -At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. - -Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. - -::: {.note} -The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a derivation are just data. -In practice this amounts to a set of files in a file system. -::: - -The command line and Nix language are what users interact with most. - -Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build plans, to produce new data. - diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index 458c1e2b8..d4add52f5 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,4 +1,4 @@ -# Nix Store +# Store A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. From d30033759a07a0d5df7aac4c22bfa274b74baf0e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 21:21:46 +0200 Subject: [PATCH 092/371] address Nix language consistently as configuration language --- doc/manual/src/design/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index 960a68f41..16c51533b 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -25,7 +25,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. ::: {.note} From 39f01176a713bad4e090e19c8a25c2447056640d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 23:25:33 +0200 Subject: [PATCH 093/371] design -> architecture, add motivation following ideas found in Architecture of Gazelle[1] [1]: https://github.com/bazelbuild/bazel-gazelle/blob/56d35f8db086bb65ef876f96f7baa7b71516daf8/Design.rst --- doc/manual/src/SUMMARY.md.in | 2 +- .../src/{design/design.md => architecture/architecture.md} | 7 ++++--- doc/manual/src/{design => architecture}/store/building.md | 0 doc/manual/src/{design => architecture}/store/drvs/ca.md | 0 doc/manual/src/{design => architecture}/store/drvs/drvs.md | 0 doc/manual/src/{design => architecture}/store/drvs/ia.md | 0 .../src/{design => architecture}/store/input-addressing.md | 0 doc/manual/src/{design => architecture}/store/nar.md | 0 doc/manual/src/{design => architecture}/store/object-ca.md | 0 doc/manual/src/{design => architecture}/store/objects.md | 0 doc/manual/src/{design => architecture}/store/paths.md | 0 .../src/{design => architecture}/store/related-work.md | 0 .../src/{design => architecture}/store/relocatability.md | 0 doc/manual/src/{design => architecture}/store/store.md | 0 14 files changed, 5 insertions(+), 4 deletions(-) rename doc/manual/src/{design/design.md => architecture/architecture.md} (88%) rename doc/manual/src/{design => architecture}/store/building.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/ca.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/drvs.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/ia.md (100%) rename doc/manual/src/{design => architecture}/store/input-addressing.md (100%) rename doc/manual/src/{design => architecture}/store/nar.md (100%) rename doc/manual/src/{design => architecture}/store/object-ca.md (100%) rename doc/manual/src/{design => architecture}/store/objects.md (100%) rename doc/manual/src/{design => architecture}/store/paths.md (100%) rename doc/manual/src/{design => architecture}/store/related-work.md (100%) rename doc/manual/src/{design => architecture}/store/relocatability.md (100%) rename doc/manual/src/{design => architecture}/store/store.md (100%) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index ec457126a..c029e30bf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,7 +15,7 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Design and Data Model](design/design.md) +- [Architecture](architecture/architecture.md) - [Store](design/store/store.md) - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/architecture/architecture.md similarity index 88% rename from doc/manual/src/design/design.md rename to doc/manual/src/architecture/architecture.md index 16c51533b..b17eacd2e 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/architecture/architecture.md @@ -1,8 +1,9 @@ -# Design and Data Model +# Architecture -This chapter is about the technical principles behind Nix. +This chapter describes how Nix works. +It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. -## Architecture +## Overview Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/architecture/store/building.md similarity index 100% rename from doc/manual/src/design/store/building.md rename to doc/manual/src/architecture/store/building.md diff --git a/doc/manual/src/design/store/drvs/ca.md b/doc/manual/src/architecture/store/drvs/ca.md similarity index 100% rename from doc/manual/src/design/store/drvs/ca.md rename to doc/manual/src/architecture/store/drvs/ca.md diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md similarity index 100% rename from doc/manual/src/design/store/drvs/drvs.md rename to doc/manual/src/architecture/store/drvs/drvs.md diff --git a/doc/manual/src/design/store/drvs/ia.md b/doc/manual/src/architecture/store/drvs/ia.md similarity index 100% rename from doc/manual/src/design/store/drvs/ia.md rename to doc/manual/src/architecture/store/drvs/ia.md diff --git a/doc/manual/src/design/store/input-addressing.md b/doc/manual/src/architecture/store/input-addressing.md similarity index 100% rename from doc/manual/src/design/store/input-addressing.md rename to doc/manual/src/architecture/store/input-addressing.md diff --git a/doc/manual/src/design/store/nar.md b/doc/manual/src/architecture/store/nar.md similarity index 100% rename from doc/manual/src/design/store/nar.md rename to doc/manual/src/architecture/store/nar.md diff --git a/doc/manual/src/design/store/object-ca.md b/doc/manual/src/architecture/store/object-ca.md similarity index 100% rename from doc/manual/src/design/store/object-ca.md rename to doc/manual/src/architecture/store/object-ca.md diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/architecture/store/objects.md similarity index 100% rename from doc/manual/src/design/store/objects.md rename to doc/manual/src/architecture/store/objects.md diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/architecture/store/paths.md similarity index 100% rename from doc/manual/src/design/store/paths.md rename to doc/manual/src/architecture/store/paths.md diff --git a/doc/manual/src/design/store/related-work.md b/doc/manual/src/architecture/store/related-work.md similarity index 100% rename from doc/manual/src/design/store/related-work.md rename to doc/manual/src/architecture/store/related-work.md diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/architecture/store/relocatability.md similarity index 100% rename from doc/manual/src/design/store/relocatability.md rename to doc/manual/src/architecture/store/relocatability.md diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/architecture/store/store.md similarity index 100% rename from doc/manual/src/design/store/store.md rename to doc/manual/src/architecture/store/store.md From c8c1b705ad5cdf115997cef387d83c4b04fe0660 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 24 Apr 2022 21:34:33 +0200 Subject: [PATCH 094/371] reword section on Nix store --- doc/manual/src/architecture/store/store.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d4add52f5..b52fe8255 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,10 +1,14 @@ # Store -A Nix store is a collection of *store objects* referred to by *store paths*. -Every store also has a "store directory path", which is a path prefix used for various purposes. +A Nix store is a collection of [store objects](objects.md) and associated operations. -There are many types of stores, but all of them at least respect this model. -Some however offer additional functionality. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. + +Store objects are accessible in a file system through [store paths](paths.md). +Every store has a *store directory*, which contains that store’s objects and is a prefix of their store paths. +It defaults to `/nix/store`, but is in principle arbitrary. + +A Nix store can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. ## A Rosetta stone for the Nix store. From 7b5c00f67f729bb3c0d026f3f4d14e727c4cf420 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 01:15:27 +0200 Subject: [PATCH 095/371] add concrete store examples, reword note on file system --- doc/manual/src/architecture/store/store.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b52fe8255..688c7e1a2 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,11 +4,18 @@ A Nix store is a collection of [store objects](objects.md) and associated operat These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -Store objects are accessible in a file system through [store paths](paths.md). -Every store has a *store directory*, which contains that store’s objects and is a prefix of their store paths. -It defaults to `/nix/store`, but is in principle arbitrary. +A Nix store allows to add and retrieve store objects. +It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. +It also keeps track of *references* between data and can therefore garbage-collect unused store objects. -A Nix store can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. +There exist different types of stores, which all follow this model. +Examples: +- store on the local file system +- remote store accessible via SSH +- binary cache store accessible via HTTP + +Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). +The store directory defaults to `/nix/store`, but is in principle arbitrary. ## A Rosetta stone for the Nix store. From 070c85499b370bf886db1a87c04ed9c319bb6323 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 09:44:46 +0200 Subject: [PATCH 096/371] fix grammar Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 688c7e1a2..96796f26c 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,7 +4,7 @@ A Nix store is a collection of [store objects](objects.md) and associated operat These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -A Nix store allows to add and retrieve store objects. +A Nix store allows adding and retrieving store objects. It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. From 5f96a0b4e830bef59b5986c916727d07b244c22c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:18:03 +0200 Subject: [PATCH 097/371] associated operations are not collected Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 96796f26c..624439f73 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,6 +1,6 @@ # Store -A Nix store is a collection of [store objects](objects.md) and associated operations. +A Nix store is a collection of [store objects](objects.md) with associated operations. These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. From 610ddf44aa86924834bf2ac3735c7cb75a6a1f1c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 25 Apr 2022 13:25:43 +0200 Subject: [PATCH 098/371] reword introduction to rosetta stone, add links attempt to explain used and documented terminology, as well as how the declarative programming paradigm relates to building software. in the future one could highlight encouraged terms to shape future material into higher consistency. --- doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/architecture/store/store.md | 29 ++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index b17eacd2e..836b40c91 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 624439f73..af0609d4b 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -17,22 +17,21 @@ Examples: Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). The store directory defaults to `/nix/store`, but is in principle arbitrary. -## A Rosetta stone for the Nix store. +## A [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology -The design of Nix is comparable to other build systems, even programming languages in general. -Here is a rough [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology. -If you are familiar with one of these columns, this might help the following sections make more sense. +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. -generic build system | Nix | Bazel | Build Systems à la Carte | lazy programming language +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- -data (build input, build result) | component | file (source, target) | value | value -build plan | derivation graph | action graph | `Tasks` | thunk -build step | derivation | rule | `Task` | thunk -build instructions | builder | (depends on action type) | `Task` | function -build | build | build | `Build` applied to arguments | evaluation -persistence layer | store | file system | `Store` | heap +data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value +build plan | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build graph | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) +build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build | realisation | build | application of `Build` | evaluation +persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap -(n.b. Bazel terms gotten from https://docs.bazel.build/versions/main/glossary.html.) - -Plenty more could be said elaborating these comparisons. -We will save that for the end of this chapter, in the [Related Work](./related-work.md) section. +All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software +Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf) (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) (2018). From ca5ebf63827f82ae63796d775070b2f9cf828625 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:34:53 +0200 Subject: [PATCH 099/371] revert build plan/step distinction, reorder rows --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index af0609d4b..f4f5a3e16 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -27,9 +27,9 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build plan | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) -build graph | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | realisation | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 40efe5b30b24e75b26017eb68c62acfc6b7ebf93 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:36:36 +0200 Subject: [PATCH 100/371] build instrcution: Task -> function --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f4f5a3e16..e851444c9 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -27,7 +27,7 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | realisation | build | application of `Build` | evaluation From a145007a577fda8a8f3b65b66f4c359410b4af82 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:48:59 +0200 Subject: [PATCH 101/371] component -> store object, realisation -> build --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index e851444c9..74d3d7f84 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,11 +26,11 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- -data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value +data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) -build | realisation | build | application of `Build` | evaluation +build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software From e5e48593c85d052585ff8a275c54823caf575a9d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:16:27 +0200 Subject: [PATCH 102/371] move git comparison to related work it should be pulled out of the branch before we go for merging --- doc/manual/src/architecture/store/objects.md | 23 ------------------ .../src/architecture/store/related-work.md | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index e4f49a170..caa00d862 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -31,29 +31,6 @@ In particular, every file system object falls into these three cases: A bare file or symlink as the "root" file system object is allowed. -### Comparison with Git - -This is close to Git's model, but with one crucial difference: -Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. - - data GitObject - = Blob ByteString - | Tree (Map FileName (Persission, FSO)) - - data Persission - = Directory -- IFF paired with tree - -- Iff paired with blob, one of: - | RegFile - | ExecutableFile - | Symlink - -So long as the root object is a directory, the representations are isomorphic. -There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. - -However, if the root object is a blob, there is loss of fidelity. -Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. - -Git's model matches Unix tradition, but Nix's model is more natural. ## References diff --git a/doc/manual/src/architecture/store/related-work.md b/doc/manual/src/architecture/store/related-work.md index b64b41988..92b7d480e 100644 --- a/doc/manual/src/architecture/store/related-work.md +++ b/doc/manual/src/architecture/store/related-work.md @@ -25,6 +25,30 @@ Do we have open terms? Do we hve thunks vs expressions distinction? c.f. John Shutt's modern fexprs, when the syntax can "leak". +## Comparison with Git file system model + +This is close to Git's model, but with one crucial difference: +Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + + data GitObject + = Blob ByteString + | Tree (Map FileName (Persission, FSO)) + + data Persission + = Directory -- IFF paired with tree + -- Iff paired with blob, one of: + | RegFile + | ExecutableFile + | Symlink + +So long as the root object is a directory, the representations are isomorphic. +There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. + +However, if the root object is a blob, there is loss of fidelity. +Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. + +Git's model matches Unix tradition, but Nix's model is more natural. + ## Machine models TODO From 90fc5b41a83575c0bd008868f989b6e1497ce3a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:18:23 +0200 Subject: [PATCH 103/371] reword file system objects - use singular for the "class" - more consistency in type definition - minor fixes in wording --- doc/manual/src/architecture/store/objects.md | 40 ++++++++------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index caa00d862..872a209c3 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -1,40 +1,32 @@ -# Store Objects +# Store Object -Data in Nix is chiefly organized into *store objects*. +Nix organizes the data it manages into *store objects*. A store object is the pair of - - A (root) file system object - - A set of references to store objects + - a [file system object](#file-system-object) + - a set of [references](#reference) to store objects. -## File system objects +## File system object {#file-system-object} -The Nix store uses a simple filesystem model. +The Nix store uses a simple file system model. data FileSystemObject - = Regular Executable ByteString - | Directory (Map FileName FSO) - | SymLink ByteString - - data Executable - = Executable - | NonExecutable - -In particular, every file system object falls into these three cases: + = File Executable Data + | Directory (Map FileName FileSystemObject) + | SymLink Path +Every file system object is one of the following: - File: an executable flag, and arbitrary data + - Directory: mapping of names to child file system objects + - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - - Directory: mapping of names to child file system objects. + In particular, symlinks pointing outside of their own root file system object, or to a store object without a matching reference, are allowed, but might not function as intended. - - Symlink: may point anywhere. +A bare file or symlink can be a root file system object. - In particular, symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. +## Reference {#reference} -A bare file or symlink as the "root" file system object is allowed. - - -## References - -Store objects can refer to both other store objects and themselves. +A store object can refer to both other store objects and itself. Self-reference may seem pointless, but tracking them is in fact useful. We can best explain why later after more concepts have been established. From fb2ec7e4ec758e17f247218625d23ceba685e8a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:54:51 +0200 Subject: [PATCH 104/371] reword section on references use file Contents instead of Data, as that flows more naturally in the prose. simplify explanation of the idea behind scanning for store paths remove references to unfinished sections. --- doc/manual/src/architecture/store/objects.md | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 872a209c3..d7a27b528 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -11,12 +11,12 @@ A store object is the pair of The Nix store uses a simple file system model. data FileSystemObject - = File Executable Data + = File Executable Contents | Directory (Map FileName FileSystemObject) | SymLink Path Every file system object is one of the following: - - File: an executable flag, and arbitrary data + - File: an executable flag, and arbitrary data for contents - Directory: mapping of names to child file system objects - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. @@ -26,17 +26,16 @@ A bare file or symlink can be a root file system object. ## Reference {#reference} -A store object can refer to both other store objects and itself. +A store object can refer to other store objects or itself. -Self-reference may seem pointless, but tracking them is in fact useful. -We can best explain why later after more concepts have been established. +Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. -References are normally calculated so as to to record the presence of textual references in store object's file systems obejcts. -This process will be described precisely in the section on [building](./building.md), once more concepts are explained, as building is the primary path new store objects with non-trivial references are created. +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness: +Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. -However, scanning for references is not mandatory. -Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, -and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. -Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. +However, having references match store paths in files is not enforced by the data model: +Store objects could have excess or incomplete references with respect to store paths found in their file contents. + +Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. -This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store object, rather than try to recompute them on the fly by scanning their contents. From 5fda995491f1f6b32b46b45c2b47f3e329c997f6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 00:24:23 +0200 Subject: [PATCH 105/371] formalize file system objects convention: describe every data type in prose, and illustrate with a class diagram, and a textual representation of an abstract data type. right now we save ourselves the trouble of doing class diagrams, we can add them later. but they are important. --- doc/manual/src/architecture/store/objects.md | 25 ++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index d7a27b528..7f341c87c 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -6,31 +6,42 @@ A store object is the pair of - a [file system object](#file-system-object) - a set of [references](#reference) to store objects. +We call a store object's outermost file system object the *root*. + +```haskell +data StoreOject = StoreObject { + root :: FileSystemObject +, references :: Set StoreObject +} +``` + ## File system object {#file-system-object} The Nix store uses a simple file system model. - data FileSystemObject - = File Executable Contents - | Directory (Map FileName FileSystemObject) - | SymLink Path - Every file system object is one of the following: - File: an executable flag, and arbitrary data for contents - Directory: mapping of names to child file system objects - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - In particular, symlinks pointing outside of their own root file system object, or to a store object without a matching reference, are allowed, but might not function as intended. +```haskell +data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } +``` A bare file or symlink can be a root file system object. +Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. + ## Reference {#reference} A store object can refer to other store objects or itself. Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness: +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. However, having references match store paths in files is not enforced by the data model: From 07d490fd895b09144684dd8b389ef02f0f1256c4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 00:58:40 +0200 Subject: [PATCH 106/371] stores can also delete objects --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 74d3d7f84..cbb5a4169 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,7 +4,7 @@ A Nix store is a collection of [store objects](objects.md) with associated opera These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -A Nix store allows adding and retrieving store objects. +A Nix store allows adding, retrieving, and deleting store objects. It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. From e90586c0a40cf59df9d39407dcc49d5944a8b853 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 01:15:33 +0200 Subject: [PATCH 107/371] add motivation for references --- doc/manual/src/architecture/store/objects.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 7f341c87c..30683d22d 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -37,9 +37,20 @@ Symlinks pointing outside of their own root, or to a store object without a matc ## Reference {#reference} -A store object can refer to other store objects or itself. +A store object can reference other store objects. -Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. +Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. + +Building, copying and deleting store objects must be done in a way that obeys this property: + +- We can only safely delete unreferenced objects. + +- When copying, to maintain correctness, either the result must be "revealed" atomically to the destination store, or objects must be copied in reference-dependency order. + +- Newly built store objects must only refer to store objects in the closure of the build inputs. + This ensures the purity of the build. + +### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. From b5ca3d12b6f3414302363c8ae362334c020448c7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 02:10:24 +0200 Subject: [PATCH 108/371] reword details on keeping closure property --- doc/manual/src/architecture/store/objects.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 30683d22d..f7587e112 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -43,13 +43,13 @@ Nix stores have the *closure property*: for each store object in the store, all Building, copying and deleting store objects must be done in a way that obeys this property: +- Build results must only refer to store objects in the closure of the build inputs. + +- Store objects being copied must refer to objects already in the destination store. + Recursive copying must either proceed in dependency order or be atomic. + - We can only safely delete unreferenced objects. -- When copying, to maintain correctness, either the result must be "revealed" atomically to the destination store, or objects must be copied in reference-dependency order. - -- Newly built store objects must only refer to store objects in the closure of the build inputs. - This ensures the purity of the build. - ### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. From b01bb65d3023281c601024578a9c19f32e7ad011 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Apr 2022 21:41:04 -0400 Subject: [PATCH 109/371] Fix rel path in doc --- doc/manual/src/architecture/store/building.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/building.md b/doc/manual/src/architecture/store/building.md index f4f2649a3..d43904126 100644 --- a/doc/manual/src/architecture/store/building.md +++ b/doc/manual/src/architecture/store/building.md @@ -2,7 +2,7 @@ ## Scanning for references -Before in the section on [store objects](../objects.md), we talked abstractly about scanning for references. +Before in the section on [store objects](./objects.md), we talked abstractly about scanning for references. Now we can make this concrete. After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. From 3d8f2f5cc1faa4d42138267f21d2996da99c49a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Apr 2022 17:53:10 -0400 Subject: [PATCH 110/371] Fix manual TOC links --- doc/manual/src/SUMMARY.md.in | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c029e30bf..80a228741 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,17 +16,17 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - - [Store](design/store/store.md) - - [Store Objects](design/store/objects.md) - - [Store Paths](design/store/paths.md) - - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Objects](design/store/object-ca.md) - - [Derivations](design/store/drvs/drvs.md) - - [Input-Addressing](design/store/drvs/ia.md) - - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - - [Building](design/store/building.md) - - [Advanced Topic: Store object relocatability](design/store/relocatability.md) - - [Advanced Topic: Related work](design/store/related-work.md) + - [Store](architecture/store/store.md) + - [Store Objects](architecture/store/objects.md) + - [Store Paths](architecture/store/paths.md) + - [Nix Archives](architecture/store/nar.md) + - [Content-Addressing Store Objects](architecture/store/object-ca.md) + - [Derivations](architecture/store/drvs/drvs.md) + - [Input-Addressing](architecture/store/drvs/ia.md) + - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) + - [Building](architecture/store/building.md) + - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) + - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) From 1ba6d8fb1d59f4d40825141f75098dfbfd3b0153 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:53:37 +0200 Subject: [PATCH 111/371] remove incomplete section: building --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/building.md | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 doc/manual/src/architecture/store/building.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 80a228741..596843057 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - - [Building](architecture/store/building.md) - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) diff --git a/doc/manual/src/architecture/store/building.md b/doc/manual/src/architecture/store/building.md deleted file mode 100644 index d43904126..000000000 --- a/doc/manual/src/architecture/store/building.md +++ /dev/null @@ -1,15 +0,0 @@ -# Building - -## Scanning for references - -Before in the section on [store objects](./objects.md), we talked abstractly about scanning for references. -Now we can make this concrete. - -After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. -There is a few steps of this, but let's start with the simple case of one input-addressed output first. - -\[Overview of things that need to happen.] - -For example, if Nix thinks `/nix/store/asdfasdfasdf-foo` and `/nix/store/qwerqwerqwer-bar` are paths the data might plausibly reference, Nix will scan all the contents of all files recursively for the "hash parts" `asdfasdfasdf`` and `qwerqwerqwer`. - -\[Explain why whitelist.] From 96876b1eaeae95e3f2f338c6b3495040fc843c57 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:54:06 +0200 Subject: [PATCH 112/371] remove incomplete section: related work --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/related-work.md | 61 ------------------- 2 files changed, 62 deletions(-) delete mode 100644 doc/manual/src/architecture/store/related-work.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 596843057..ee7f5d7ac 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -25,7 +25,6 @@ - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/related-work.md b/doc/manual/src/architecture/store/related-work.md deleted file mode 100644 index 92b7d480e..000000000 --- a/doc/manual/src/architecture/store/related-work.md +++ /dev/null @@ -1,61 +0,0 @@ -# Advanced Topic: Related Work - -## Bazel - -TODO skylark and layering. - -TODO being monadic, if RFC 92. - -## Build Systems à la Carte - -TODO user-choosen keys vs keys chosen automatically? -Purity in face of dynamic tasks (no conflicts, guaranteed). - -TODO Does Nix constitute a different way to be be monadic? -Purity of keys, as mentioned. -Dynamic tasks/keys vs dynamic dependencies. -(Not sure yet.) - -## Lazy evaluation - -We clearly have thunks that produce thunks, but less clearly functions that produce functions. - -Do we have open terms? - -Do we hve thunks vs expressions distinction? -c.f. John Shutt's modern fexprs, when the syntax can "leak". - -## Comparison with Git file system model - -This is close to Git's model, but with one crucial difference: -Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. - - data GitObject - = Blob ByteString - | Tree (Map FileName (Persission, FSO)) - - data Persission - = Directory -- IFF paired with tree - -- Iff paired with blob, one of: - | RegFile - | ExecutableFile - | Symlink - -So long as the root object is a directory, the representations are isomorphic. -There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. - -However, if the root object is a blob, there is loss of fidelity. -Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. - -Git's model matches Unix tradition, but Nix's model is more natural. - -## Machine models - -TODO -Derivations as store objects via drv files makes Nix a "Von Neumann" archicture. -Can also imagine a "Harvard" archicture where derivations are stored separately? -Can we in general imagine N heaps for N different sorts of objects? - -TODO -Also, leaning on the notion of "builtin builders" more, having multiple different sorts of user-defined builders too. -The builder is a black box as far as the Nix model is concerned. From 7cec9ee3612c7255e6dc4fb94fc238f3d81ced03 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:54:24 +0200 Subject: [PATCH 113/371] remove incomplete section: relocatability --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/relocatability.md | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 doc/manual/src/architecture/store/relocatability.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index ee7f5d7ac..c35505bbe 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/relocatability.md b/doc/manual/src/architecture/store/relocatability.md deleted file mode 100644 index c7f869135..000000000 --- a/doc/manual/src/architecture/store/relocatability.md +++ /dev/null @@ -1,15 +0,0 @@ -## Advanced Topic: Store object relocation - -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissible to relocate a store object to a store with a different mount point. - -Recall from the section on [store paths](./store-paths.md) that concrete store paths look like `/-`. - -~~The two final restrictions of the previous section yield an alternative view of the same information.~~ -Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. - -This corresponds to the observation that a store object with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store object without any references, i.e. thus without a store directory, can exist in any store. - -Lastly, this illustrates the purpose of tracking self references. -Store objects without self-references or other references are relocatable, while store paths with self-references aren't. -This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. -\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] From b18852eb3fc1d62ec307d3b2f102c6bd8574d069 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:55:11 +0200 Subject: [PATCH 114/371] remove incomplete section: content-addressed objects --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/object-ca.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 doc/manual/src/architecture/store/object-ca.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c35505bbe..11a974b47 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,7 +20,6 @@ - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - [Nix Archives](architecture/store/nar.md) - - [Content-Addressing Store Objects](architecture/store/object-ca.md) - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) diff --git a/doc/manual/src/architecture/store/object-ca.md b/doc/manual/src/architecture/store/object-ca.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/object-ca.md +++ /dev/null @@ -1 +0,0 @@ -TODO From 3bd125ebbeae38c9efb5790e13265b7534e66c45 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:55:52 +0200 Subject: [PATCH 115/371] remove incomplete section: nix archives --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/nar.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 doc/manual/src/architecture/store/nar.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 11a974b47..38277aaf2 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,7 +19,6 @@ - [Store](architecture/store/store.md) - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - - [Nix Archives](architecture/store/nar.md) - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) diff --git a/doc/manual/src/architecture/store/nar.md b/doc/manual/src/architecture/store/nar.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/nar.md +++ /dev/null @@ -1 +0,0 @@ -TODO From ad8c2ed7f0566a5fe1b4a3591240eb06804bd958 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:59:03 +0200 Subject: [PATCH 116/371] remove incomplete section: input/content-addressing --- doc/manual/src/SUMMARY.md.in | 2 -- doc/manual/src/architecture/store/drvs/ca.md | 0 doc/manual/src/architecture/store/drvs/ia.md | 0 doc/manual/src/architecture/store/input-addressing.md | 1 - 4 files changed, 3 deletions(-) delete mode 100644 doc/manual/src/architecture/store/drvs/ca.md delete mode 100644 doc/manual/src/architecture/store/drvs/ia.md delete mode 100644 doc/manual/src/architecture/store/input-addressing.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 38277aaf2..257880efe 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,8 +20,6 @@ - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - [Derivations](architecture/store/drvs/drvs.md) - - [Input-Addressing](architecture/store/drvs/ia.md) - - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/ca.md b/doc/manual/src/architecture/store/drvs/ca.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/manual/src/architecture/store/drvs/ia.md b/doc/manual/src/architecture/store/drvs/ia.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/manual/src/architecture/store/input-addressing.md b/doc/manual/src/architecture/store/input-addressing.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/input-addressing.md +++ /dev/null @@ -1 +0,0 @@ -TODO From d3effd014b17bc957b9af8ed35b2f25a3b54e02c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 14:05:46 +0200 Subject: [PATCH 117/371] update architecture diagram --- doc/manual/src/architecture/architecture.md | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 836b40c91..25b6a9616 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -8,20 +8,20 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). ``` - [ commmand line interface ] - | - | evaluates - V - [ configuration language ] - | - | evaluates to - | - reference V build -[ build inputs ] --> [ build plans ] --> [ build results ] - \ | / - \ | persisted to / - \ V / - [ store ] + [ commmand line interface ]--------+ + | | + | evaluates | manages + V | + [ configuration language ] | + | V ++-------------------------------|---------------------------------+ +| store | | +| | evaluates to | +| | | +| references V builds | +| [ build input ] --> [ build plan ] --> [ build result ] | +| | ++-----------------------------------------------------------------+ ``` At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. From 87523f01e3dc27d4cf57354a67346f8473e93cc5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:27:44 +0200 Subject: [PATCH 118/371] match grammatical case to arrow direction --- doc/manual/src/architecture/architecture.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 25b6a9616..a0aa713f3 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -18,8 +18,8 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | store | | | | evaluates to | | | | -| references V builds | -| [ build input ] --> [ build plan ] --> [ build result ] | +| referenced by V builds | +| [ build input ] ----> [ build plan ] ----> [ build result ] | | | +-----------------------------------------------------------------+ ``` From 902638c519fb46b0f0a3f7092ccb7cd2a622b206 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 14:06:51 +0200 Subject: [PATCH 119/371] build step -> build rule "step" sounds atomic, while "rule" hints at internal structure, which in our case consists of mapping inputs to outputs using build instructions. --- doc/manual/src/architecture/architecture.md | 7 ++++--- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index a0aa713f3..81348c570 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build rules*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -37,6 +37,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build plans, to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build rules, data, and references between them. +It can also execute *build instructions*, captured in the build rules, to produce new data. +A series of build rules is a *build plan*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cbb5a4169..b03aa690e 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,10 +2,10 @@ A Nix store is a collection of [store objects](objects.md) with associated operations. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build rules. A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. +It can perform builds, that is, transform build inputs using instructions from the build rules into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. There exist different types of stores, which all follow this model. @@ -28,7 +28,7 @@ generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [B -- | -- | -- | -- | -- data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build rule | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 2a8532fb61ba2b988448e93f5074590e40df7e04 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 May 2022 16:39:26 +0200 Subject: [PATCH 120/371] build rule -> build task closer to "build systems a la carte", satisfies all other complaints --- doc/manual/src/architecture/architecture.md | 10 +++++----- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 81348c570..407e8d0f8 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -19,7 +19,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | | evaluates to | | | | | referenced by V builds | -| [ build input ] ----> [ build plan ] ----> [ build result ] | +| [ build input ] --> [ build task ] --> [ build result ] | | | +-----------------------------------------------------------------+ ``` @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build rules*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build tasks*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -37,7 +37,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build rules, data, and references between them. -It can also execute *build instructions*, captured in the build rules, to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. +It can also execute *build instructions*, captured in the build tasks, to produce new data. -A series of build rules is a *build plan*. +A series of build tasks is a *build plan*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b03aa690e..cc6698a40 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,10 +2,10 @@ A Nix store is a collection of [store objects](objects.md) with associated operations. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build rules. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build tasks. A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build rules into build outputs. +It can perform builds, that is, transform build inputs using instructions from the build tasks into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. There exist different types of stores, which all follow this model. @@ -28,7 +28,7 @@ generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [B -- | -- | -- | -- | -- data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build rule | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build task | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 689b32a543240db992c9e8240401f32bd39e6736 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 12 May 2022 12:30:28 +0200 Subject: [PATCH 121/371] clarify relation of tasks and plans --- doc/manual/src/architecture/architecture.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 407e8d0f8..e8d87f8ae 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -13,21 +13,21 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | evaluates | manages V | [ configuration language ] | - | V -+-------------------------------|---------------------------------+ -| store | | -| | evaluates to | -| | | -| referenced by V builds | -| [ build input ] --> [ build task ] --> [ build result ] | -| | + | | ++-------------------------------|---------------------V-----------+ +| store | evaluates to | +| .............................V............................... | +| : build plan : | +| : referenced by builds : | +| : [ build input ] --> [ build task ] --> [ build result ] : | +| :...........................................................: | +-----------------------------------------------------------------+ ``` At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build tasks*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -40,4 +40,3 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. -A series of build tasks is a *build plan*. From 75ce32405235614235dbe343beb13bb0d17934eb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 May 2022 09:55:24 +0200 Subject: [PATCH 122/371] use singular for class names consistently --- doc/manual/src/SUMMARY.md.in | 6 +++--- doc/manual/src/architecture/store/drvs/drvs.md | 2 +- doc/manual/src/architecture/store/paths.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 257880efe..f356bd07d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,9 +17,9 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Objects](architecture/store/objects.md) - - [Store Paths](architecture/store/paths.md) - - [Derivations](architecture/store/drvs/drvs.md) + - [Store Object](architecture/store/objects.md) + - [Store Path](architecture/store/paths.md) + - [Derivation](architecture/store/drvs/drvs.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md index 766a7b47f..5f374357d 100644 --- a/doc/manual/src/architecture/store/drvs/drvs.md +++ b/doc/manual/src/architecture/store/drvs/drvs.md @@ -1,4 +1,4 @@ -# Derivations +# Derivation Derivations are recipes to create store objects. diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index cf51eb866..402e55e69 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -1,4 +1,4 @@ -# Store Paths +# Store Path A store path is a pair of a 20-byte digest and a name. From 68d26010f6d7341b73675aa30b24b67dcd479a7b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:40:00 +0200 Subject: [PATCH 123/371] architecture overview: add link to Nix expression language reference update summary title to match file contents --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index e8d87f8ae..b818b7aa1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -26,7 +26,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} From ef81276cc161b0ce38cb2681867529c1cbdbd7cb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:40:28 +0200 Subject: [PATCH 124/371] architecture overview: add link to command line reference --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index b818b7aa1..77d5ebc2c 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -24,7 +24,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ +-----------------------------------------------------------------+ ``` -At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. +At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. From 0e63b9bf8872f3556ea8925cea0c639f329d7c6e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 May 2022 19:02:28 +0200 Subject: [PATCH 125/371] add link from overview to store section the overview should only link to the three main concepts presented. the store is now fairly fleshed out. others can follow later. --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 77d5ebc2c..ae977a7e1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,6 +37,6 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. +Underlying everything is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. From 25926c5fc673264288b5d9f7f175178e5b1fdad4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 09:37:28 +0200 Subject: [PATCH 126/371] Nix store does not underly literally everything Co-authored-by: Robert Hensing --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index ae977a7e1..fa992e563 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,6 +37,6 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. +Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. From 2303f84a684e41139a549595edf5d57aada4c685 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 23:46:34 +0200 Subject: [PATCH 127/371] revert to "build plan" in overview diagram this displays correct composition again. build inputs and build results are not part of build plans in terms of data objects. also this is a much less complicated setup. this will be the first impression of architecture, and we want to get it right. --- doc/manual/src/architecture/architecture.md | 29 ++++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index fa992e563..c0fd1b0b5 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -8,19 +8,22 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). ``` - [ commmand line interface ]--------+ - | | - | evaluates | manages - V | - [ configuration language ] | - | | -+-------------------------------|---------------------V-----------+ -| store | evaluates to | -| .............................V............................... | -| : build plan : | -| : referenced by builds : | -| : [ build input ] --> [ build task ] --> [ build result ] : | -| :...........................................................: | ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | | | +| +-------------------------------------------------------------+ | +-----------------------------------------------------------------+ ``` From 4639b36b53fd369024d90e95b0178e4fae0a4203 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 23:56:47 +0200 Subject: [PATCH 128/371] use reference links for URLs --- doc/manual/src/architecture/architecture.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index c0fd1b0b5..4dfe34fd6 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -5,7 +5,7 @@ It should help users understand why Nix behaves as it does, and it should help d ## Overview -Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). +Nix consists of [hierarchical layers][layer-architecture]. ``` +-----------------------------------------------------------------+ @@ -29,7 +29,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} @@ -43,3 +43,5 @@ The command line and Nix language are what users interact with most. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. +[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers +[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From 7c3bca1372aeeb5074fa4038df2984b316e99bb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 21 May 2022 00:16:24 +0200 Subject: [PATCH 129/371] revert to build plans in top-level overview do not introduce build tasks yet, that is the next level of detail. --- doc/manual/src/architecture/architecture.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 4dfe34fd6..6d4b86935 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -30,7 +30,9 @@ Nix consists of [hierarchical layers][layer-architecture]. At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. + +The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -38,10 +40,8 @@ As far as we are concerned here, the inputs and results of a derivation are just In practice this amounts to a set of files in a file system. ::: -The command line and Nix language are what users interact with most. - Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build tasks, to produce new data. +It can also execute build plans to produce new data. [layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers [purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From d5eea66615266e7c341e8d4e982af12d6cb82887 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:52:10 +0200 Subject: [PATCH 130/371] introduce build tasks while it appears a bit much for the overview, this way we set the stage for going directly into data types when describing the store, instead of first having to say what build tasks are and how they relate to build plans. --- doc/manual/src/architecture/architecture.md | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 6d4b86935..8000aecd1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -43,5 +43,37 @@ In practice this amounts to a set of files in a file system. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data. +A build plan is a series of *build tasks*. +Each build task has a special build input which is used as *build instructions*. +The result of a build task can be input to another build task. + +``` ++-----------------------------------------------------------------------------------------+ +| store | +| ................................................. | +| : build plan : | +| : : | +| [ build input ]-----instructions-, : | +| : | : | +| : v : | +| [ build input ]----------->[ build task ]--instructions-, : | +| : | : | +| : | : | +| : v : | +| : [ build task ]----->[ build result ] | +| [ build input ]-----instructions-, ^ : | +| : | | : | +| : v | : | +| [ build input ]----------->[ build task ]---------------' : | +| : ^ : | +| : | : | +| [ build input ]------------------' : | +| : : | +| : : | +| :...............................................: | +| | ++-----------------------------------------------------------------------------------------+ +``` + [layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers [purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From b6b112b366bed363f94231410946ad82122b317c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:28:40 +0200 Subject: [PATCH 131/371] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cc6698a40..d710ca4c0 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -17,21 +17,35 @@ Examples: Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). The store directory defaults to `/nix/store`, but is in principle arbitrary. -## A [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology +## A [Rosetta stone][rosetta-stone] for build system terminology The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. -generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language +generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language -- | -- | -- | -- | -- -data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build task | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) -build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) +data (build input, build result) | store object | [artifact][bazel-artifact] | value | value +build instructions | builder | ([depends on action type][bazel-actions]) | function | function +build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] +build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] build | build | build | application of `Build` | evaluation -persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap +persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap -All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software -Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf) (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) (2018). +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf From e72a7874dc59e7dd9783fb2783cfaf88806bf085 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:29:59 +0200 Subject: [PATCH 132/371] beautify rosetta table while this may eventually introduce ugly diffs, the table will now render readably on the terminal (e.g. for `man nix` or `nix --help`) without further intervention. --- doc/manual/src/architecture/store/store.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d710ca4c0..808e87581 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -24,14 +24,14 @@ Usage of terms is, for historic reasons, not entirely consistent within the Nix The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. -generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language --- | -- | -- | -- | -- -data (build input, build result) | store object | [artifact][bazel-artifact] | value | value -build instructions | builder | ([depends on action type][bazel-actions]) | function | function -build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] -build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] -build | build | build | application of `Build` | evaluation -persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). From 207992a71d5d9c9ee5d09c90f30a9dd35991691d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 13:38:47 +0200 Subject: [PATCH 133/371] introduce store and store objects without file system details this leaves open implementation details, especially about store paths and file system objects, and allows explaining them together were it is more appropriate. also leaves room to carefully introduce the key insight behind Nix: applying results from programming language theory to the operating system paradigm of files and processes. --- doc/manual/src/architecture/store/store.md | 57 ++++++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 808e87581..f421a9ec3 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,12 +1,59 @@ # Store -A Nix store is a collection of [store objects](objects.md) with associated operations. +A Nix store is a collection of *store objects*. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build tasks. +Store objects can hold arbitrary *data* and *references* to one another. +Nix makes no distinction if they are used as build inputs, build results, or build tasks. -A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build tasks into build outputs. -It also keeps track of *references* between data and can therefore garbage-collect unused store objects. +```haskell +data Store = Set StoreObject + +data StoreObject = StoreObject { + data :: Data +, references :: Set Reference +} +``` + +A Nix store can *add*, *retrieve*, and *delete* store objects. + +It can *perform builds*, that is, create new store objects by transforming build inputs, using instructions from the build tasks, into build outputs. + +As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. + +```haskell +add :: Store -> Data -> (Store, Reference) +get :: Store -> Reference -> StoreObject +delete :: Store -> Reference -> Store + +build :: Store -> Reference -> Maybe (Store, Reference) + +collectGarbage :: Store -> Store +``` + +Store objects are [immutable](https://en.m.wikipedia.org/wiki/Immutable_object): once created, they do not change until they are deleted. + +References are [opaque](https://en.m.wikipedia.org/wiki/Opaque_data_type), [unique identifiers](https://en.m.wikipedia.org/wiki/Unique_identifier): +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +An added store object cannot have references, unless it is a build task. + +Building a store object will add appropriate references, according to provided build instructions. +These references can only come from declared build inputs, and are not known by build instructions a priori. + +```haskell +data Data = Data | Task BuildTask + +data BuildTask = BuildTask { + instructions :: Reference +, inputs :: [Reference] +} +``` + +A store object cannot be deleted as long as it is reachable from a reference still in use. +Garbage collection will delete all store objects that cannot be reached from any reference in use. + + There exist different types of stores, which all follow this model. Examples: From b84f2bdfdd21b9aa65cbfac7e0292971d52c40e4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 14:20:56 +0200 Subject: [PATCH 134/371] introduce mapping to Unix files and processes --- doc/manual/src/architecture/architecture.md | 1 - doc/manual/src/architecture/store/store.md | 51 +++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 8000aecd1..3d17074cc 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,7 +37,6 @@ The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. As far as we are concerned here, the inputs and results of a derivation are just data. -In practice this amounts to a set of files in a file system. ::: Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f421a9ec3..d2d64dd53 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -39,7 +39,7 @@ A reference will always point to exactly one store object. An added store object cannot have references, unless it is a build task. Building a store object will add appropriate references, according to provided build instructions. -These references can only come from declared build inputs, and are not known by build instructions a priori. +These references can only come from declared build inputs, and are not known to build instructions a priori. ```haskell data Data = Data | Task BuildTask @@ -55,14 +55,59 @@ Garbage collection will delete all store objects that cannot be reached from any +## Files and Processes + +Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) on the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). + +Nix encodes immutable store objects and opaque identifiers as file system primitives: files, directories, and paths. +That allows processes to resolve references contained in files and thus access the contents of store objects. + +``` ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | ^ | | | +| +---------|----------------------------------------|----------+ | ++-----------|----------------------------------------|------------+ + | | + file system object store path + | | ++-----------|----------------------------------------|------------+ +| operating system +------------+ | | +| '------------ | | <-----------' | +| | file | | +| ,-- | | <-, | +| | +------------+ | | +| execute as | | read, write, execute | +| | +------------+ | | +| '-> | process | --' | +| +------------+ | ++-----------------------------------------------------------------+ +``` + +Store objects are therefore implemented as the pair of + + - a *file system object* for data + - a set of *store paths* for references. + There exist different types of stores, which all follow this model. Examples: - store on the local file system - remote store accessible via SSH - binary cache store accessible via HTTP -Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). -The store directory defaults to `/nix/store`, but is in principle arbitrary. +Every store ultimately has to make store objects accessible to processes through the file system. ## A [Rosetta stone][rosetta-stone] for build system terminology From 4eb11d45928a412643aaf2c2ed40aa7c35cd345c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 00:30:08 +0200 Subject: [PATCH 135/371] fix grammar for clarity --- doc/manual/src/architecture/store/store.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d2d64dd53..99a92bf8e 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -16,7 +16,7 @@ data StoreObject = StoreObject { A Nix store can *add*, *retrieve*, and *delete* store objects. -It can *perform builds*, that is, create new store objects by transforming build inputs, using instructions from the build tasks, into build outputs. +It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. @@ -57,9 +57,9 @@ Garbage collection will delete all store objects that cannot be reached from any ## Files and Processes -Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) on the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). +Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). -Nix encodes immutable store objects and opaque identifiers as file system primitives: files, directories, and paths. +Nix encodes immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. ``` From 4adb6602bdf274f1b0471d8a7ab53e80eaff4854 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:58:25 +0200 Subject: [PATCH 136/371] clarify first sentence on store objects --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 99a92bf8e..18d25ca4d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,8 +2,8 @@ A Nix store is a collection of *store objects*. -Store objects can hold arbitrary *data* and *references* to one another. -Nix makes no distinction if they are used as build inputs, build results, or build tasks. +A store object can hold arbitrary *data* and *references* to other store objects. +Nix makes no distinction if store objects are used as build inputs, build results, or build tasks. ```haskell data Store = Set StoreObject From db8703bcac78ed2d132e39fe9bc13a2cd0fe6efc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 03:05:44 +0200 Subject: [PATCH 137/371] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 18d25ca4d..1640944ad 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -18,7 +18,7 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. -As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. +As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. ```haskell add :: Store -> Data -> (Store, Reference) @@ -30,9 +30,9 @@ build :: Store -> Reference -> Maybe (Store, Reference) collectGarbage :: Store -> Store ``` -Store objects are [immutable](https://en.m.wikipedia.org/wiki/Immutable_object): once created, they do not change until they are deleted. +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -References are [opaque](https://en.m.wikipedia.org/wiki/Opaque_data_type), [unique identifiers](https://en.m.wikipedia.org/wiki/Unique_identifier): +References are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. @@ -55,6 +55,11 @@ Garbage collection will delete all store objects that cannot be reached from any +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + ## Files and Processes Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). From 445f753a820cb4b6076fec1d69d66c649618c4f0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 03:40:17 +0200 Subject: [PATCH 138/371] replace pseudo code by diagrams change prose description to visually resemble the data structure --- doc/manual/src/architecture/store/store.md | 57 ++++++++++++---------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 1640944ad..5a0f4a2df 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,33 +2,49 @@ A Nix store is a collection of *store objects*. -A store object can hold arbitrary *data* and *references* to other store objects. -Nix makes no distinction if store objects are used as build inputs, build results, or build tasks. +A store object can hold -```haskell -data Store = Set StoreObject +- arbitrary *data* +- *references* to other store objects. -data StoreObject = StoreObject { - data :: Data -, references :: Set Reference -} -``` +Nix makes no distinction if store objects are build inputs, build results, or build tasks. A Nix store can *add*, *retrieve*, and *delete* store objects. + [ data ] + | + V + [ store ] ---> add ----> [ store' ] [ reference ] + + + + [ reference ] + | + V + [ store ] ---> get ----> [ store object ] + + + + [ reference ] + | + V + [ store ] --> delete --> [ store' ] + + It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. + + [ reference ] + | + V + [ store ] --> build --(maybe)--> [ store' ] [ reference' ] + + As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. -```haskell -add :: Store -> Data -> (Store, Reference) -get :: Store -> Reference -> StoreObject -delete :: Store -> Reference -> Store -build :: Store -> Reference -> Maybe (Store, Reference) + [ store ] --> collect garbage --> [ store' ] -collectGarbage :: Store -> Store -``` Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. @@ -41,15 +57,6 @@ An added store object cannot have references, unless it is a build task. Building a store object will add appropriate references, according to provided build instructions. These references can only come from declared build inputs, and are not known to build instructions a priori. -```haskell -data Data = Data | Task BuildTask - -data BuildTask = BuildTask { - instructions :: Reference -, inputs :: [Reference] -} -``` - A store object cannot be deleted as long as it is reachable from a reference still in use. Garbage collection will delete all store objects that cannot be reached from any reference in use. From 4341849193cfd9b8e1e494292516a60734dfcc53 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:15:53 +0200 Subject: [PATCH 139/371] move closure property to discussion references --- doc/manual/src/architecture/store/objects.md | 15 ------------- doc/manual/src/architecture/store/store.md | 23 +++++++++++++++----- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index f7587e112..8ab0b9368 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -35,21 +35,6 @@ A bare file or symlink can be a root file system object. Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. -## Reference {#reference} - -A store object can reference other store objects. - -Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. - -Building, copying and deleting store objects must be done in a way that obeys this property: - -- Build results must only refer to store objects in the closure of the build inputs. - -- Store objects being copied must refer to objects already in the destination store. - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete unreferenced objects. - ### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5a0f4a2df..a3d9e2eeb 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -52,15 +52,26 @@ References are [opaque][opaque-data-type], [unique identifiers][unique-identifie The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -An added store object cannot have references, unless it is a build task. +Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. -Building a store object will add appropriate references, according to provided build instructions. -These references can only come from declared build inputs, and are not known to build instructions a priori. +Adding, building, copying and deleting store objects must be done in a way that obeys this property: -A store object cannot be deleted as long as it is reachable from a reference still in use. -Garbage collection will delete all store objects that cannot be reached from any reference in use. +- A newly added store object cannot have references, unless it is a build task. - +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to provided build instructions. + These references can only come from declared build inputs. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + Garbage collection will delete all store objects that cannot be reached from any reference in use. + + [garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) [immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object From 843288a451461610f67798af2598a2034668153a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:16:33 +0200 Subject: [PATCH 140/371] add subsections for objects and references group description of data instead of spreading it across the section. that should help direct skimming. as it turns out, people do not actually read any of that. --- doc/manual/src/architecture/store/store.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index a3d9e2eeb..de12ac1dc 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,6 +2,8 @@ A Nix store is a collection of *store objects*. +## Store Object + A store object can hold - arbitrary *data* @@ -9,6 +11,16 @@ A store object can hold Nix makes no distinction if store objects are build inputs, build results, or build tasks. +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. + +## Reference + +References to store objects are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +## Operations + A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] @@ -46,11 +58,7 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. - -References are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: -The only way to obtain references is by adding or building store objects. -A reference will always point to exactly one store object. +## Closure Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. From e63a768e21d9bff605e51de54f9ec1a8f74650a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:18:07 +0200 Subject: [PATCH 141/371] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index de12ac1dc..710d3265d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -88,11 +88,12 @@ Adding, building, copying and deleting store objects must be done in a way that ## Files and Processes -Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). - -Nix encodes immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. +Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. +[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file +[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor + ``` +-----------------------------------------------------------------+ | Nix | From 7b7e4c6340513efb838aa576d95c5032a90dc50d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:20:21 +0200 Subject: [PATCH 142/371] use singular to match section heading --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 710d3265d..7367b4626 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -15,7 +15,7 @@ Store objects are [immutable][immutable-object]: once created, they do not chang ## Reference -References to store objects are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: +A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. From 3794618ccecf56a468eb9befd351e3cd39595ab5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:11:58 +0200 Subject: [PATCH 143/371] add commas between output values --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 7367b4626..6ac210856 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,7 +26,7 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] | V - [ store ] ---> add ----> [ store' ] [ reference ] + [ store ] ---> add ----> [ store' ], [ reference ] @@ -49,7 +49,7 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build --(maybe)--> [ store' ] [ reference' ] + [ store ] --> build --(maybe)--> [ store' ], [ reference' ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From 80de4a4804bf16fe3266618cb79ad486dc6659ae Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:20:11 +0200 Subject: [PATCH 144/371] operations diagram: store' to the right this also looks more diverse, hopefully easier to distinguish Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 6ac210856..33777dc0d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,14 +26,20 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] | V - [ store ] ---> add ----> [ store' ], [ reference ] + [ store ] ---> add ----> [ store' ] + | + V + [ reference ] [ reference ] | V - [ store ] ---> get ----> [ store object ] + [ store ] ---> get + | + V + [ store object ] @@ -49,7 +55,12 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build --(maybe)--> [ store' ], [ reference' ] + [ store ] --> build + \ + (maybe) --> [ store' ] + | + V + [ reference ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From 195aa28ff73ee21bfe2d398e570bf458f07f6757 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:29:32 +0200 Subject: [PATCH 145/371] references are added according to build task at this level of abstraction we do not really care about build instructions or what they are, and also build instructions including their arguments really amount to the build task. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 33777dc0d..aff41d511 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -79,7 +79,7 @@ Adding, building, copying and deleting store objects must be done in a way that - Build results must only refer to store objects in the closure of the build inputs. - Building a store object will add appropriate references, according to provided build instructions. + Building a store object will add appropriate references, according to the build task. These references can only come from declared build inputs. - Store objects being copied must refer to objects already in the destination store. From 7993ba1f388e509d7617808b4a947ed1bf1533a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:34:24 +0200 Subject: [PATCH 146/371] constrain garbage collection scope garbage collection is now incremental, and may (in theory) never delete all unreferenced objects if it is slow enough. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index aff41d511..5b4c3072d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -88,7 +88,7 @@ Adding, building, copying and deleting store objects must be done in a way that - We can only safely delete store objects which are not reachable from any reference still in use. - Garbage collection will delete all store objects that cannot be reached from any reference in use. + Garbage collection will delete those store objects that cannot be reached from any reference in use. From a90fc62b618f571bc5e6512e66e9c8c9a8eb6e78 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:50:51 +0200 Subject: [PATCH 147/371] make clear that file system is for processes Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5b4c3072d..3d1ad0bad 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -150,7 +150,7 @@ Examples: - remote store accessible via SSH - binary cache store accessible via HTTP -Every store ultimately has to make store objects accessible to processes through the file system. +To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. ## A [Rosetta stone][rosetta-stone] for build system terminology From 19d8a5d83942f13c3f532b4ac21cc91c7158c21a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:45:04 +0200 Subject: [PATCH 148/371] move first mention of file system object before diagram --- doc/manual/src/architecture/store/store.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 3d1ad0bad..a22735e42 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -102,6 +102,11 @@ Adding, building, copying and deleting store objects must be done in a way that Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. +Store objects are therefore implemented as the pair of + + - a *file system object* for data + - a set of *store paths* for references. + [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor @@ -139,11 +144,6 @@ That allows processes to resolve references contained in files and thus access t +-----------------------------------------------------------------+ ``` -Store objects are therefore implemented as the pair of - - - a *file system object* for data - - a set of *store paths* for references. - There exist different types of stores, which all follow this model. Examples: - store on the local file system From 93f721b0d26b6525d8eaa8a1606ebeb4b71a9b6a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:36:29 +0200 Subject: [PATCH 149/371] remove draft on derivations --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/drvs/drvs.md | 59 ------------------- 2 files changed, 60 deletions(-) delete mode 100644 doc/manual/src/architecture/store/drvs/drvs.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f356bd07d..5e2d26bf7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,7 +19,6 @@ - [Store](architecture/store/store.md) - [Store Object](architecture/store/objects.md) - [Store Path](architecture/store/paths.md) - - [Derivation](architecture/store/drvs/drvs.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md deleted file mode 100644 index 5f374357d..000000000 --- a/doc/manual/src/architecture/store/drvs/drvs.md +++ /dev/null @@ -1,59 +0,0 @@ -# Derivation - -Derivations are recipes to create store objects. - -Derivations are the heart of Nix. -Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. -This is where Nix comes in. - -Derivations produce data by running arbitrary commands, like Make or Ninja rules. -Unlike those systems, derivations do not produce arbitrary files, but only specific store objects. -They cannot modify the store in any way, other than creating those store objects. -This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. - -Based on the above, we can conceptually break derivations down into 3 parts: - -1. What command will be run? - -2. What existing store objects are needed as inputs? - -3. What store objects will be produced as outputs? - -## What command will be run? - -The original core of Nix was very simple about this, in the mold of traditional Unix. -Commands consist of 3 parts: - -1. Path to executable - -2. Arguments (Except for `argv[0]`, which is taken from the path in the usual way) - -3. Environment variables. - -## What existing store objects are needed as inputs? - -The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" -It's a good questions! - -## What store objects will be produced as outputs? - -## Extra extensions - -### `__structuredAttrs` - -Historically speaking, most users of Nix made GNU Bash with a script the command run, regardless of what they were doing. -Bash variable are automatically created from env vars, but bash also supports array and string-keyed map variables in addition to string variables. -People also usually create derivations using language which also support these richer data types. -It was thus desired a way to get this data from the language "planning" the derivation to language to bash, the language evaluated at "run time". - -`__structuredAttrs` does this by smuggling inside the core derivation format a map of named richer data. -At run time, this becomes two things: - -1. A JSON file containing that map. -2. A bash script setting those variables. - -The bash command can be passed a script which will "source" that Nix-created bash script, setting those variables with the richer data. -The outer script can then do whatever it likes with those richer variables as input. - -However, since derivations can already contain arbitary input sources, the vast majority of `__structuredAttrs` can be handled by upper layers. -We might consider implementing `__structuredAttrs` in higher layers in the future, and simplifying the store layer. From 84ddfbf8fd9b1e77fefb3c2c11a9cedab2e530c2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:57:19 +0200 Subject: [PATCH 150/371] remove diagonal from operations diagram --- doc/manual/src/architecture/store/store.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index a22735e42..3b99982e4 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -55,12 +55,10 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build - \ - (maybe) --> [ store' ] - | - V - [ reference ] + [ store ] --> build --(maybe)--> [ store' ] + | + V + [ reference ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From f632816cbaefbba9cc27a8e0de6cffd39fa7a8dd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:09:27 +0200 Subject: [PATCH 151/371] add explanation and examples of file system objects --- doc/manual/src/architecture/store/fso.md | 64 ++++++++++++++++++++++ doc/manual/src/architecture/store/store.md | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 doc/manual/src/architecture/store/fso.md diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md new file mode 100644 index 000000000..958b8f62a --- /dev/null +++ b/doc/manual/src/architecture/store/fso.md @@ -0,0 +1,64 @@ +# File System Object + +The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object). + +Every file system object is one of the following: + + - File: an executable flag, and arbitrary data for contents + - Directory: mapping of names to child file system objects + - [Symbolic link][symlink]: may point anywhere. + +We call a store object's outermost file system object the *root*. + + data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } + +Examples: + +- a directory with contents + + /nix/store/-hello-2.10 + ├── bin + │   └── hello + └── share + ├── info + │   └── hello.info + └── man + └── man1 + └── hello.1.gz + +- a directory with relative symlink and other contents + + /nix/store/-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ + +- a directory with absolute symlink + + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. + +A bare file or symlink can be a root file system object. +Examles: + + /nix/store/-hello-2.10.tar.gz + + /nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc + +Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended. +Examples: + +- an arbitrarily symlinked file may change or not exist at all + + /nix/store/-foo + └── foo -> /home/foo + +- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted + + /nix/store/-bar + └── bar -> /nix/store/abc...-foo + +[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 3b99982e4..68bdadc4a 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,7 +2,7 @@ A Nix store is a collection of *store objects*. -## Store Object +## Store Object {#store-object} A store object can hold @@ -102,7 +102,7 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - - a *file system object* for data + - a [file system object](fso.md) for data - a set of *store paths* for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file From fa7ad4593d09d04afe1d215d2da09be02c2e2836 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 11:07:50 +0200 Subject: [PATCH 152/371] explain store directory --- doc/manual/src/SUMMARY.md.in | 6 +- doc/manual/src/architecture/store/objects.md | 48 ------- doc/manual/src/architecture/store/paths.md | 129 +++++++++++-------- doc/manual/src/architecture/store/store.md | 21 +-- 4 files changed, 94 insertions(+), 110 deletions(-) delete mode 100644 doc/manual/src/architecture/store/objects.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5e2d26bf7..997d75444 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,8 +17,10 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Object](architecture/store/objects.md) - - [Store Path](architecture/store/paths.md) + - [Store Path](architecture/store/path.md) + - [Digest](architecture/store/path.md#digest) + - [Input Addressing](architecture/store/path.md#input-addressing) + - [Content Addressing](architecture/store/path.md#content-addressing) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md deleted file mode 100644 index 8ab0b9368..000000000 --- a/doc/manual/src/architecture/store/objects.md +++ /dev/null @@ -1,48 +0,0 @@ -# Store Object - -Nix organizes the data it manages into *store objects*. -A store object is the pair of - - - a [file system object](#file-system-object) - - a set of [references](#reference) to store objects. - -We call a store object's outermost file system object the *root*. - -```haskell -data StoreOject = StoreObject { - root :: FileSystemObject -, references :: Set StoreObject -} -``` - -## File system object {#file-system-object} - -The Nix store uses a simple file system model. - -Every file system object is one of the following: - - File: an executable flag, and arbitrary data for contents - - Directory: mapping of names to child file system objects - - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - -```haskell -data FileSystemObject - = File { isExecutable :: Bool, contents :: Bytes } - | Directory { entries :: Map FileName FileSystemObject } - | SymLink { target :: Path } -``` - -A bare file or symlink can be a root file system object. - -Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. - -### Reference scanning - -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. -Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. - -However, having references match store paths in files is not enforced by the data model: -Store objects could have excess or incomplete references with respect to store paths found in their file contents. - -Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. -Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. - diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index 402e55e69..4867e7fd3 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -1,78 +1,103 @@ # Store Path -A store path is a pair of a 20-byte digest and a name. +Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*. -## String representation +Store paths are pairs of -A store path is rendered as the concatenation of +- a 20-byte [digest](#digest) for identification +- a symbolic name for people to read. - - a store directory - - - a path-separator (`/`) - - - the digest rendered as Base-32 (20 arbitrary bytes becomes 32 ASCII chars) - - - a hyphen (`-`) - - - the name - -Let's take the store path from the very beginning of this manual as an example: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - -This parses like so: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ - store dir digest name - -We then can discard the store dir to recover the conceptual pair that is a store path: +Example: { digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", name: "firefox-33.1", } -### Where did the "store directory" come from? +It is rendered to a file system path as the concatenation of -If you notice, the above references a "store directory", but that is *not* part of the definition of a store path. -We can discard it when parsing, but what about when printing? -We need to get a store directory from *somewhere*. + - [store directory](#store-directory) + - path-separator (`/`) + - [digest](#digest) rendered in [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - hyphen (`-`) + - name -The answer is, the store directory is a property of the store that contains the store path. -The explanation for this is simple enough: a store is notionally mounted as a directory at some location, and the store object's root file system likewise mounted at this path within that directory. +Example: -This does, however, mean the string representation of a store path is not derived just from the store path itself, but is in fact "context dependent". + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 + |--------| |------------------------------| |----------| + store directory digest name -## The digest +## Store Directory {#store-directory} -The calculation of the digest is quite complicated for historical reasons. -The details of the algorithms will be discussed later once more concepts have been introduced. -For now, we just concern ourselves with the *key properties* of those algorithms. +Every [store](./store.md) has a store directory. + +If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path). + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. ::: {.note} -**Historical note** The 20 byte restriction is because originally a digests were SHA-1 hashes. -This is no longer true, but longer hashes and other information are still boiled down to 20 bytes. +The store directory defaults to `/nix/store`, but is in principle arbitrary. ::: -Store paths are either *content-addressed* or *input-addressed*. +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. + +Therefore one can only copy store objects if + +- the source and target stores' directories match + + or + +- the store object in question has no references, that is, contains no store paths. + +To move a store object to a store with a different store directory, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may break internal offsets or content hashes. + +# Digest {#digest} + +In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. + +Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). ::: {.note} -The former is a standard term used elsewhere. -The later is our own creation to evoke a contrast with content addressing. +**Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +This is no longer true, but longer hashes and other information are still truncated to 20 bytes for compatibility. ::: -Content addressing means that the store path digest ultimately derives from referred store object's contents, namely its file system objects and references. -There is more than one *method* of content-addressing, however. -Still, if one does know the content addressing schema that was used, -(or guesses, there isn't that many yet!) -one can recalculate the store path and thus verify the store object. +[digest]: https://en.m.wiktionary.org/wiki/digest#Noun +[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function +[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 -Input addressing means that the store path digest derives from how the store path was produced, namely the "inputs" and plan that it was built from. -Store paths of this sort can *not* be validated from the content of the store object. -Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. -The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. -While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. -That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. -In the parlance of referencing schemes, this means that store paths are not "self-describing". +### Reference scanning + +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. +Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. + +However, having references match store paths in files is not enforced by the data model: +Store objects could have excess or incomplete references with respect to store paths found in their file contents. + +Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. + +## Input Addressing {#input-addressing} + +Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. + +To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes. + +Nix has a custom serialisation format called Nix Archive (NAR) + +Store object references of this sort can *not* be validated from the content of the store object. +Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. + +## Content Addressing {#content-addressing} + +Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. +If one knows content addressing was used, one can recalculate the reference and thus verify the store object. + +Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance. +Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md). + diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 68bdadc4a..21a876f75 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -67,18 +67,19 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -## Closure +## Closure {#closure} -Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. -Adding, building, copying and deleting store objects must be done in a way that obeys this property: +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - A newly added store object cannot have references, unless it is a build task. - Build results must only refer to store objects in the closure of the build inputs. Building a store object will add appropriate references, according to the build task. - These references can only come from declared build inputs. - Store objects being copied must refer to objects already in the destination store. @@ -86,16 +87,15 @@ Adding, building, copying and deleting store objects must be done in a way that - We can only safely delete store objects which are not reachable from any reference still in use. - Garbage collection will delete those store objects that cannot be reached from any reference in use. - +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity [garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) [immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object [opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -## Files and Processes +## Files and Processes {#files-and-processes} Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. @@ -103,11 +103,16 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - a [file system object](fso.md) for data - - a set of *store paths* for references. + - a set of [store paths](paths.md) for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor +The following diagram shows a radical simplification of how Nix interacts with the operating system: +It uses files as build inputs, and build outputs are files again. +On the operating system, files are either "dead" data, or "live" as processes, which in turn operate on files, or can bring them to life. +A build function also amounts to an operating system process (not depicted). + ``` +-----------------------------------------------------------------+ | Nix | From 1681f4e9f3608973e3247e447fd9236eb32d66f3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 13 Jun 2022 17:10:31 +0200 Subject: [PATCH 153/371] better explain reference scanning --- doc/manual/src/architecture/store/paths.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index 4867e7fd3..956049341 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -70,18 +70,21 @@ This is no longer true, but longer hashes and other information are still trunca [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function [sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 - ### Reference scanning -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. -Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. +When a new store object is built, Nix scans its file contents for store paths to construct its set of references. -However, having references match store paths in files is not enforced by the data model: -Store objects could have excess or incomplete references with respect to store paths found in their file contents. +The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data. +Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives. -Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +This way, scanning files captures run time dependencies without the user having to declare them explicitly. Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. +::: {.note} +In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +This depends on the specifics of the software to build and run. +::: + ## Input Addressing {#input-addressing} Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. From 9c544813d7883fa003d25f528f50c46faf06e1a2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:01:31 +0200 Subject: [PATCH 154/371] paths -> path use singular for terminology uniformly --- doc/manual/src/architecture/store/{paths.md => path.md} | 0 doc/manual/src/architecture/store/store.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename doc/manual/src/architecture/store/{paths.md => path.md} (100%) diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/path.md similarity index 100% rename from doc/manual/src/architecture/store/paths.md rename to doc/manual/src/architecture/store/path.md diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 21a876f75..f662ed3e7 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -103,7 +103,7 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - a [file system object](fso.md) for data - - a set of [store paths](paths.md) for references. + - a set of [store paths](path.md) for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor From c10dcccccda331499db75753ab6fed8938c31cca Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:08:13 +0200 Subject: [PATCH 155/371] make example a simple list using JSON notation is unwarranted and not explained. --- doc/manual/src/architecture/store/path.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 956049341..2cedc4234 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -9,10 +9,8 @@ Store paths are pairs of Example: - { - digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", - name: "firefox-33.1", - } +- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +- name: `firefox-33.1` It is rendered to a file system path as the concatenation of From 631ca18ee62502dd2e51a8f3c9e2ab656b8382ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:08:50 +0200 Subject: [PATCH 156/371] reword notes on copying clarify that we are copying between different stores. we have not introduced that notion or why it would be interesting, but for now it should be fine to keep it in context of the store directory. we could move that later to a more detailed explanation of different store types. --- doc/manual/src/architecture/store/path.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 2cedc4234..3f2a4bbff 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -42,7 +42,7 @@ It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. -Therefore one can only copy store objects if +Therefore one can only copy store objects to a different store if - the source and target stores' directories match @@ -50,8 +50,9 @@ Therefore one can only copy store objects if - the store object in question has no references, that is, contains no store paths. -To move a store object to a store with a different store directory, it has to be rebuilt, together with all its dependencies. -It is in general not enough to replace the store directory string in file contents, as this may break internal offsets or content hashes. +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. # Digest {#digest} From 7c656d90ca2c3364fd72543c0aa27115d96ac463 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:25:30 +0200 Subject: [PATCH 157/371] simplify description of diagram try not to be too fancy, it's just for reading the diagram out loud. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f662ed3e7..b3517f9b6 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -110,7 +110,7 @@ Store objects are therefore implemented as the pair of The following diagram shows a radical simplification of how Nix interacts with the operating system: It uses files as build inputs, and build outputs are files again. -On the operating system, files are either "dead" data, or "live" as processes, which in turn operate on files, or can bring them to life. +On the operating system, files can be run as processes, which in turn operate on files. A build function also amounts to an operating system process (not depicted). ``` From ec43977553e605370d160ad0f852c15d0c754e74 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:10:44 +0200 Subject: [PATCH 158/371] store: match chapter introduction to outline the various levels of detail should describe the same things. --- doc/manual/src/architecture/store/store.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b3517f9b6..403a23bbd 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,6 +1,7 @@ # Store -A Nix store is a collection of *store objects*. +A Nix store is a collection of *store objects* with references between them. +It supports operations to manipulate that collection. ## Store Object {#store-object} From 348432f48ffaf084c7ba1397fe63d805891bb68d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:17:28 +0200 Subject: [PATCH 159/371] store: add concept map this should help nativate the chapter by indicating which terms should be known to understand a given concept. --- doc/manual/src/architecture/store/store.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 403a23bbd..ed250ce0c 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -3,6 +3,28 @@ A Nix store is a collection of *store objects* with references between them. It supports operations to manipulate that collection. +The following concept map is a graphical outline of this chapter. +Arrows indicate suggested reading order. + +``` + ,----------------------[ store ] + | | + v v + [ store object ] [ operations ]----------------------, + | | | + v v | + [ files and processes ] [ build ]--------->[ derivation ] | + | | | | | + v v v v | + [ file system object ] [ store path ]--->[ reference scanning ]--->[ closure ] | + | ^ | | + v | v v + [ digest ]--------------' [ garbage collection ] + / \ + v v +[ input addressing ] [ content addressing ] +``` + ## Store Object {#store-object} A store object can hold From d8b2f9f772a7fb33db17f98605f221f5254c51b4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:21:12 +0200 Subject: [PATCH 160/371] make concept map more compact invert arrows to/from derivation: - we need closures to form derivations - we need derivations to perform builds --- doc/manual/src/architecture/store/store.md | 35 ++++++++++++---------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index ed250ce0c..fba2f90fd 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -7,22 +7,25 @@ The following concept map is a graphical outline of this chapter. Arrows indicate suggested reading order. ``` - ,----------------------[ store ] - | | - v v - [ store object ] [ operations ]----------------------, - | | | - v v | - [ files and processes ] [ build ]--------->[ derivation ] | - | | | | | - v v v v | - [ file system object ] [ store path ]--->[ reference scanning ]--->[ closure ] | - | ^ | | - v | v v - [ digest ]--------------' [ garbage collection ] - / \ - v v -[ input addressing ] [ content addressing ] + ,----------[ store ]---------, + | | + | v + | [ operations ] + | / \ + v v v + [ store object ] [ garbage collection ] [ build ] + | ^ ^ | + v | | | + [ files and processes ] | [ derivation ]--' | + / \ | ^ | + v v | | | +[ file system object ] [ store path ] '--[ closure ] | + | ^ \ | | + v | v v | + [ digest ]-----' [ reference scanning ]<------' + / \ + v v +[ input addressing ] [ content addressing ] ``` ## Store Object {#store-object} From 475a332025fd9bc399206f79298d9897dde45380 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:25:19 +0200 Subject: [PATCH 161/371] make concept map even more compact --- doc/manual/src/architecture/store/store.md | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index fba2f90fd..c4eea33fa 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -11,18 +11,18 @@ Arrows indicate suggested reading order. | | | v | [ operations ] - | / \ - v v v - [ store object ] [ garbage collection ] [ build ] - | ^ ^ | - v | | | - [ files and processes ] | [ derivation ]--' | - / \ | ^ | - v v | | | -[ file system object ] [ store path ] '--[ closure ] | - | ^ \ | | - v | v v | - [ digest ]-----' [ reference scanning ]<------' + | | | + v v | + [ store object ] [ garbage collection ] | + | ^ | + v | v + [ files and processes ] [ closure ] [ build ] + / \ | | ^ | + v v | v | | +[ file system object ] [ store path ] | [ derivation ] | + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ] <----' / \ v v [ input addressing ] [ content addressing ] From a28d6878b263270c7db271c414bef2c688bb872d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:27:08 +0200 Subject: [PATCH 162/371] concept map: put closure as it is in the chapter this is not as compact any more, but it more closely resembles the chapter structure, and clearly shows that the closure property is the key idea on which most of Nix operates. --- doc/manual/src/architecture/store/store.md | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index c4eea33fa..0f43d5a62 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -7,22 +7,19 @@ The following concept map is a graphical outline of this chapter. Arrows indicate suggested reading order. ``` - ,----------[ store ]---------, - | | - | v - | [ operations ] - | | | - v v | - [ store object ] [ garbage collection ] | - | ^ | - v | v - [ files and processes ] [ closure ] [ build ] - / \ | | ^ | - v v | v | | -[ file system object ] [ store path ] | [ derivation ] | - | ^ | | | - v | v v | - [ digest ]----' [ reference scanning ] <----' + ,--------------[ store ]----------------, + | | | + v v v + [ store object ] [ closure ]--, [ operations ] + | | | | | | + v | | v v | + [ files and processes ] | | [ garbage collection ] | + / \ | V v + v v | [ derivation ]--->[ building ] +[ file system object ] [ store path ] | | + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ] <-----------' / \ v v [ input addressing ] [ content addressing ] From c345345deeb188830e62e9b2e957cf1d09d1c6b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:33:31 +0200 Subject: [PATCH 163/371] concept map: align hights this has the weird but nice emergent property that terms at the same height are roughly at the same level of abstraction. --- doc/manual/src/architecture/store/store.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 0f43d5a62..cba875092 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -14,12 +14,12 @@ Arrows indicate suggested reading order. | | | | | | v | | v v | [ files and processes ] | | [ garbage collection ] | - / \ | V v - v v | [ derivation ]--->[ building ] -[ file system object ] [ store path ] | | + / \ | | | + v v | v v +[ file system object ] [ store path ] | [ derivation ]--->[ building ] | ^ | | | v | v v | - [ digest ]----' [ reference scanning ] <-----------' + [ digest ]----' [ reference scanning ]<------------' / \ v v [ input addressing ] [ content addressing ] From def80d5777dcd5d4b8a10e54be24d745b5d7562b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:50:24 +0200 Subject: [PATCH 164/371] add subsections to table of contents --- doc/manual/src/SUMMARY.md.in | 8 +++++++- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 997d75444..5581bad3a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,7 +17,13 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Path](architecture/store/path.md) + - [Store Object](architecture/store/store.md#store-object) + - [Reference](architecture/store/store.md#reference) + - [Operations](architecture/store/store.md#operations) + - [Closure](architecture/store/store.md#closure) + - [Files and Processes](architecture/store/store.md#files-and-processes) + - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Store Path](architecture/store/path.md) - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) - [Content Addressing](architecture/store/path.md#content-addressing) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cba875092..e1d983d99 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -36,13 +36,13 @@ Nix makes no distinction if store objects are build inputs, build results, or bu Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -## Reference +## Reference {#reference} A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -## Operations +## Operations {#operations} A Nix store can *add*, *retrieve*, and *delete* store objects. @@ -178,7 +178,7 @@ Examples: To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. -## A [Rosetta stone][rosetta-stone] for build system terminology +## A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. From fe4c0b8d75b9380ab64a752a7f1da680b1163731 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 12 Jul 2022 12:28:09 +0100 Subject: [PATCH 165/371] fix typo Co-authored-by: Thomas --- doc/manual/src/architecture/store/fso.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index 958b8f62a..c51996e41 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -42,7 +42,7 @@ Examples: └── nix_node -> /nix/store/f20...-nodejs-10.24. A bare file or symlink can be a root file system object. -Examles: +Examples: /nix/store/-hello-2.10.tar.gz From de5dea45f7443dd14068e6289e8844816f445c35 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 09:37:16 +0100 Subject: [PATCH 166/371] use correct Nix entity Nix itself does care a lot about what type of store object you have. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index e1d983d99..5ac512621 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -32,7 +32,7 @@ A store object can hold - arbitrary *data* - *references* to other store objects. -Nix makes no distinction if store objects are build inputs, build results, or build tasks. +A Nix store makes no distinction if store objects are build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. From 5a5a95696f4728010486ee01b375b861acc2fddd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 10:42:21 +0100 Subject: [PATCH 167/371] note customized base32 Nix omits E O U T characters for some reason. --- doc/manual/src/architecture/store/path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 3f2a4bbff..9f47b877f 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -16,7 +16,7 @@ It is rendered to a file system path as the concatenation of - [store directory](#store-directory) - path-separator (`/`) - - [digest](#digest) rendered in [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) - hyphen (`-`) - name From bac86231add178e4def3e3fe0bc88ace83d86c21 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 10:59:09 +0100 Subject: [PATCH 168/371] use "build plan" consistently --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 3d17074cc..d9fb2e7e5 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -36,7 +36,7 @@ The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a derivation are just data. +As far as we are concerned here, the inputs and results of a build plan are just data. ::: Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. From 9cabba1fc35c4989b0e280c12a845adf0e300dab Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:17:46 +0100 Subject: [PATCH 169/371] mention hard links Co-authored-by: Thomas --- doc/manual/src/architecture/store/fso.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index c51996e41..2ec365eb6 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -61,4 +61,9 @@ Examples: /nix/store/-bar └── bar -> /nix/store/abc...-foo +Nix file system objects do not support [hard links][hardlink]: +each file system object which is not the root has exactly one parent and one name. +However, as store objects are immutable, an underlying file system can use hard links for optimization. + [symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link +[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link From 29c062537d37fc10997767edd3edb6af41f1fb00 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:42:16 +0100 Subject: [PATCH 170/371] hashes: truncate -> reduce, mention SHA-256 the longer SHA-256 hashes are not truncated, but in fact processed. Co-authored-by: Thomas --- doc/manual/src/architecture/store/path.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 9f47b877f..64bc4a522 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -62,12 +62,13 @@ Store objects are therefore said to be either [input-addressed](#input-addressin ::: {.note} **Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -This is no longer true, but longer hashes and other information are still truncated to 20 bytes for compatibility. +Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. ::: [digest]: https://en.m.wiktionary.org/wiki/digest#Noun [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function [sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 +[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Reference scanning From 0228eb8ae76d048b16caffff9628aec347ff1485 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:53:59 +0100 Subject: [PATCH 171/371] add Java example on manual dependency declaration --- doc/manual/src/architecture/store/path.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 64bc4a522..7d793749b 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -83,6 +83,8 @@ Doing it at build time and persisting references in the store object avoids repe ::: {.note} In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. + +For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. ::: ## Input Addressing {#input-addressing} From db6faf44a9befeaabbbd1bb025b3f5a7d743dd16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 23:23:43 +0200 Subject: [PATCH 172/371] clarify what store objects can be the store of course makes a distinction, but that is not relevant here --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5ac512621..920d483d5 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -32,7 +32,7 @@ A store object can hold - arbitrary *data* - *references* to other store objects. -A Nix store makes no distinction if store objects are build inputs, build results, or build tasks. +Store objects can be build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. From 00a7eaed6759c284444c161c69a3a585579a7d5f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:30:27 +0200 Subject: [PATCH 173/371] add file system object to table of contents --- doc/manual/src/SUMMARY.md.in | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5581bad3a..d589156ec 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -27,6 +27,7 @@ - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) - [Content Addressing](architecture/store/path.md#content-addressing) + - [File System Object](architecture/store/fso.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) From b7309ce624d17a2dbe97f63700d59830c35b074a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:31:24 +0200 Subject: [PATCH 174/371] move architecture to the end --- doc/manual/src/SUMMARY.md.in | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index d589156ec..9c2ae2ad3 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,19 +15,6 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Architecture](architecture/architecture.md) - - [Store](architecture/store/store.md) - - [Store Object](architecture/store/store.md#store-object) - - [Reference](architecture/store/store.md#reference) - - [Operations](architecture/store/store.md#operations) - - [Closure](architecture/store/store.md#closure) - - [Files and Processes](architecture/store/store.md#files-and-processes) - - [Build system terminology](architecture/store/store.md#build-system-terminology) - - [Store Path](architecture/store/path.md) - - [Digest](architecture/store/path.md#digest) - - [Input Addressing](architecture/store/path.md#input-addressing) - - [Content Addressing](architecture/store/path.md#content-addressing) - - [File System Object](architecture/store/fso.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) @@ -79,6 +66,19 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) +- [Architecture](architecture/architecture.md) + - [Store](architecture/store/store.md) + - [Store Object](architecture/store/store.md#store-object) + - [Reference](architecture/store/store.md#reference) + - [Operations](architecture/store/store.md#operations) + - [Closure](architecture/store/store.md#closure) + - [Files and Processes](architecture/store/store.md#files-and-processes) + - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Store Path](architecture/store/path.md) + - [Digest](architecture/store/path.md#digest) + - [Input Addressing](architecture/store/path.md#input-addressing) + - [Content Addressing](architecture/store/path.md#content-addressing) + - [File System Object](architecture/store/fso.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) From 3df1ee2ba573a999aebb87cc9c5cac21df158120 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:32:35 +0200 Subject: [PATCH 175/371] clarify what explicitly declaring certain dependencies means --- doc/manual/src/architecture/store/path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 7d793749b..3b14b2f3c 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -81,7 +81,7 @@ This way, scanning files captures run time dependencies without the user having Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. ::: {.note} -In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. From 3c2de2da01704ad2b43182b3b9080dc541e7efc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 4 Aug 2022 06:23:36 -0500 Subject: [PATCH 176/371] Update doc/manual/src/installation/installing-binary.md Co-authored-by: Valentin Gagarin --- doc/manual/src/installation/installing-binary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index a2f284d5a..d94e78a91 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -149,7 +149,7 @@ and `/etc/zshrc` which you may remove. 4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or + `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic mounting of the Nix Store volume. From 84a26882f86e68df3f2d0f66ff627e0dbd6365f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 4 Aug 2022 06:23:45 -0500 Subject: [PATCH 177/371] Update doc/manual/src/installation/installing-binary.md Co-authored-by: Valentin Gagarin --- doc/manual/src/installation/installing-binary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index d94e78a91..ed0f65177 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -176,7 +176,7 @@ and `/etc/zshrc` which you may remove. This will remove the Nix Store volume and everything that was added to the store. - If this command indicates that it couldn't remove the volume, you should + If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. Look for a "Nix Store" volume in the output of the following command: From 8cec32e7f5e073946ef7bbf5ec1839e40b203bbc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 13:50:44 +0200 Subject: [PATCH 178/371] fix directory tree renderings --- doc/manual/src/architecture/store/fso.md | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index 2ec365eb6..e0eb69f60 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -19,27 +19,27 @@ Examples: - a directory with contents - /nix/store/-hello-2.10 - ├── bin - │   └── hello - └── share - ├── info - │   └── hello.info - └── man - └── man1 - └── hello.1.gz + /nix/store/-hello-2.10 + ├── bin + │   └── hello + └── share + ├── info + │   └── hello.info + └── man + └── man1 + └── hello.1.gz - a directory with relative symlink and other contents - /nix/store/-go-1.16.9 - ├── bin -> share/go/bin - ├── nix-support/ - └── share/ + /nix/store/-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ - a directory with absolute symlink - /nix/store/d3k...-nodejs - └── nix_node -> /nix/store/f20...-nodejs-10.24. + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. A bare file or symlink can be a root file system object. Examples: @@ -53,13 +53,13 @@ Examples: - an arbitrarily symlinked file may change or not exist at all - /nix/store/-foo - └── foo -> /home/foo + /nix/store/-foo + └── foo -> /home/foo - if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted - /nix/store/-bar - └── bar -> /nix/store/abc...-foo + /nix/store/-bar + └── bar -> /nix/store/abc...-foo Nix file system objects do not support [hard links][hardlink]: each file system object which is not the root has exactly one parent and one name. From cc3a5f4ba2cb6859e0f324a5e74a42668b96aff0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 14:04:02 +0200 Subject: [PATCH 179/371] use correct mdBook syntax for callouts --- doc/manual/src/architecture/architecture.md | 7 +++---- doc/manual/src/architecture/store/path.md | 21 +++++++++------------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index d9fb2e7e5..96371056b 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -34,10 +34,9 @@ It is used to compose expressions which ultimately evaluate to self-contained *b The command line and Nix language are what users interact with most. -::: {.note} -The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a build plan are just data. -::: +> **Note** +> The Nix language itself does not have a notion of *packages* or *configurations*. +> As far as we are concerned here, the inputs and results of a build plan are just data. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data. diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 3b14b2f3c..5a18df67a 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -34,9 +34,8 @@ If the store has a [file system representation](./store.md#files-and-processes), This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. -::: {.note} -The store directory defaults to `/nix/store`, but is in principle arbitrary. -::: +> **Note** +> The store directory defaults to `/nix/store`, but is in principle arbitrary. It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. @@ -60,10 +59,9 @@ In a [store path](#store-path), the [digest][digest] is the output of a [cryptog Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). -::: {.note} -**Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. -::: +> **Historical Note** +> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. [digest]: https://en.m.wiktionary.org/wiki/digest#Noun [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function @@ -80,12 +78,11 @@ Nix uses the [closure](store.md#closure) of build inputs to derive the list of a This way, scanning files captures run time dependencies without the user having to declare them explicitly. Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. -::: {.note} -In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +> **Note** +> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. - -For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. -::: +> +> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. ## Input Addressing {#input-addressing} From b63174227704c2a5fc11e815aaf3b4a6fd6f4576 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 14:31:33 +0200 Subject: [PATCH 180/371] fix page rendering apparently it is not possible to link to page anchors with `mdBook`[1] [1]: https://github.com/rust-lang/mdBook/issues/167 --- doc/manual/src/SUMMARY.md.in | 8 +-- doc/manual/src/architecture/store/path.md | 2 +- doc/manual/src/architecture/store/store.md | 61 ------------------- .../src/architecture/store/store/closure.md | 29 +++++++++ .../architecture/store/store/terminology.md | 32 ++++++++++ 5 files changed, 64 insertions(+), 68 deletions(-) create mode 100644 doc/manual/src/architecture/store/store/closure.md create mode 100644 doc/manual/src/architecture/store/store/terminology.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9c2ae2ad3..a95c385e4 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -68,12 +68,8 @@ - [nix.conf](command-ref/conf-file.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Object](architecture/store/store.md#store-object) - - [Reference](architecture/store/store.md#reference) - - [Operations](architecture/store/store.md#operations) - - [Closure](architecture/store/store.md#closure) - - [Files and Processes](architecture/store/store.md#files-and-processes) - - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Closure](architecture/store/store/closure.md) + - [Build system terminology](architecture/store/build-system-terminology.md) - [Store Path](architecture/store/path.md) - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 5a18df67a..039d8b1c6 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -39,7 +39,7 @@ This means a store path is not just derived from the referenced store object its It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. -Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. +Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries. Therefore one can only copy store objects to a different store if diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 920d483d5..9f580f9da 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -89,35 +89,6 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] - -## Closure {#closure} - -Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. - -The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. - -Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - -- A newly added store object cannot have references, unless it is a build task. - -- Build results must only refer to store objects in the closure of the build inputs. - - Building a store object will add appropriate references, according to the build task. - -- Store objects being copied must refer to objects already in the destination store. - - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete store objects which are not reachable from any reference still in use. - - - -[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity -[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) -[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object -[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type -[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier - ## Files and Processes {#files-and-processes} Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. @@ -178,35 +149,3 @@ Examples: To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. -## A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} - -The Nix store's design is comparable to other build systems. -Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. - -The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. - -| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | -| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | -| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | -| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | -| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | -| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | -| build | build | build | application of `Build` | evaluation | -| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | - -All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). - -[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone -[bazel]: https://bazel.build/start/bazel-intro -[bazel-artifact]: https://bazel.build/reference/glossary#artifact -[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html -[bazel-action]: https://bazel.build/reference/glossary#action -[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph -[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph -[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache -[thunk]: https://en.m.wikipedia.org/wiki/Thunk -[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph -[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming -[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf -[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf -[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf diff --git a/doc/manual/src/architecture/store/store/closure.md b/doc/manual/src/architecture/store/store/closure.md new file mode 100644 index 000000000..065b95ffc --- /dev/null +++ b/doc/manual/src/architecture/store/store/closure.md @@ -0,0 +1,29 @@ +# Closure + +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. + +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: + +- A newly added store object cannot have references, unless it is a build task. + +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to the build task. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + + +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + + diff --git a/doc/manual/src/architecture/store/store/terminology.md b/doc/manual/src/architecture/store/store/terminology.md new file mode 100644 index 000000000..3dd7aa756 --- /dev/null +++ b/doc/manual/src/architecture/store/store/terminology.md @@ -0,0 +1,32 @@ +# A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} + +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. + +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | + +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf From b74a3f51c2ecb200ceb877bf5dda528664ef7e10 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:01:27 -0400 Subject: [PATCH 181/371] Fix gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba8e95191..0c1b89ace 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ perl/Makefile.config /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/expressions/builtins.md +/doc/manual/src/language/builtins.md # /scripts/ /scripts/nix-profile.sh From b430a6743c59aebc08968da3cf01e476edbc36c2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:01:55 -0400 Subject: [PATCH 182/371] Remove sections within from SUMMARY --- doc/manual/src/SUMMARY.md.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 812a3d732..a47d39f31 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -62,11 +62,8 @@ - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - [Closure](architecture/store/store/closure.md) - - [Build system terminology](architecture/store/build-system-terminology.md) + - [Build system terminology](architecture/store/store/build-system-terminology.md) - [Store Path](architecture/store/path.md) - - [Digest](architecture/store/path.md#digest) - - [Input Addressing](architecture/store/path.md#input-addressing) - - [Content Addressing](architecture/store/path.md#content-addressing) - [File System Object](architecture/store/fso.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) From 016d7a8f78bee2181d1d60bcf9a8a59a4f18e91f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:19 -0400 Subject: [PATCH 183/371] Fix rosetta stone file name --- .../store/store/{terminology.md => build-system-terminology.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename doc/manual/src/architecture/store/store/{terminology.md => build-system-terminology.md} (97%) diff --git a/doc/manual/src/architecture/store/store/terminology.md b/doc/manual/src/architecture/store/store/build-system-terminology.md similarity index 97% rename from doc/manual/src/architecture/store/store/terminology.md rename to doc/manual/src/architecture/store/store/build-system-terminology.md index 3dd7aa756..eefbaa630 100644 --- a/doc/manual/src/architecture/store/store/terminology.md +++ b/doc/manual/src/architecture/store/store/build-system-terminology.md @@ -1,4 +1,4 @@ -# A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} +# A [Rosetta stone][rosetta-stone] for build system terminology The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. From 6f6498f59c9263d3d4bdc6d2c3776e6abad2e516 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:38 -0400 Subject: [PATCH 184/371] Remove header fragments which is not needd --- doc/manual/src/architecture/store/path.md | 8 ++++---- doc/manual/src/architecture/store/store.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 039d8b1c6..663f04f46 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -26,7 +26,7 @@ Example: |--------| |------------------------------| |----------| store directory digest name -## Store Directory {#store-directory} +## Store Directory Every [store](./store.md) has a store directory. @@ -53,7 +53,7 @@ One cannot copy a store object to a store with a different store directory. Instead, it has to be rebuilt, together with all its dependencies. It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. -# Digest {#digest} +# Digest In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. @@ -84,7 +84,7 @@ This depends on the specifics of the software to build and run. > > For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. -## Input Addressing {#input-addressing} +## Input Addressing Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. @@ -95,7 +95,7 @@ Nix has a custom serialisation format called Nix Archive (NAR) Store object references of this sort can *not* be validated from the content of the store object. Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. -## Content Addressing {#content-addressing} +## Content Addressing Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. If one knows content addressing was used, one can recalculate the reference and thus verify the store object. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 9f580f9da..08b6701d5 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -25,7 +25,7 @@ Arrows indicate suggested reading order. [ input addressing ] [ content addressing ] ``` -## Store Object {#store-object} +## Store Object A store object can hold @@ -36,13 +36,13 @@ Store objects can be build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -## Reference {#reference} +## Reference A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -## Operations {#operations} +## Operations A Nix store can *add*, *retrieve*, and *delete* store objects. @@ -89,7 +89,7 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -## Files and Processes {#files-and-processes} +## Files and Processes Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. From 39d32ac4c63f4aa3784d114b19c0eca83e306ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:58 -0400 Subject: [PATCH 185/371] Add disclaimer that arch section is WIP and links may rot So we can iterate without worrying so much. --- doc/manual/src/architecture/architecture.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 96371056b..41deb07af 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -1,5 +1,7 @@ # Architecture +*(This chapter is unstable and a work in progress. Incoming links may rot.)* + This chapter describes how Nix works. It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. From a5be5e01200a12cc34d0e3a2e3f964d5c95208b9 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 4 Aug 2022 14:07:06 -0700 Subject: [PATCH 186/371] doc/manual: define {local,remote} store, binary cache, substituter Nix veterans intuitively know what the following terms mean. They are used in several places in the nix documentation, but never defined: - local store - remote store - binary cache - substituter In particular, I found the last two terms to be confusingly similar. Let's give definitions for them. --- doc/manual/src/SUMMARY.md.in | 1 + .../src/package-management/terminology.md | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 doc/manual/src/package-management/terminology.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a47d39f31..f8da2247b 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -22,6 +22,7 @@ - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - [Channels](package-management/channels.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) + - [Terminology](package-management/terminology.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md new file mode 100644 index 000000000..28e2a1f0b --- /dev/null +++ b/doc/manual/src/package-management/terminology.md @@ -0,0 +1,27 @@ +# Terminology + +A *local store* exists on the local filesystem of the machine where +Nix is invoked. The `/nix/store` directory is one example of a +local store. You can use other local stores by passing the +`--store` flag to `nix`. + +A *remote store* is a store which exists anywhere other than the +local filesystem. One example is the `/nix/store` directory on +another machine, accessed via `ssh` or served by the `nix-serve` +Perl script. + +A *binary cache* is a remote store which is not the local store of +any machine. Examples of binary caches include S3 buckets and the +[NixOS binary cache](https://cache.nixos.org). Binary caches use a +disk layout that is different from local stores; in particular, they +keep metadata and signatures in `.narinfo` files rather than in +`/nix/var/nix/db`. + +A *substituter* is a store other than `/nix/store` from which nix will +copy the realisation of a derivation instead of building it. Nix will +not copy a realisation from a remote store unless one of the following +is true: + +- the realisation is signed by one of the `trusted-public-key`s +- the substituter is in the `trusted-substituters` list +- the `no-require-sigs` option has been set to disable signature checking From 56d4fc194ba90ee4e559a07a895f6bf4a61ef462 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 4 Aug 2022 14:14:24 -0700 Subject: [PATCH 187/371] fourth trust condition: FODs --- doc/manual/src/package-management/terminology.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 28e2a1f0b..1722a9fc8 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -25,3 +25,5 @@ is true: - the realisation is signed by one of the `trusted-public-key`s - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking +- the derivation is a fixed-output derivation + From 8f44d24c525160b2ddef5e18a4af4ce667e23e9f Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 4 Aug 2022 14:19:25 -0700 Subject: [PATCH 188/371] !fixup whitespace --- doc/manual/src/package-management/terminology.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 1722a9fc8..d800bafc1 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -26,4 +26,3 @@ is true: - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking - the derivation is a fixed-output derivation - From 62674659ed7b7fc6a2c884f52df2474e344400f8 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 4 Aug 2022 14:21:17 -0700 Subject: [PATCH 189/371] !fixup capitalize Nix --- doc/manual/src/package-management/terminology.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index d800bafc1..8dc2ede33 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -17,7 +17,7 @@ disk layout that is different from local stores; in particular, they keep metadata and signatures in `.narinfo` files rather than in `/nix/var/nix/db`. -A *substituter* is a store other than `/nix/store` from which nix will +A *substituter* is a store other than `/nix/store` from which Nix will copy the realisation of a derivation instead of building it. Nix will not copy a realisation from a remote store unless one of the following is true: From 1b97f3872ed70d9ad5d19d27dc56b42ba4d26382 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 4 Aug 2022 14:22:14 -0700 Subject: [PATCH 190/371] !fixup: transposed characters --- doc/manual/src/package-management/terminology.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 8dc2ede33..6ad0f6833 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -22,7 +22,7 @@ copy the realisation of a derivation instead of building it. Nix will not copy a realisation from a remote store unless one of the following is true: -- the realisation is signed by one of the `trusted-public-key`s +- the realisation is signed by one of the `trusted-public-keys` - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking - the derivation is a fixed-output derivation From 4eb566603a1f6f08eae074d48e5c860e1d5d561b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Aug 2022 11:01:40 +0200 Subject: [PATCH 191/371] Comment out the architecture section This needs more review and probably should be a separate book. --- doc/manual/src/SUMMARY.md.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a47d39f31..8fbb59716 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -59,12 +59,14 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) + - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) From aae771cad26a3803ef0a0855c782823d22949cf3 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 5 Aug 2022 10:12:46 -0700 Subject: [PATCH 192/371] !implement https://github.com/NixOS/nix/pull/6870#discussion_r938912244 --- doc/manual/src/package-management/terminology.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 6ad0f6833..241bb6c5a 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -18,11 +18,10 @@ keep metadata and signatures in `.narinfo` files rather than in `/nix/var/nix/db`. A *substituter* is a store other than `/nix/store` from which Nix will -copy the realisation of a derivation instead of building it. Nix will -not copy a realisation from a remote store unless one of the following -is true: +copy a store path instead of building it. Nix will not copy a store +path from a remote store unless one of the following is true: -- the realisation is signed by one of the `trusted-public-keys` +- the store object is signed by one of the `trusted-public-keys` - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking -- the derivation is a fixed-output derivation +- the store object is the realisation of a fixed-output derivation From 2eb74c918dc7dc04ed36b3fdcd95406007d97690 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 5 Aug 2022 10:13:41 -0700 Subject: [PATCH 193/371] derivations do not need to be signed --- doc/manual/src/package-management/terminology.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 241bb6c5a..4b9e68de9 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -24,4 +24,5 @@ path from a remote store unless one of the following is true: - the store object is signed by one of the `trusted-public-keys` - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking +- the store object is a derivation - the store object is the realisation of a fixed-output derivation From 66a93a76b9842ac18188b91f5a30c4ac4f2b6118 Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+amjoseph-nixpkgs@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:15:37 +0000 Subject: [PATCH 194/371] Update doc/manual/src/package-management/terminology.md Co-authored-by: Attila Gulyas --- .../src/package-management/terminology.md | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 4b9e68de9..564667f93 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -1,14 +1,22 @@ # Terminology -A *local store* exists on the local filesystem of the machine where -Nix is invoked. The `/nix/store` directory is one example of a -local store. You can use other local stores by passing the -`--store` flag to `nix`. +From the perspective of the location where Nix is +invoked1, the Nix store can be referred to +as a "_local_" or a "_remote_" one: -A *remote store* is a store which exists anywhere other than the -local filesystem. One example is the `/nix/store` directory on -another machine, accessed via `ssh` or served by the `nix-serve` -Perl script. +\[1]: Where "invoking Nix" means an executing a Nix core +action/operation on a Nix store. For example, using any CLI +commands from the `NixOS/nix` implementation. + ++ A *local store* exists on the local filesystem of + the machine where Nix is invoked. You can use other + local stores by passing the `--store` flag to the + `nix` command. + ++ A *remote store* exists anywhere other than the + local filesystem. One example is the `/nix/store` + directory on another machine, accessed via `ssh` or + served by the `nix-serve` Perl script. A *binary cache* is a remote store which is not the local store of any machine. Examples of binary caches include S3 buckets and the From d5506aa71200425b65cc1777077478f5ff8d2aff Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+amjoseph-nixpkgs@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:19:52 +0000 Subject: [PATCH 195/371] Update doc/manual/src/package-management/terminology.md Co-authored-by: Attila Gulyas --- doc/manual/src/package-management/terminology.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index 564667f93..b3e9ea040 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -18,12 +18,10 @@ commands from the `NixOS/nix` implementation. directory on another machine, accessed via `ssh` or served by the `nix-serve` Perl script. -A *binary cache* is a remote store which is not the local store of -any machine. Examples of binary caches include S3 buckets and the -[NixOS binary cache](https://cache.nixos.org). Binary caches use a -disk layout that is different from local stores; in particular, they -keep metadata and signatures in `.narinfo` files rather than in -`/nix/var/nix/db`. +A *binary cache* is a specialized Nix store whose metadata and +signatures are kept in `.narinfo` files rather than in the Nix +database. Examples of binary caches include S3 buckets and the +[NixOS binary cache](https://cache.nixos.org). A *substituter* is a store other than `/nix/store` from which Nix will copy a store path instead of building it. Nix will not copy a store From 4de95f7f565df71d8ebddb7434e2b0feb49a833b Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 5 Aug 2022 10:33:48 -0700 Subject: [PATCH 196/371] gesture at explanation of why binary caches exist --- doc/manual/src/package-management/terminology.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md index b3e9ea040..493f5f03e 100644 --- a/doc/manual/src/package-management/terminology.md +++ b/doc/manual/src/package-management/terminology.md @@ -18,10 +18,12 @@ commands from the `NixOS/nix` implementation. directory on another machine, accessed via `ssh` or served by the `nix-serve` Perl script. -A *binary cache* is a specialized Nix store whose metadata and -signatures are kept in `.narinfo` files rather than in the Nix -database. Examples of binary caches include S3 buckets and the -[NixOS binary cache](https://cache.nixos.org). +A *binary cache* is a Nix store which uses a different format: its +metadata and signatures are kept in `.narinfo` files rather than in a +Nix database. This different format simplifies serving store objects +over the network, but cannot host builds. Examples of binary caches +include S3 buckets and the [NixOS binary +cache](https://cache.nixos.org). A *substituter* is a store other than `/nix/store` from which Nix will copy a store path instead of building it. Nix will not copy a store From 1d3b92e80ca1564bf2c5ee207df707e215188633 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 5 Aug 2022 10:39:43 -0700 Subject: [PATCH 197/371] move package-management/terminology into glossary.md --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/glossary.md | 37 +++++++++++++++++++ .../src/package-management/terminology.md | 36 ------------------ 3 files changed, 37 insertions(+), 37 deletions(-) delete mode 100644 doc/manual/src/package-management/terminology.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f8da2247b..a47d39f31 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -22,7 +22,6 @@ - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - [Channels](package-management/channels.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - - [Terminology](package-management/terminology.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index aa0ac78cb..f4c51588d 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -11,6 +11,32 @@ The location in the file system where store objects live. Typically `/nix/store`. + From the perspective of the location where Nix is + invoked1, the Nix store can be referred to + as a "_local_" or a "_remote_" one: + + \[1]: Where "invoking Nix" means an executing a Nix core + action/operation on a Nix store. For example, using any CLI + commands from the `NixOS/nix` implementation. + + + A *local store* exists on the local filesystem of + the machine where Nix is invoked. You can use other + local stores by passing the `--store` flag to the + `nix` command. + + + A *remote store* exists anywhere other than the + local filesystem. One example is the `/nix/store` + directory on another machine, accessed via `ssh` or + served by the `nix-serve` Perl script. + + - [binary cache]{#binary-cache}\ + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + Nix database. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary + cache](https://cache.nixos.org). + - [store path]{#gloss-store-path}\ The location in the file system of a store object, i.e., an immediate child of the Nix store directory. @@ -29,6 +55,17 @@ store object by downloading a pre-built version of the store object from some server. + - [substituter]{#gloss-substituter}\ + A *substituter* is a store other than `/nix/store` from which Nix will + copy a store path instead of building it. Nix will not copy a store + path from a remote store unless one of the following is true: + + - the store object is signed by one of the `trusted-public-keys` + - the substituter is in the `trusted-substituters` list + - the `no-require-sigs` option has been set to disable signature checking + - the store object is a derivation + - the store object is the realisation of a fixed-output derivation + - [purity]{#gloss-purity}\ The assumption that equal Nix derivations when run always produce the same output. This cannot be guaranteed in general (e.g., a diff --git a/doc/manual/src/package-management/terminology.md b/doc/manual/src/package-management/terminology.md deleted file mode 100644 index 493f5f03e..000000000 --- a/doc/manual/src/package-management/terminology.md +++ /dev/null @@ -1,36 +0,0 @@ -# Terminology - -From the perspective of the location where Nix is -invoked1, the Nix store can be referred to -as a "_local_" or a "_remote_" one: - -\[1]: Where "invoking Nix" means an executing a Nix core -action/operation on a Nix store. For example, using any CLI -commands from the `NixOS/nix` implementation. - -+ A *local store* exists on the local filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. - -+ A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. - -A *binary cache* is a Nix store which uses a different format: its -metadata and signatures are kept in `.narinfo` files rather than in a -Nix database. This different format simplifies serving store objects -over the network, but cannot host builds. Examples of binary caches -include S3 buckets and the [NixOS binary -cache](https://cache.nixos.org). - -A *substituter* is a store other than `/nix/store` from which Nix will -copy a store path instead of building it. Nix will not copy a store -path from a remote store unless one of the following is true: - -- the store object is signed by one of the `trusted-public-keys` -- the substituter is in the `trusted-substituters` list -- the `no-require-sigs` option has been set to disable signature checking -- the store object is a derivation -- the store object is the realisation of a fixed-output derivation From b5d85f0922e0f4a9585a281f6d938ec67cd07349 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 5 Aug 2022 13:49:18 -0500 Subject: [PATCH 198/371] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9a371afa7..86c6522f2 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -113,11 +113,12 @@ You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix- Testing the install scripts has traditionally been tedious, but you can now do this much more easily via the GitHub Actions CI runs (at least for platforms that Github Actions supports). -If you've already pushed to a fork of Nix on GitHub before, you may have noticed that the CI workflows in your fork list skipped "installer" and "installer_test" jobs. Once your Nix fork is set up correctly, pushing to it will also run these jobs. -- The `installer` job will generate installers for these platforms: x86_64-linux, armv6l-linux, armv7l-linux, x86_64-darwin. While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. +If you've already pushed to a fork of Nix on GitHub before, you may have noticed that the CI workflows in your fork list skipped `installer` and `installer_test` jobs. Once your Nix fork is set up correctly, pushing to it will also run these jobs. +- The `installer` job will generate installers for these platforms: `x86_64-linux`, `armv6l-linux`, `armv7l-linux`, `x86_64-darwin`. While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. - the `installer_test` job will try to use this installer and run a trivial Nix command on `ubuntu-latest` and `macos-latest`. ### One-time setup + 1. Have a GitHub account with a fork of the Nix repo. 2. At cachix.org: - Create or log in to an account. From cb6794a0d983eb364601f26fc32ead98ed67bfb4 Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Sun, 7 Aug 2022 10:13:11 -0400 Subject: [PATCH 199/371] Do not spam logs if the owned-homedir check results in a noop --- src/libutil/util.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..e11cb9c60 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -577,6 +577,7 @@ Path getHome() { static Path homeDir = []() { + std::optional unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); if (homeDir) { // Only use $HOME if doesn't exist or is owned by the current user. @@ -588,8 +589,7 @@ Path getHome() homeDir.reset(); } } else if (st.st_uid != geteuid()) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); - homeDir.reset(); + unownedUserHomeDir.swap(homeDir); } } if (!homeDir) { @@ -600,6 +600,9 @@ Path getHome() || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } } return *homeDir; }(); From 96b4339d86a22175da22900acecd6b0fb4d5e12b Mon Sep 17 00:00:00 2001 From: Tom Franklin Date: Mon, 8 Aug 2022 00:42:18 +0100 Subject: [PATCH 200/371] Only fail if selinux is in enforcing --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..e6864eaaf 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -640,7 +640,7 @@ place_channel_configuration() { check_selinux() { if command -v getenforce > /dev/null 2>&1; then - if ! [ "$(getenforce)" = "Disabled" ]; then + if [ "$(getenforce)" = "Enforcing" ]; then failure < Date: Mon, 8 Aug 2022 14:34:22 +0200 Subject: [PATCH 201/371] Remove the explicit `c++fs` linkage on darwin Doesn't seem needed on a recent-enough clang anymore (and even seems to break stuff) --- src/libutil/local.mk | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 13e8d426a..f880c0fc5 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,7 +11,3 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif - -ifdef HOST_DARWIN - libutil_LDFLAGS += -lc++fs -endif From 9b7bd2dd1fc83f6df449fce3967a95cb098ca4b2 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 8 Aug 2022 10:04:27 -0500 Subject: [PATCH 202/371] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 86c6522f2..d8a8c8591 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -114,7 +114,13 @@ You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix- Testing the install scripts has traditionally been tedious, but you can now do this much more easily via the GitHub Actions CI runs (at least for platforms that Github Actions supports). If you've already pushed to a fork of Nix on GitHub before, you may have noticed that the CI workflows in your fork list skipped `installer` and `installer_test` jobs. Once your Nix fork is set up correctly, pushing to it will also run these jobs. -- The `installer` job will generate installers for these platforms: `x86_64-linux`, `armv6l-linux`, `armv7l-linux`, `x86_64-darwin`. While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. +- The `installer` job will generate installers for these platforms: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin`. + + While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. - the `installer_test` job will try to use this installer and run a trivial Nix command on `ubuntu-latest` and `macos-latest`. ### One-time setup From bac1e1bf8c359b5e6831c3974a05bdce867775a5 Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+amjoseph-nixpkgs@users.noreply.github.com> Date: Mon, 8 Aug 2022 17:52:31 +0000 Subject: [PATCH 203/371] Update doc/manual/src/glossary.md Co-authored-by: Valentin Gagarin --- doc/manual/src/glossary.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index f4c51588d..77de58965 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -12,13 +12,9 @@ `/nix/store`. From the perspective of the location where Nix is - invoked1, the Nix store can be referred to + invoked, the Nix store can be referred to as a "_local_" or a "_remote_" one: - \[1]: Where "invoking Nix" means an executing a Nix core - action/operation on a Nix store. For example, using any CLI - commands from the `NixOS/nix` implementation. - + A *local store* exists on the local filesystem of the machine where Nix is invoked. You can use other local stores by passing the `--store` flag to the From 5f37c5191a3a8f5c7ab31a0dd8bffe14aaa6b76c Mon Sep 17 00:00:00 2001 From: Winter Date: Tue, 9 Aug 2022 16:48:34 -0400 Subject: [PATCH 204/371] nix-shell: specify which outputs from bashInteractive to build --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 7eb8c8f6a..df292dce6 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -401,7 +401,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { .drvPath = bashDrv, - .outputs = {}, + .outputs = {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; From 0eb9946e1d3621cfc2fcffc9378dba334b25fb26 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 9 Aug 2022 23:21:09 -0400 Subject: [PATCH 205/371] docker.nix: Provide boolean for whether to bundle nixpkgs --- docker.nix | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docker.nix b/docker.nix index 8e6aa227f..e95caf274 100644 --- a/docker.nix +++ b/docker.nix @@ -2,6 +2,7 @@ , lib ? pkgs.lib , name ? "nix" , tag ? "latest" +, bundleNixpkgs ? true , channelName ? "nixpkgs" , channelURL ? "https://nixos.org/channels/nixpkgs-unstable" , extraPkgs ? [] @@ -139,10 +140,12 @@ let baseSystem = let nixpkgs = pkgs.path; - channel = pkgs.runCommand "channel-nixos" { } '' + channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } '' mkdir $out - ln -s ${nixpkgs} $out/nixpkgs - echo "[]" > $out/manifest.nix + if [ "$bundleNixpkgs" ]; then + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + fi ''; rootEnv = pkgs.buildPackages.buildEnv { name = "root-profile-env"; From 64c3adbe1ad14e043b772c3e981d511922cb06e5 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 8 Aug 2022 15:45:23 -0400 Subject: [PATCH 206/371] install-multi-user: abstract is_root, is_os_linux, is_os_darwin --- scripts/install-multi-user.sh | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a18280ef..472d25842 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -59,6 +59,30 @@ headless() { fi } +is_root() { + if [ "$EUID" -eq 0 ]; then + return 0 + else + return 1 + fi +} + +is_os_linux() { + if [ "$(uname -s)" = "Linux" ]; then + return 0 + else + return 1 + fi +} + +is_os_darwin() { + if [ "$(uname -s)" = "Darwin" ]; then + return 0 + else + return 1 + fi +} + contact_us() { echo "You can open an issue at https://github.com/nixos/nix/issues" echo "" @@ -423,7 +447,7 @@ EOF fi done - if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then + if is_os_linux && [ ! -e /run/systemd/system ]; then warning < Date: Mon, 8 Aug 2022 16:01:59 -0400 Subject: [PATCH 207/371] Strip whitespace in installing-binary.md --- doc/manual/src/installation/installing-binary.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 9fb9c80c3..18fc8fff5 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -13,7 +13,7 @@ for your platform: - multi-user on macOS > **Notes on read-only filesystem root in macOS 10.15 Catalina +** - > + > > - It took some time to support this cleanly. You may see posts, > examples, and tutorials using obsolete workarounds. > - Supporting it cleanly made macOS installs too complex to qualify @@ -75,7 +75,7 @@ should run this under your usual user account, *not* as root. The script will invoke `sudo` as needed. > **Note** -> +> > If you need Nix to use a different group ID or user ID set, you will > have to download the tarball manually and [edit the install > script](#installing-from-a-binary-tarball). @@ -167,7 +167,7 @@ and `/etc/zshrc` which you may remove. removed next. 7. Remove the Nix Store volume: - + ```console sudo diskutil apfs deleteVolume /nix ``` @@ -176,7 +176,7 @@ and `/etc/zshrc` which you may remove. store. > **Note** -> +> > After you complete the steps here, you will still have an empty `/nix` > directory. This is an expected sign of a successful uninstall. The empty > `/nix` directory will disappear the next time you reboot. From 7bb1e913b33499d3ce74929749977774bcc35aed Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Mon, 8 Aug 2022 15:46:17 -0400 Subject: [PATCH 208/371] Don't prompt about using sudo if we're already root --- .../src/installation/installing-binary.md | 6 ++--- scripts/install-multi-user.sh | 25 ++++++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 18fc8fff5..a46735196 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -31,8 +31,8 @@ $ sh <(curl -L https://nixos.org/nix/install) --no-daemon ``` This will perform a single-user installation of Nix, meaning that `/nix` -is owned by the invoking user. You should run this under your usual user -account, *not* as root. The script will invoke `sudo` to create `/nix` +is owned by the invoking user. You can run this under your usual user +account or root. The script will invoke `sudo` to create `/nix` if it doesn’t already exist. If you don’t have `sudo`, you should manually create `/nix` first as root, e.g.: @@ -71,7 +71,7 @@ $ sh <(curl -L https://nixos.org/nix/install) --daemon The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. You -should run this under your usual user account, *not* as root. The script +can run this under your usual user account or root. The script will invoke `sudo` as needed. > **Note** diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 472d25842..e7bdc1227 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -337,10 +337,15 @@ __sudo() { _sudo() { local expl="$1" shift - if ! headless; then + if ! headless || is_root; then __sudo "$expl" "$*" >&2 fi - sudo "$@" + + if is_root; then + env "$@" + else + sudo "$@" + fi } @@ -891,17 +896,6 @@ EOF main() { - # TODO: I've moved this out of validate_starting_assumptions so we - # can fail faster in this case. Sourcing install-darwin... now runs - # `touch /` to detect Read-only root, but it could update times on - # pre-Catalina macOS if run as root user. - if is_root; then - failure < Date: Wed, 10 Aug 2022 18:49:29 -0500 Subject: [PATCH 209/371] docfix: bundlers --- src/nix/bundle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 2bb70711f..a18161a3c 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -44,7 +44,7 @@ flake output attributes: * `bundlers..default` -If an attribute *name* is given, `nix run` tries the following flake +If an attribute *name* is given, `nix bundle` tries the following flake output attributes: * `bundlers..` From 703b335c1d187fb18ca5a5450b33e4e9a98abc7c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 15 Aug 2022 11:34:39 +0200 Subject: [PATCH 210/371] reword description of language properties the list style is supposed to give more structure. each property is explained as concisely as possible while trying not to sound too fancy. --- doc/manual/src/language/index.md | 38 ++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index c4b3abf75..a4b402f8b 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,13 +1,33 @@ # Nix Language -The Nix language is a pure, lazy, functional language. Purity -means that operations in the language don't have side-effects (for -instance, there is no variable assignment). Laziness means that -arguments to functions are evaluated only when they are needed. -Functional means that functions are “normal” values that can be passed -around and manipulated in interesting ways. The language is not a -full-featured, general purpose language. Its main job is to describe -packages, compositions of packages, and the variability within packages. +The Nix language is -This section presents the various features of the language. +- *domain-specific* + + It only exists for the Nix package manager: + to describe packages and configurations as well as their variants and compositions. + It is not intended for general purpose use. + +- *declarative* + + There is no notion of executing sequential steps. + Dependencies between operations are established only through data. + +- *pure* + + Values cannot change during computation. + Functions always produce the same output if their input does not change. + +- *functional* + + Functions are like any other value. + Functions can be assigned to names, taken as arguments, or returned by functions. + +- *lazy* + + Expressions are only evaluated when their value is needed. + +- *dynamically typed* + + Type errors are only detected when expressions are evaluated. From 3d4489b623deaceef720da7d884a41452f928db6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 15:57:25 +0200 Subject: [PATCH 211/371] Show when we're evaluating a flake --- src/libcmd/installables.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 59162c4df..e097f23b3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -616,6 +616,8 @@ InstallableFlake::InstallableFlake( std::tuple InstallableFlake::toDerivation() { + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + auto attr = getCursor(*state); auto attrPath = attr->getAttrPathStr(); From e62160579f40a0425061c2223e0a303d42736ea2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Aug 2022 14:58:08 +0200 Subject: [PATCH 212/371] nix flake metadata: Don't show "Inputs" if there are no inputs --- src/nix/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e01bc6d10..3967f1102 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -212,7 +212,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); - logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + if (!lockedFlake.lockFile.root->inputs.empty()) + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); std::unordered_set> visited; From c3769c68465bae971ab6bb48cfcdea85b61ea83a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 15:56:08 +0200 Subject: [PATCH 213/371] ProgressBar: Delay before showing a new activity Some activities are numerous but usually very short (e.g. copying a source file to the store) which would cause a lot of flickering. So only show activities that have been running for at least 10 ms. --- src/libmain/progress-bar.cc | 43 ++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index f4306ab91..5183f212f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -8,6 +8,7 @@ #include #include #include +#include namespace nix { @@ -48,6 +49,7 @@ private: bool visible = true; ActivityId parent; std::optional name; + std::chrono::time_point startTime; }; struct ActivitiesByType @@ -91,10 +93,11 @@ public: state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); + auto nextWakeup = std::chrono::milliseconds::max(); while (state->active) { if (!state->haveUpdate) - state.wait(updateCV); - draw(*state); + state.wait_for(updateCV, nextWakeup); + nextWakeup = draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } }); @@ -118,7 +121,8 @@ public: updateThread.join(); } - bool isVerbose() override { + bool isVerbose() override + { return printBuildLogs; } @@ -159,11 +163,13 @@ public: if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); - state->activities.emplace_back(ActInfo()); + state->activities.emplace_back(ActInfo { + .s = s, + .type = type, + .parent = parent, + .startTime = std::chrono::steady_clock::now() + }); auto i = std::prev(state->activities.end()); - i->s = s; - i->type = type; - i->parent = parent; state->its.emplace(act, i); state->activitiesByType[type].its.emplace(act, i); @@ -327,10 +333,12 @@ public: updateCV.notify_one(); } - void draw(State & state) + std::chrono::milliseconds draw(State & state) { + auto nextWakeup = std::chrono::milliseconds::max(); + state.haveUpdate = false; - if (!state.active) return; + if (!state.active) return nextWakeup; std::string line; @@ -341,12 +349,25 @@ public: line += "]"; } + auto now = std::chrono::steady_clock::now(); + if (!state.activities.empty()) { if (!status.empty()) line += " "; auto i = state.activities.rbegin(); - while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + while (i != state.activities.rend()) { + if (i->visible && (!i->s.empty() || !i->lastLine.empty())) { + /* Don't show activities until some time has + passed, to avoid displaying very short + activities. */ + auto delay = std::chrono::milliseconds(10); + if (i->startTime + delay < now) + break; + else + nextWakeup = std::min(nextWakeup, std::chrono::duration_cast(delay - (now - i->startTime))); + } ++i; + } if (i != state.activities.rend()) { line += i->s; @@ -366,6 +387,8 @@ public: if (width <= 0) width = std::numeric_limits::max(); writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + + return nextWakeup; } std::string getStatus(State & state) From 53e7b7e8ac44704199a868c0f6850ac53a0b8ad1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 12:28:02 +0200 Subject: [PATCH 214/371] Remove warnLargeDump() This message was unhelpful (#1184) and probably misleading since memory is O(1) in most cases now. --- src/libstore/remote-store.cc | 2 -- src/libutil/serialise.cc | 20 -------------------- src/libutil/serialise.hh | 4 +--- 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..eaaf9669f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -580,7 +580,6 @@ ref RemoteStore::addCAToStore( try { conn->to.written = 0; - conn->to.warn = true; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); @@ -591,7 +590,6 @@ ref RemoteStore::addCAToStore( dumpString(contents, conn->to); } } - conn->to.warn = false; conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8ff904583..2c3597775 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -48,24 +48,9 @@ FdSink::~FdSink() } -size_t threshold = 256 * 1024 * 1024; - -static void warnLargeDump() -{ - warn("dumping very large path (> 256 MiB); this may run out of memory"); -} - - void FdSink::write(std::string_view data) { written += data.size(); - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } try { writeFull(fd, data); } catch (SysError & e) { @@ -448,11 +433,6 @@ Error readError(Source & source) void StringSink::operator () (std::string_view data) { - static bool warned = false; - if (!warned && s.size() > threshold) { - warnLargeDump(); - warned = true; - } s.append(data); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 13da26c6a..84847835a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -97,19 +97,17 @@ protected: struct FdSink : BufferedSink { int fd; - bool warn = false; size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink && s) + FdSink & operator=(FdSink && s) { flush(); fd = s.fd; s.fd = -1; - warn = s.warn; written = s.written; return *this; } From 81e42e0d3f0345c28f3d19841c89c4b1975c37a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jul 2022 16:41:26 +0200 Subject: [PATCH 215/371] Fix onError --- tests/common.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.sh.in b/tests/common.sh.in index 79da10199..73c2d2309 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -193,7 +193,7 @@ fi onError() { set +x echo "$0: test failed at:" >&2 - for ((i = 1; i < 16; i++)); do + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 done From c80a74b7d52f0cf5fc5c1c7497e059ebf301f789 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 16:59:02 +0200 Subject: [PATCH 216/371] Don't pass --force to 'git add' Fixes #5810. --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d01aaa7a..c1a21e764 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, From 6547dcde2a37179f98cc9a8702bc15a2fd1e6a4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:41:19 +0200 Subject: [PATCH 217/371] Use plain mktemp This fixes the case where $TMPDIR doesn't end in a slash. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a9f3e74dc..7c8e159b5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,7 +349,7 @@ _sudo() { } -readonly SCRATCH=$(mktemp -d "${TMPDIR:-/tmp/}tmp.XXXXXXXXXX") +readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" } From 823e1017d809219e7e00b24ef4ccb0a8b568449c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 21:47:01 +0200 Subject: [PATCH 218/371] Ensure that $TMPDIR exists if defined --- scripts/install-multi-user.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7c8e159b5..1431857d5 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -349,6 +349,11 @@ _sudo() { } +# Ensure that $TMPDIR exists if defined. +if [[ -v TMPDIR ]]; then + mkdir -m 0700 -p "$TMPDIR" +fi + readonly SCRATCH=$(mktemp -d) finish_cleanup() { rm -rf "$SCRATCH" From 8188b1d0abc2eba6497b5dc47f7e848cbacb7677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:59:04 +0200 Subject: [PATCH 219/371] json: write null on abnormal placeholder destruction Avoids leaving dangling attributes like { "foo": } in case of exceptions. --- src/libutil/json.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index abe0e6e74..2f9e97ff5 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + if (first) { + assert(std::uncaught_exceptions()); + if (state->stack != 0) + write(nullptr); + } } } From 7535ee345da6c0aea1dd81e7de7725b37d8bf8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:33:46 +0200 Subject: [PATCH 220/371] nix-env: don't output incomplete JSON --- src/nix-env/nix-env.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..fdd66220a 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -940,12 +940,12 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin JSONObject metaObj = pkgObj.object("meta"); StringSet metaNames = i.queryMetaNames(); for (auto & j : metaNames) { - auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); - placeholder.write(nullptr); + metaObj.attr(j, nullptr); } else { + auto placeholder = metaObj.placeholder(j); PathSet context; printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } From 7d934f7880d460cba1ab908fd35b3d43e97a4749 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 19 Aug 2022 11:26:26 +0200 Subject: [PATCH 221/371] don't read outputs into memory for output rewriting RewritingSink can handle being fed input where a reference crosses a chunk boundary. we don't need to load the whole source into memory, and in fact *not* loading the whole source lets nix build FODs that do not fit into memory (eg fetchurl'ing data files larger than system memory). --- src/libstore/build/local-derivation-goal.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6843173a7..18b682e13 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2374,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() if (*scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - rsink2(sink.s); + dumpPath(actualPath, rsink2); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; From 0d2bf7acf994ba331d6f72c746721b354931be76 Mon Sep 17 00:00:00 2001 From: Solene Rapenne Date: Fri, 19 Aug 2022 12:40:22 +0200 Subject: [PATCH 222/371] add a nix.conf option to set a download speed limit --- src/libstore/filetransfer.cc | 3 +++ src/libstore/globals.hh | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8454ad7d2..252403cb5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..ca8fc6d5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -746,6 +746,13 @@ public: /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maxium transfer rate in kilobytes per second you want + nix to use for download. + )"}; + Setting netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( From 0bf52b73f4cb61bc12c95a015a7be45f7174ca01 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Fri, 19 Aug 2022 15:03:37 -0500 Subject: [PATCH 223/371] install: only create TMPDIR if missing --- scripts/install-multi-user.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 1431857d5..01dbf0c0e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -348,10 +348,9 @@ _sudo() { fi } - # Ensure that $TMPDIR exists if defined. -if [[ -v TMPDIR ]]; then - mkdir -m 0700 -p "$TMPDIR" +if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then + mkdir -m 0700 -p "${TMPDIR:-}" fi readonly SCRATCH=$(mktemp -d) From 7d800909e94c482a2093bc95a2f3dca565c148b2 Mon Sep 17 00:00:00 2001 From: Jakub Kuczys Date: Sat, 20 Aug 2022 03:48:42 +0200 Subject: [PATCH 224/371] Fix default profile path for root in nix profile documentation --- src/nix/profile.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/profile.md b/src/nix/profile.md index 8dade051d..be3c5ba1a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to -`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`/nix/var/nix/profiles/default` if Nix is invoked by the `root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. You can specify another profile location using `--profile` *path*. From caad87e6dbfbc62fafb4a055b45d7f2eb3d11efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Sat, 20 Aug 2022 18:21:36 +0200 Subject: [PATCH 225/371] Better documentation wording Co-authored-by: Anderson Torres --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca8fc6d5f..1ff7d2d0a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -749,7 +749,7 @@ public: Setting downloadSpeed { this, 0, "download-speed", R"( - Specify the maxium transfer rate in kilobytes per second you want + Specify the maximum transfer rate in kilobytes per second you want nix to use for download. )"}; From d8e54d19f71f78540dd967b2e42be6a5d8a0b1bb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 22 Aug 2022 12:50:48 +0200 Subject: [PATCH 226/371] Revert "Merge pull request #6420 from nix-community/doc-what-is-nix" This reverts commit 81e101345fda2a8651c470f08b364a1ca6fa37cf, reversing changes made to 7d1280bbaf7f4cd142c2259dec620c42bf6f96fd. --- doc/manual/src/SUMMARY.md.in | 8 - doc/manual/src/architecture/architecture.md | 79 --------- doc/manual/src/architecture/store/fso.md | 69 -------- doc/manual/src/architecture/store/path.md | 105 ------------ doc/manual/src/architecture/store/store.md | 151 ------------------ .../store/store/build-system-terminology.md | 32 ---- .../src/architecture/store/store/closure.md | 29 ---- 7 files changed, 473 deletions(-) delete mode 100644 doc/manual/src/architecture/architecture.md delete mode 100644 doc/manual/src/architecture/store/fso.md delete mode 100644 doc/manual/src/architecture/store/path.md delete mode 100644 doc/manual/src/architecture/store/store.md delete mode 100644 doc/manual/src/architecture/store/store/build-system-terminology.md delete mode 100644 doc/manual/src/architecture/store/store/closure.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8fbb59716..084c8f442 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -59,14 +59,6 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md deleted file mode 100644 index 41deb07af..000000000 --- a/doc/manual/src/architecture/architecture.md +++ /dev/null @@ -1,79 +0,0 @@ -# Architecture - -*(This chapter is unstable and a work in progress. Incoming links may rot.)* - -This chapter describes how Nix works. -It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. - -## Overview - -Nix consists of [hierarchical layers][layer-architecture]. - -``` -+-----------------------------------------------------------------+ -| Nix | -| [ commmand line interface ]------, | -| | | | -| evaluates | | -| | manages | -| V | | -| [ configuration language ] | | -| | | | -| +-----------------------------|-------------------V-----------+ | -| | store evaluates to | | -| | | | | -| | referenced by V builds | | -| | [ build input ] ---> [ build plan ] ---> [ build result ] | | -| | | | -| +-------------------------------------------------------------+ | -+-----------------------------------------------------------------+ -``` - -At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. - -Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. - -The command line and Nix language are what users interact with most. - -> **Note** -> The Nix language itself does not have a notion of *packages* or *configurations*. -> As far as we are concerned here, the inputs and results of a build plan are just data. - -Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. -It can also execute build plans to produce new data. - -A build plan is a series of *build tasks*. -Each build task has a special build input which is used as *build instructions*. -The result of a build task can be input to another build task. - -``` -+-----------------------------------------------------------------------------------------+ -| store | -| ................................................. | -| : build plan : | -| : : | -| [ build input ]-----instructions-, : | -| : | : | -| : v : | -| [ build input ]----------->[ build task ]--instructions-, : | -| : | : | -| : | : | -| : v : | -| : [ build task ]----->[ build result ] | -| [ build input ]-----instructions-, ^ : | -| : | | : | -| : v | : | -| [ build input ]----------->[ build task ]---------------' : | -| : ^ : | -| : | : | -| [ build input ]------------------' : | -| : : | -| : : | -| :...............................................: | -| | -+-----------------------------------------------------------------------------------------+ -``` - -[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers -[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md deleted file mode 100644 index e0eb69f60..000000000 --- a/doc/manual/src/architecture/store/fso.md +++ /dev/null @@ -1,69 +0,0 @@ -# File System Object - -The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object). - -Every file system object is one of the following: - - - File: an executable flag, and arbitrary data for contents - - Directory: mapping of names to child file system objects - - [Symbolic link][symlink]: may point anywhere. - -We call a store object's outermost file system object the *root*. - - data FileSystemObject - = File { isExecutable :: Bool, contents :: Bytes } - | Directory { entries :: Map FileName FileSystemObject } - | SymLink { target :: Path } - -Examples: - -- a directory with contents - - /nix/store/-hello-2.10 - ├── bin - │   └── hello - └── share - ├── info - │   └── hello.info - └── man - └── man1 - └── hello.1.gz - -- a directory with relative symlink and other contents - - /nix/store/-go-1.16.9 - ├── bin -> share/go/bin - ├── nix-support/ - └── share/ - -- a directory with absolute symlink - - /nix/store/d3k...-nodejs - └── nix_node -> /nix/store/f20...-nodejs-10.24. - -A bare file or symlink can be a root file system object. -Examples: - - /nix/store/-hello-2.10.tar.gz - - /nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc - -Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended. -Examples: - -- an arbitrarily symlinked file may change or not exist at all - - /nix/store/-foo - └── foo -> /home/foo - -- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted - - /nix/store/-bar - └── bar -> /nix/store/abc...-foo - -Nix file system objects do not support [hard links][hardlink]: -each file system object which is not the root has exactly one parent and one name. -However, as store objects are immutable, an underlying file system can use hard links for optimization. - -[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link -[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md deleted file mode 100644 index 663f04f46..000000000 --- a/doc/manual/src/architecture/store/path.md +++ /dev/null @@ -1,105 +0,0 @@ -# Store Path - -Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*. - -Store paths are pairs of - -- a 20-byte [digest](#digest) for identification -- a symbolic name for people to read. - -Example: - -- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` -- name: `firefox-33.1` - -It is rendered to a file system path as the concatenation of - - - [store directory](#store-directory) - - path-separator (`/`) - - [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) - - hyphen (`-`) - - name - -Example: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - |--------| |------------------------------| |----------| - store directory digest name - -## Store Directory - -Every [store](./store.md) has a store directory. - -If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path). - -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. - -> **Note** -> The store directory defaults to `/nix/store`, but is in principle arbitrary. - -It is important which store a given store object belongs to: -Files in the store object can contain store paths, and processes may read these paths. -Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries. - -Therefore one can only copy store objects to a different store if - -- the source and target stores' directories match - - or - -- the store object in question has no references, that is, contains no store paths. - -One cannot copy a store object to a store with a different store directory. -Instead, it has to be rebuilt, together with all its dependencies. -It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. - -# Digest - -In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. - -Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). - -> **Historical Note** -> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. - -[digest]: https://en.m.wiktionary.org/wiki/digest#Noun -[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function -[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 - -### Reference scanning - -When a new store object is built, Nix scans its file contents for store paths to construct its set of references. - -The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data. -Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives. - -This way, scanning files captures run time dependencies without the user having to declare them explicitly. -Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. - -> **Note** -> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. -This depends on the specifics of the software to build and run. -> -> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. - -## Input Addressing - -Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. - -To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes. - -Nix has a custom serialisation format called Nix Archive (NAR) - -Store object references of this sort can *not* be validated from the content of the store object. -Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. - -## Content Addressing - -Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. -If one knows content addressing was used, one can recalculate the reference and thus verify the store object. - -Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance. -Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md). - diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md deleted file mode 100644 index 08b6701d5..000000000 --- a/doc/manual/src/architecture/store/store.md +++ /dev/null @@ -1,151 +0,0 @@ -# Store - -A Nix store is a collection of *store objects* with references between them. -It supports operations to manipulate that collection. - -The following concept map is a graphical outline of this chapter. -Arrows indicate suggested reading order. - -``` - ,--------------[ store ]----------------, - | | | - v v v - [ store object ] [ closure ]--, [ operations ] - | | | | | | - v | | v v | - [ files and processes ] | | [ garbage collection ] | - / \ | | | - v v | v v -[ file system object ] [ store path ] | [ derivation ]--->[ building ] - | ^ | | | - v | v v | - [ digest ]----' [ reference scanning ]<------------' - / \ - v v -[ input addressing ] [ content addressing ] -``` - -## Store Object - -A store object can hold - -- arbitrary *data* -- *references* to other store objects. - -Store objects can be build inputs, build results, or build tasks. - -Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. - -## Reference - -A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: -The only way to obtain references is by adding or building store objects. -A reference will always point to exactly one store object. - -## Operations - -A Nix store can *add*, *retrieve*, and *delete* store objects. - - [ data ] - | - V - [ store ] ---> add ----> [ store' ] - | - V - [ reference ] - - - - [ reference ] - | - V - [ store ] ---> get - | - V - [ store object ] - - - - [ reference ] - | - V - [ store ] --> delete --> [ store' ] - - -It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. - - - [ reference ] - | - V - [ store ] --> build --(maybe)--> [ store' ] - | - V - [ reference ] - - -As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. - - - [ store ] --> collect garbage --> [ store' ] - -## Files and Processes - -Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. -That allows processes to resolve references contained in files and thus access the contents of store objects. - -Store objects are therefore implemented as the pair of - - - a [file system object](fso.md) for data - - a set of [store paths](path.md) for references. - -[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file -[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor - -The following diagram shows a radical simplification of how Nix interacts with the operating system: -It uses files as build inputs, and build outputs are files again. -On the operating system, files can be run as processes, which in turn operate on files. -A build function also amounts to an operating system process (not depicted). - -``` -+-----------------------------------------------------------------+ -| Nix | -| [ commmand line interface ]------, | -| | | | -| evaluates | | -| | manages | -| V | | -| [ configuration language ] | | -| | | | -| +-----------------------------|-------------------V-----------+ | -| | store evaluates to | | -| | | | | -| | referenced by V builds | | -| | [ build input ] ---> [ build plan ] ---> [ build result ] | | -| | ^ | | | -| +---------|----------------------------------------|----------+ | -+-----------|----------------------------------------|------------+ - | | - file system object store path - | | -+-----------|----------------------------------------|------------+ -| operating system +------------+ | | -| '------------ | | <-----------' | -| | file | | -| ,-- | | <-, | -| | +------------+ | | -| execute as | | read, write, execute | -| | +------------+ | | -| '-> | process | --' | -| +------------+ | -+-----------------------------------------------------------------+ -``` - -There exist different types of stores, which all follow this model. -Examples: -- store on the local file system -- remote store accessible via SSH -- binary cache store accessible via HTTP - -To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. - diff --git a/doc/manual/src/architecture/store/store/build-system-terminology.md b/doc/manual/src/architecture/store/store/build-system-terminology.md deleted file mode 100644 index eefbaa630..000000000 --- a/doc/manual/src/architecture/store/store/build-system-terminology.md +++ /dev/null @@ -1,32 +0,0 @@ -# A [Rosetta stone][rosetta-stone] for build system terminology - -The Nix store's design is comparable to other build systems. -Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. - -The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. - -| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | -| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | -| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | -| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | -| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | -| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | -| build | build | build | application of `Build` | evaluation | -| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | - -All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). - -[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone -[bazel]: https://bazel.build/start/bazel-intro -[bazel-artifact]: https://bazel.build/reference/glossary#artifact -[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html -[bazel-action]: https://bazel.build/reference/glossary#action -[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph -[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph -[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache -[thunk]: https://en.m.wikipedia.org/wiki/Thunk -[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph -[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming -[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf -[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf -[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf diff --git a/doc/manual/src/architecture/store/store/closure.md b/doc/manual/src/architecture/store/store/closure.md deleted file mode 100644 index 065b95ffc..000000000 --- a/doc/manual/src/architecture/store/store/closure.md +++ /dev/null @@ -1,29 +0,0 @@ -# Closure - -Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. - -The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. - -Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - -- A newly added store object cannot have references, unless it is a build task. - -- Build results must only refer to store objects in the closure of the build inputs. - - Building a store object will add appropriate references, according to the build task. - -- Store objects being copied must refer to objects already in the destination store. - - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete store objects which are not reachable from any reference still in use. - - - -[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity -[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) -[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object -[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type -[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier - - From c21b1a7e67cc28b6e95a563daa786f385bc716b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:14:14 +0200 Subject: [PATCH 227/371] Spelling --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ff7d2d0a..e9d721e59 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -750,7 +750,7 @@ public: this, 0, "download-speed", R"( Specify the maximum transfer rate in kilobytes per second you want - nix to use for download. + Nix to use for downloads. )"}; Setting netrcFile{ From 0d2163c6dcf03463fa91ec6d0d96c928ad907366 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:27:36 +0200 Subject: [PATCH 228/371] nix repl: Stop the progress bar The repl was broken since c3769c68465bae971ab6bb48cfcdea85b61ea83a. In general, the progress bar is incompatible with the repl. --- src/libcmd/repl.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 23df40337..150bd42ac 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -35,6 +35,7 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" +#include "progress-bar.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -252,6 +253,10 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif + /* Stop the progress bar because it interferes with the display of + the repl. */ + stopProgressBar(); + std::string input; while (true) { @@ -1037,9 +1042,10 @@ void runRepl( struct CmdRepl : InstallablesCommand { - CmdRepl(){ + CmdRepl() { evalSettings.pureEval = false; } + void prepare() { if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { @@ -1053,12 +1059,15 @@ struct CmdRepl : InstallablesCommand } installables = InstallablesCommand::load(); } + std::vector files; + Strings getDefaultFlakeAttrPaths() override { return {""}; } - virtual bool useDefaultInstallables() override + + bool useDefaultInstallables() override { return file.has_value() or expr.has_value(); } From 062e4fcdde145ec6780df8d1002dc7380f6eb4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 16 Aug 2022 12:23:37 +0200 Subject: [PATCH 229/371] JSON: print paths as strings without copying them to the store Makes `printValueAsJSON` not copy paths to the store for `nix eval --json`, `nix-instantiate --eval --json` and `nix-env --json`. Fixes https://github.com/NixOS/nix/issues/5612 --- src/libexpr/value-to-json.cc | 21 ++++++++++++--------- src/libexpr/value-to-json.hh | 4 ++-- src/libexpr/value.hh | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/eval.cc | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 03504db61..4d63d8b49 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,7 +10,7 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context) + Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore) { checkInterrupt(); @@ -32,7 +32,10 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nPath: - out.write(state.copyPathToStore(context, v.path)); + if (copyToStore) + out.write(state.copyPathToStore(context, v.path)); + else + out.write(v.path); break; case nNull: @@ -54,10 +57,10 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context); + printValueAsJSON(state, strict, *a.value, a.pos, placeholder, context, copyToStore); } } else - printValueAsJSON(state, strict, *i->value, i->pos, out, context); + printValueAsJSON(state, strict, *i->value, i->pos, out, context, copyToStore); break; } @@ -65,13 +68,13 @@ void printValueAsJSON(EvalState & state, bool strict, auto list(out.list()); for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *elem, pos, placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context, copyToStore); } break; } case nExternal: - v.external->printValueAsJSON(state, strict, out, context); + v.external->printValueAsJSON(state, strict, out, context, copyToStore); break; case nFloat: @@ -91,14 +94,14 @@ void printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context) + Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, pos, out, context); + printValueAsJSON(state, strict, v, pos, out, context, copyToStore); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const + JSONPlaceholder & out, PathSet & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index c020a817a..7ddc8a5b1 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context); + Value & v, const PosIdx pos, JSONPlaceholder & out, PathSet & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context); + Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 2008df74d..590ba7783 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -99,7 +99,7 @@ class ExternalValueBase /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual void printValueAsJSON(EvalState & state, bool strict, - JSONPlaceholder & out, PathSet & context) const; + JSONPlaceholder & out, PathSet & context, bool copyToStore = true) const; /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a69d3700d..c4e981d49 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -947,7 +947,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context, false); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index d3144e131..6181d2190 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -53,7 +53,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); else { if (strict) state.forceValueDeep(vRes); vRes.print(state.symbols, std::cout); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 967dc8519..cc022ae43 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -116,7 +116,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, pos, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); } else { From 4c2ff4a0f4c2106d5792a87a1ba9ee1fd18c0e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 22 Aug 2022 15:07:52 +0200 Subject: [PATCH 230/371] JSON: add missing newlines after `nix eval --json` and `nix-instantiate --eval --json`. --- src/nix-instantiate/nix-instantiate.cc | 5 +++-- src/nix/eval.cc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6181d2190..80f35828c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -52,9 +52,10 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); - else if (output == okJSON) + else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); - else { + std::cout << std::endl; + } else { if (strict) state.forceValueDeep(vRes); vRes.print(state.symbols, std::cout); std::cout << std::endl; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index cc022ae43..ddd2790c6 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -117,6 +117,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, pos, jsonOut, context, false); + std::cout << std::endl; } else { From f865048332a26f69a007881877203b9428783357 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 15:30:38 +0200 Subject: [PATCH 231/371] Indentation --- src/libstore/remote-store.hh | 10 ++++------ src/libstore/store-api.hh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 5a599997e..11d089cd2 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -89,12 +89,10 @@ public: CheckSigsFlag checkSigs) override; void addMultipleToStore( - PathsSource & pathsToCopy, - Activity & act, - RepairFlag repair, - CheckSigsFlag checkSigs) override; - - + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0a61115b..c8a667c6d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,12 +369,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, - CheckSigsFlag checkSigs = CheckSigs - ); + CheckSigsFlag checkSigs = CheckSigs); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. From f0358ed4650e4608a383bd9f59ee23545f86c4ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:14:47 +0200 Subject: [PATCH 232/371] Fix a hang in nix-copy-ssh.sh This hang for some reason didn't trigger in the Nix build, but did running 'make installcheck' interactively. What happened: * Store::addMultipleToStore() calls a SinkToSource object to copy a path, which in turn calls LegacySSHStore::narFromPath(), which acquires a connection. * The SinkToSource object is not destroyed after the last bytes has been read, so the coroutine's stack is still alive and its destructors are not run. So the connection is not released. * Then when the next path is copied, because max-connections = 1, LegacySSHStore::narFromPath() hangs forever waiting for a connection to be released. The fix is to make sure that the source object is destroyed when we're done with it. --- src/libstore/store-api.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1406bf657..9c3a0b3d6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,7 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include "finally.hh" #include @@ -271,7 +272,7 @@ void Store::addMultipleToStore( using PathWithInfo = std::pair>; - std::map infosMap; + std::map infosMap; StorePathSet storePathsToAdd; for (auto & thingToAdd : pathsToCopy) { infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); @@ -288,7 +289,8 @@ void Store::addMultipleToStore( storePathsToAdd, [&](const StorePath & path) { - auto & [info, source] = *infosMap.at(path); + + auto & [info, _] = *infosMap.at(path); if (isValidPath(info.path)) { nrDone++; @@ -309,12 +311,21 @@ void Store::addMultipleToStore( auto info = info_; info.ultimate = false; + /* Make sure that the Source object is destroyed when + we're done. In particular, a SinkToSource object must + be destroyed to ensure that the destructors on its + stack frame are run; this includes + LegacySSHStore::narFromPath()'s connection lock. */ + Finally cleanupSource{[&]() { + source.reset(); + }}; + if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); showProgress(); try { addToStore(info, *source, repair, checkSigs); - } catch (Error &e) { + } catch (Error & e) { nrFailed++; if (!settings.keepGoing) throw e; From ff0b5a778c41a94075d7c651477c4a3a8b4b00cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Tue, 23 Aug 2022 14:40:27 +0200 Subject: [PATCH 233/371] Revert to `copyStore = true` for `nix-instantiate` and `nix-env` --- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c4e981d49..a69d3700d 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -947,7 +947,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context, false); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 80f35828c..6b5ba595d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -53,7 +53,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) { - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context, false); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); std::cout << std::endl; } else { if (strict) state.forceValueDeep(vRes); From db026103b18fb8b5a719594502edd0f89eb9c268 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:57:08 +0200 Subject: [PATCH 234/371] nix develop: Ignore some more bash special variables Fixes #6940. --- src/nix/get-env.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 42c806450..a7a8a01b9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -43,6 +43,7 @@ __dumpEnv() { local __var_name="${BASH_REMATCH[2]}" if [[ $__var_name =~ ^BASH_ || \ + $__var_name =~ ^COMP_ || \ $__var_name = _ || \ $__var_name = DIRSTACK || \ $__var_name = EUID || \ @@ -54,7 +55,9 @@ __dumpEnv() { $__var_name = PWD || \ $__var_name = RANDOM || \ $__var_name = SHLVL || \ - $__var_name = SECONDS \ + $__var_name = SECONDS || \ + $__var_name = EPOCHREALTIME || \ + $__var_name = EPOCHSECONDS \ ]]; then continue; fi if [[ -z $__first ]]; then printf ',\n'; else __first=; fi From 4d4f2d10e7ffceac7a700abe0004ffb5b2b23b33 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Tue, 23 Aug 2022 19:38:53 -0400 Subject: [PATCH 235/371] darwin-install: fix shell hint --- scripts/install-darwin-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index afaa6783b..5111a5dde 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -167,7 +167,7 @@ poly_user_shell_get() { } poly_user_shell_set() { - _sudo "in order to give $1 a safe home directory" \ + _sudo "in order to give $1 a safe shell" \ /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" } From c2d74569269d9642448a701aa6efe08ec0fe6f00 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 24 Aug 2022 01:54:43 +0200 Subject: [PATCH 236/371] Fix a misplaced parenthese in serve protocol check This issue made it impossible for clients using a serve protocol of version <= 2.3 to use the `cmdBuildDerivation` command of servers using a protocol of version >= 2.6. The faulty version check makes the server send back build outputs that the client is not expecting. --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b453ea1ca..23f2ad3cf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -922,7 +922,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion >= 6)) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { worker_proto::write(*store, out, status.builtOutputs); } From 8d906b1f3bd4343b6b309ddfca824d5dd00a09b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:11:03 +0200 Subject: [PATCH 237/371] Fix macOS build --- src/libstore/store-api.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9c3a0b3d6..2cd6c15ec 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -307,8 +307,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info_, source] = *infosMap.at(path); + auto & [info_, source_] = *infosMap.at(path); auto info = info_; + auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when From 56d97d4b4df02e3464a2f003a90b7f6abae16722 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:49:58 +0200 Subject: [PATCH 238/371] Remove redundant Finally --- src/libstore/store-api.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2cd6c15ec..86b12257a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,7 +11,6 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" -#include "finally.hh" #include @@ -309,7 +308,6 @@ void Store::addMultipleToStore( auto & [info_, source_] = *infosMap.at(path); auto info = info_; - auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when @@ -317,9 +315,7 @@ void Store::addMultipleToStore( be destroyed to ensure that the destructors on its stack frame are run; this includes LegacySSHStore::narFromPath()'s connection lock. */ - Finally cleanupSource{[&]() { - source.reset(); - }}; + auto source = std::move(source_); if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); From 8e5659423eb9c64cefa3c85af8bed78c4e82eb08 Mon Sep 17 00:00:00 2001 From: Winter Date: Wed, 24 Aug 2022 13:09:44 -0400 Subject: [PATCH 239/371] fix(libstore): allow access to trustd on macOS --- src/libstore/sandbox-network.sb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb index 56beec761..19e9eea9a 100644 --- a/src/libstore/sandbox-network.sb +++ b/src/libstore/sandbox-network.sb @@ -14,3 +14,7 @@ ; Allow DNS lookups. (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) + +; Allow access to trustd. +(allow mach-lookup (global-name "com.apple.trustd")) +(allow mach-lookup (global-name "com.apple.trustd.agent")) From a17ce0a8a9c3dd96d0a5043a80acab0c6810a199 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 21:19:43 +0200 Subject: [PATCH 240/371] Fix evaluation cache 98e361ad4c1a26d4ffe4762a6f33bb9e39321a39 introduced a regression where previously stored attributes were replaced by placeholders. As a result, a command like 'nix build nixpkgs#hello' had to be executed at least twice to get caching. This code does not seem necessary for suggestions to work. --- src/libexpr/eval-cache.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 0d83b6cfe..b259eec63 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -507,11 +507,6 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); - for (auto & attr : *v.attrs) { - if (root->db) - root->db->setPlaceholder({cachedValue->first, attr.name}); - } - auto attr = v.attrs->get(name); if (!attr) { From bb411e4ae16d6a5c61ea595c0c12e2ecee081ff9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:36:40 +0200 Subject: [PATCH 241/371] Fix progress bar flicker with -L This was caused by -L calling setLogFormat() again, which caused the creation of a new progress bar without destroying the old one. So we had two progress bars clobbering each other. We should change 'logger' to be a smart pointer, but I'll do that in a future PR. Fixes #6931. --- src/libmain/loggers.cc | 7 +++++-- src/libmain/progress-bar.cc | 23 ++++++++++++----------- src/libmain/progress-bar.hh | 4 ++-- src/libutil/logging.hh | 3 +++ src/nix/main.cc | 2 +- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cdf23859b..cda5cb939 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -30,8 +30,11 @@ Logger * makeDefaultLogger() { return makeJSONLogger(*makeSimpleLogger(true)); case LogFormat::bar: return makeProgressBar(); - case LogFormat::barWithLogs: - return makeProgressBar(true); + case LogFormat::barWithLogs: { + auto logger = makeProgressBar(); + logger->setPrintBuildLogs(true); + return logger; + } default: abort(); } diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 5183f212f..0bbeaff8d 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -81,14 +81,13 @@ private: std::condition_variable quitCV, updateCV; - bool printBuildLogs; + bool printBuildLogs = false; bool isTTY; public: - ProgressBar(bool printBuildLogs, bool isTTY) - : printBuildLogs(printBuildLogs) - , isTTY(isTTY) + ProgressBar(bool isTTY) + : isTTY(isTTY) { state_.lock()->active = isTTY; updateThread = std::thread([&]() { @@ -503,19 +502,21 @@ public: draw(*state); return s[0]; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { + this->printBuildLogs = printBuildLogs; + } }; -Logger * makeProgressBar(bool printBuildLogs) +Logger * makeProgressBar() { - return new ProgressBar( - printBuildLogs, - shouldANSI() - ); + return new ProgressBar(shouldANSI()); } -void startProgressBar(bool printBuildLogs) +void startProgressBar() { - logger = makeProgressBar(printBuildLogs); + logger = makeProgressBar(); } void stopProgressBar() diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 7f0dafecf..3a76f8448 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -4,9 +4,9 @@ namespace nix { -Logger * makeProgressBar(bool printBuildLogs = false); +Logger * makeProgressBar(); -void startProgressBar(bool printBuildLogs = false); +void startProgressBar(); void stopProgressBar(); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 6f81b92de..d0817b4a9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -111,6 +111,9 @@ public: virtual std::optional ask(std::string_view s) { return {}; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { } }; ActivityId getCurActivity(); diff --git a/src/nix/main.cc b/src/nix/main.cc index a8404a2ea..f434e9655 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -82,7 +82,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .shortName = 'L', .description = "Print full build logs on standard error.", .category = loggingCategory, - .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, + .handler = {[&]() { logger->setPrintBuildLogs(true); }}, }); addFlag({ From d046eb1463fad967d47cc5becfe5b7a08b5adfa8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:42:34 +0200 Subject: [PATCH 242/371] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index f161b5d80..ed0edc885 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.10.0 \ No newline at end of file +2.11.0 \ No newline at end of file From b0488a29dc7401e5ecd9221215da5ea9879e56d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:44:58 +0200 Subject: [PATCH 243/371] Branch 2.11 release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.11.md | 5 +++++ doc/manual/src/release-notes/rl-next.md | 3 --- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.11.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8fbb59716..9b66ec3db 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -73,6 +73,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) diff --git a/doc/manual/src/release-notes/rl-2.11.md b/doc/manual/src/release-notes/rl-2.11.md new file mode 100644 index 000000000..b322a4e5e --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.11.md @@ -0,0 +1,5 @@ +# Release 2.11 (2022-08-24) + +* `nix copy` now copies the store paths in parallel as much as possible (again). + This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything + in one batch to avoid latencies issues. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7d82c3dc4..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,2 @@ # Release X.Y (202?-??-??) -* `nix copy` now copies the store paths in parallel as much as possible (again). - This doesn't apply for the `daemon` and `ssh-ng` stores which copy everything - in one batch to avoid latencies issues. \ No newline at end of file From 57cf36f81e4f00ed7c67f159f2de11978470563f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Aug 2022 11:50:14 +0200 Subject: [PATCH 244/371] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ed0edc885..3ca2c9b2c 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.11.0 \ No newline at end of file +2.12.0 \ No newline at end of file From e7dcacbd7ca75bbac8b77e6873160feff875c9c7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 26 Aug 2022 14:03:40 +0200 Subject: [PATCH 245/371] do not render relative links in help pages this simplifies the setup a lot, and avoids weird looking `./file.md` links showing up. it also does not show regular URLs any more. currently the command reference only has few of them, and not showing them in the offline documentation is hopefully not a big deal. instead of building more special-case solutions, clumsily preprocessing the input, or issuing verbal rules on dealing with URLs, should better be solved sustainably by not rendering relative links in `lowdown`: https://github.com/kristapsdz/lowdown/issues/105 --- doc/manual/generate-manpage.nix | 6 ++---- doc/manual/local.mk | 4 ++-- src/libcmd/markdown.cc | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 244cfa0c2..17701c3a3 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,4 +1,4 @@ -{ command, renderLinks ? false }: +{ command }: with builtins; with import ./utils.nix; @@ -21,9 +21,7 @@ let listCommands = cmds: concatStrings (map (name: "* " - + (if renderLinks - then "[`${command} ${name}`](./${appendName filename name}.md)" - else "`${command} ${name}`") + + "[`${command} ${name}`](./${appendName filename name}.md)" + " - ${cmds.${name}.description}\n") (attrNames cmds)); in diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 66a8cb7de..364e02967 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -50,7 +50,7 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; renderLinks = true; }' + $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { command = builtins.readFile $<; }' $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp @@ -96,7 +96,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli if [[ $$name = SUMMARY ]]; then continue; fi; \ printf "Title: %s\n\n" "$$name" > $$tmpFile; \ cat $$i >> $$tmpFile; \ - lowdown -sT man -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ + lowdown -sT man --nroff-nolinks -M section=1 $$tmpFile -o $(DESTDIR)$$(dirname $@)/$$name.1; \ rm $$tmpFile; \ done @touch $@ diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 71f9c8dff..668a07763 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -18,7 +18,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown) .hmargin = 0, .vmargin = 0, .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, - .oflags = 0, + .oflags = LOWDOWN_TERM_NOLINK, }; auto doc = lowdown_doc_new(&opts); From 2b9d38130161116081d24630b99d1a437fa3bdb2 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Sun, 5 Jun 2022 22:45:37 +0200 Subject: [PATCH 246/371] Fix nested flake input overrides --- src/libexpr/flake/flake.cc | 47 ++++++++++++++++++++++++++---------- tests/flakes/follow-paths.sh | 30 +++++++++++++++++++++++ 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 105e76bc6..81762a0af 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -90,11 +90,11 @@ static void expectType(EvalState & state, ValueType type, static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath); + const std::optional & baseDir, InputPath lockRootPath, unsigned depth); static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) + const std::optional & baseDir, InputPath lockRootPath, unsigned depth) { expectType(state, nAttrs, *value, pos); @@ -118,7 +118,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath, depth + 1); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->string.s)); @@ -163,7 +163,11 @@ static FlakeInput parseFlakeInput(EvalState & state, input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); } - if (!input.follows && !input.ref) + if (!input.follows && !input.ref && depth == 0) + // in `input.nixops.inputs.nixpkgs.url = ...`, we assume `nixops` is from + // the flake registry absent `ref`/`follows`, but we should not assume so + // about `nixpkgs` (where `depth == 1`) as the `nixops` flake should + // determine its default source input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); return input; @@ -171,7 +175,7 @@ static FlakeInput parseFlakeInput(EvalState & state, static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath) + const std::optional & baseDir, InputPath lockRootPath, unsigned depth) { std::map inputs; @@ -184,7 +188,8 @@ static std::map parseFlakeInputs( inputAttr.value, inputAttr.pos, baseDir, - lockRootPath)); + lockRootPath, + depth)); } return inputs; @@ -230,7 +235,7 @@ static Flake getFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath, 0); auto sOutputs = state.symbols.create("outputs"); @@ -313,6 +318,19 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } +/* Recursively merge `overrides` into `overrideMap` */ +static void updateOverrides(std::map & overrideMap, const FlakeInputs & overrides, + const InputPath & inputPathPrefix) +{ + for (auto & [id, input] : overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + // Do not override existing assignment from outer flake + overrideMap.insert({inputPath, input}); + updateOverrides(overrideMap, input.overrides, inputPath); + } +} + /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( @@ -375,12 +393,9 @@ LockedFlake lockFlake( /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { - for (auto & [idOverride, inputOverride] : input.overrides) { - auto inputPath(inputPathPrefix); - inputPath.push_back(id); - inputPath.push_back(idOverride); - overrides.insert_or_assign(inputPath, inputOverride); - } + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + updateOverrides(overrides, input.overrides, inputPath); } /* Check whether this input has overrides for a @@ -415,6 +430,12 @@ LockedFlake lockFlake( // Respect the “flakeness” of the input even if we // override it i->second.isFlake = input2.isFlake; + if (!i->second.ref) + i->second.ref = input2.ref; + if (!i->second.follows) + i->second.follows = input2.follows; + // Note that `input.overrides` is not used in the following, + // so no need to merge it here (already done by `updateOverrides`) } auto & input = hasOverride ? i->second : input2; diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index 19cc1bafa..e6751b06e 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -148,3 +148,33 @@ git -C $flakeFollowsA add flake.nix nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" + +# Test nested flake overrides + +cat < $flakeFollowsD/flake.nix +{ outputs = _: {}; } +EOF +cat < $flakeFollowsC/flake.nix +{ + inputs.D.url = "path:nosuchflake"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsB/flake.nix +{ + inputs.C.url = "path:$flakeFollowsC"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsA/flake.nix +{ + inputs.B.url = "path:$flakeFollowsB"; + inputs.D.url = "path:$flakeFollowsD"; + inputs.B.inputs.C.inputs.D.follows = "D"; + outputs = _: {}; +} +EOF + +nix flake lock $flakeFollowsA + +[[ $(jq -c .nodes.C.inputs.D $flakeFollowsA/flake.lock) = '["D"]' ]] From 6f65c117802ef4683eeaf64bc079cc3a8b9608c2 Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Sun, 28 Aug 2022 11:46:29 +0200 Subject: [PATCH 247/371] Fix overlapping flake follows --- src/libexpr/flake/lockfile.cc | 2 +- tests/flakes/follow-paths.sh | 32 +++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 60b52d578..384ead05b 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -338,7 +338,7 @@ void LockFile::check() for (auto & [inputPath, input] : inputs) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !get(inputs, *follows)) + if (!follows->empty() && !findInput(*follows)) throw Error("input '%s' follows a non-existent input '%s'", printInputPath(inputPath), printInputPath(*follows)); diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index e6751b06e..c12dbe0f6 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -149,7 +149,7 @@ git -C $flakeFollowsA add flake.nix nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" -# Test nested flake overrides +# Test nested flake overrides: A overrides B/C/D cat < $flakeFollowsD/flake.nix { outputs = _: {}; } @@ -178,3 +178,33 @@ EOF nix flake lock $flakeFollowsA [[ $(jq -c .nodes.C.inputs.D $flakeFollowsA/flake.lock) = '["D"]' ]] + +# Test overlapping flake follows: B has D follow C/D, while A has B/C follow C + +cat < $flakeFollowsC/flake.nix +{ + inputs.D.url = "path:$flakeFollowsD"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsB/flake.nix +{ + inputs.C.url = "path:nosuchflake"; + inputs.D.url = "path:nosuchflake"; + inputs.D.follows = "C/D"; + outputs = _: {}; +} +EOF +cat < $flakeFollowsA/flake.nix +{ + inputs.B.url = "path:$flakeFollowsB"; + inputs.C.url = "path:$flakeFollowsC"; + inputs.B.inputs.C.follows = "C"; + outputs = _: {}; +} +EOF + +# bug was not triggered without recreating the lockfile +nix flake lock $flakeFollowsA --recreate-lock-file + +[[ $(jq -c .nodes.B.inputs.D $flakeFollowsA/flake.lock) = '["B","C","D"]' ]] From 90ba893329e4eda93682632d2e672f5016617e3e Mon Sep 17 00:00:00 2001 From: Alex <93376818+sashashura@users.noreply.github.com> Date: Sun, 28 Aug 2022 21:59:29 +0100 Subject: [PATCH 248/371] Update backport.yml --- .github/workflows/backport.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 3a2d4de0e..75be788ef 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -2,9 +2,15 @@ name: Backport on: pull_request_target: types: [closed, labeled] +permissions: + contents: read jobs: backport: name: Backport Pull Request + permissions: + # for zeebe-io/backport-action + contents: write + pull-requests: write if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: From 85248543b5520e0ac7655b0ce1ebfa2543992498 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Mon, 29 Aug 2022 22:17:06 +1000 Subject: [PATCH 249/371] Only push Docker image when Docker secrets are set --- .github/workflows/ci.yml | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 956f81684..86b5dfd2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ permissions: read-all jobs: tests: - needs: [check_cachix] + needs: [check_secrets] strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -22,30 +22,34 @@ jobs: - uses: cachix/install-nix-action@v17 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 - if: needs.check_cachix.outputs.secret == 'true' + if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' flake check -L - check_cachix: + check_secrets: permissions: contents: none - name: Cachix secret present for installer tests + name: Check Cachix and Docker secrets present for installer tests runs-on: ubuntu-latest outputs: - secret: ${{ steps.secret.outputs.secret }} + cachix: ${{ steps.secret.outputs.cachix }} + docker: ${{ steps.secret.outputs.docker }} steps: - - name: Check for Cachix secret + - name: Check for secrets id: secret env: _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" installer: - needs: [tests, check_cachix] - if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' + needs: [tests, check_secrets] + if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' runs-on: ubuntu-latest outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} @@ -64,8 +68,8 @@ jobs: run: scripts/prepare-installer-for-github-actions installer_test: - needs: [installer, check_cachix] - if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' + needs: [installer, check_secrets] + if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' strategy: matrix: os: [ubuntu-latest, macos-latest] @@ -80,11 +84,12 @@ jobs: - run: nix-instantiate -E 'builtins.currentTime' --eval docker_push_image: - needs: [check_cachix, tests] + needs: [check_secrets, tests] if: >- github.event_name == 'push' && github.ref_name == 'master' && - needs.check_cachix.outputs.secret == 'true' + needs.check_secrets.outputs.cachix == 'true' && + needs.check_secrets.outputs.docker == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -94,7 +99,7 @@ jobs: - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 - if: needs.check_cachix.outputs.secret == 'true' + if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' From a2b7baa42f9e5af5648d1b838aff72f1ad03974c Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Wed, 31 Aug 2022 17:25:26 -0400 Subject: [PATCH 250/371] Set `HOME` var to `root`'s home when running `nix-store` as `root` A [recent-ish change](https://github.com/NixOS/nix/pull/6676) logs a warning when a potentially counterintuitive situation happens. This now causes the multi-user installer to [emit a warning](https://github.com/NixOS/nixpkgs/issues/189043) when it's doing the "seed the Nix database" step via a low-level `nix-store --load-db` invocation. `nix-store` functionality implementations don't actually use profiles or channels or homedir as far as i can tell. So why are we hitting this code at all? Well, the current command approach for functionality here builds a [fat `nix` binary](https://github.com/NixOS/nix/blob/master/src/nix/local.mk#L23-L26) which has _all_ the functionality of previous individual binaries (nix-env, nix-store, etc) bundled in, then [uses the invocation name](https://github.com/NixOS/nix/blob/master/src/nix/main.cc#L274-L277) to select the set of commands to expose. `nix` itself has this behavior, even when just trying to parse the (sub)command and arguments: ``` dave @ davembp2 $ nix error: no subcommand specified Try 'nix --help' for more information. dave @ davembp2 $ sudo nix warning: $HOME ('/Users/dave') is not owned by you, falling back to the one defined in the 'passwd' file error: no subcommand specified Try 'nix --help' for more information. dave @ davembp2 $ HOME=~root sudo nix error: no subcommand specified Try 'nix --help' for more information. ``` This behavior can also be seen pretty easily with an arbitrary `nix-store` invocation: ``` dave @ davembp2 $ nix-store --realize dave @ davembp2 $ sudo nix-store --realize # what installer is doing now warning: $HOME ('/Users/dave') is not owned by you, falling back to the one defined in the 'passwd' file dave @ davembp2 $ sudo HOME=~root nix-store --realize # what this PR effectively does dave @ davembp2 $ ``` --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 01dbf0c0e..105b84af6 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -810,7 +810,7 @@ EOF fi _sudo "to load data for the first time in to the Nix Database" \ - "$NIX_INSTALLED_NIX/bin/nix-store" --load-db < ./.reginfo + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-store" --load-db < ./.reginfo echo " Just finished getting the nix database ready." ) From a8b3d777fbdaf0b732f129e5be62cd2a1227674b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Sep 2022 15:26:19 +0200 Subject: [PATCH 251/371] Revert "Merge pull request #6621 from Kha/nested-follows" This reverts commit c530cda345377370c52a616d608de88b9d67cd40, reversing changes made to 4adcdff5c1d5f9f135c4ec61d690890443c19e6a. --- src/libexpr/flake/flake.cc | 47 ++++++++------------------- src/libexpr/flake/lockfile.cc | 2 +- tests/flakes/follow-paths.sh | 60 ----------------------------------- 3 files changed, 14 insertions(+), 95 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 81762a0af..105e76bc6 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -90,11 +90,11 @@ static void expectType(EvalState & state, ValueType type, static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath, unsigned depth); + const std::optional & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath, unsigned depth) + const std::optional & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -118,7 +118,7 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath, depth + 1); + input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); auto follows(parseInputPath(attr.value->string.s)); @@ -163,11 +163,7 @@ static FlakeInput parseFlakeInput(EvalState & state, input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); } - if (!input.follows && !input.ref && depth == 0) - // in `input.nixops.inputs.nixpkgs.url = ...`, we assume `nixops` is from - // the flake registry absent `ref`/`follows`, but we should not assume so - // about `nixpkgs` (where `depth == 1`) as the `nixops` flake should - // determine its default source + if (!input.follows && !input.ref) input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); return input; @@ -175,7 +171,7 @@ static FlakeInput parseFlakeInput(EvalState & state, static std::map parseFlakeInputs( EvalState & state, Value * value, const PosIdx pos, - const std::optional & baseDir, InputPath lockRootPath, unsigned depth) + const std::optional & baseDir, InputPath lockRootPath) { std::map inputs; @@ -188,8 +184,7 @@ static std::map parseFlakeInputs( inputAttr.value, inputAttr.pos, baseDir, - lockRootPath, - depth)); + lockRootPath)); } return inputs; @@ -235,7 +230,7 @@ static Flake getFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath, 0); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath); auto sOutputs = state.symbols.create("outputs"); @@ -318,19 +313,6 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } -/* Recursively merge `overrides` into `overrideMap` */ -static void updateOverrides(std::map & overrideMap, const FlakeInputs & overrides, - const InputPath & inputPathPrefix) -{ - for (auto & [id, input] : overrides) { - auto inputPath(inputPathPrefix); - inputPath.push_back(id); - // Do not override existing assignment from outer flake - overrideMap.insert({inputPath, input}); - updateOverrides(overrideMap, input.overrides, inputPath); - } -} - /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( @@ -393,9 +375,12 @@ LockedFlake lockFlake( /* Get the overrides (i.e. attributes of the form 'inputs.nixops.inputs.nixpkgs.url = ...'). */ for (auto & [id, input] : flakeInputs) { - auto inputPath(inputPathPrefix); - inputPath.push_back(id); - updateOverrides(overrides, input.overrides, inputPath); + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } } /* Check whether this input has overrides for a @@ -430,12 +415,6 @@ LockedFlake lockFlake( // Respect the “flakeness” of the input even if we // override it i->second.isFlake = input2.isFlake; - if (!i->second.ref) - i->second.ref = input2.ref; - if (!i->second.follows) - i->second.follows = input2.follows; - // Note that `input.overrides` is not used in the following, - // so no need to merge it here (already done by `updateOverrides`) } auto & input = hasOverride ? i->second : input2; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 384ead05b..60b52d578 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -338,7 +338,7 @@ void LockFile::check() for (auto & [inputPath, input] : inputs) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !findInput(*follows)) + if (!follows->empty() && !get(inputs, *follows)) throw Error("input '%s' follows a non-existent input '%s'", printInputPath(inputPath), printInputPath(*follows)); diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index c12dbe0f6..19cc1bafa 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -148,63 +148,3 @@ git -C $flakeFollowsA add flake.nix nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" - -# Test nested flake overrides: A overrides B/C/D - -cat < $flakeFollowsD/flake.nix -{ outputs = _: {}; } -EOF -cat < $flakeFollowsC/flake.nix -{ - inputs.D.url = "path:nosuchflake"; - outputs = _: {}; -} -EOF -cat < $flakeFollowsB/flake.nix -{ - inputs.C.url = "path:$flakeFollowsC"; - outputs = _: {}; -} -EOF -cat < $flakeFollowsA/flake.nix -{ - inputs.B.url = "path:$flakeFollowsB"; - inputs.D.url = "path:$flakeFollowsD"; - inputs.B.inputs.C.inputs.D.follows = "D"; - outputs = _: {}; -} -EOF - -nix flake lock $flakeFollowsA - -[[ $(jq -c .nodes.C.inputs.D $flakeFollowsA/flake.lock) = '["D"]' ]] - -# Test overlapping flake follows: B has D follow C/D, while A has B/C follow C - -cat < $flakeFollowsC/flake.nix -{ - inputs.D.url = "path:$flakeFollowsD"; - outputs = _: {}; -} -EOF -cat < $flakeFollowsB/flake.nix -{ - inputs.C.url = "path:nosuchflake"; - inputs.D.url = "path:nosuchflake"; - inputs.D.follows = "C/D"; - outputs = _: {}; -} -EOF -cat < $flakeFollowsA/flake.nix -{ - inputs.B.url = "path:$flakeFollowsB"; - inputs.C.url = "path:$flakeFollowsC"; - inputs.B.inputs.C.follows = "C"; - outputs = _: {}; -} -EOF - -# bug was not triggered without recreating the lockfile -nix flake lock $flakeFollowsA --recreate-lock-file - -[[ $(jq -c .nodes.B.inputs.D $flakeFollowsA/flake.lock) = '["B","C","D"]' ]] From f4d7208e235d129ad3b0fe25b518fa36d960ec4d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 1 Sep 2022 11:48:50 -0500 Subject: [PATCH 252/371] Update boehmgc-coroutine-sp-fallback.diff for darwin The darwin_stop_world implementation is slightly different. sp goes to altstack_lo instead of lo in this case. Assuming that is an implementation detail. But the fix is the same, when we detect alstack_lo outside of the expected stack range, we reset it to hi - stack_limit. Here stack_limit is calculated with pthread_get_stacksize_np since that is the BSD equivalent to pthread_attr_getstacksize. --- boehmgc-coroutine-sp-fallback.diff | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index e659bf470..8fdafbecb 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,3 +1,35 @@ +diff --git a/darwin_stop_world.c b/darwin_stop_world.c +index 3dbaa3fb..36a1d1f7 100644 +--- a/darwin_stop_world.c ++++ b/darwin_stop_world.c +@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) + int nthreads = 0; + word total_size = 0; + mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; ++ size_t stack_limit; + if (!EXPECT(GC_thr_initialized, TRUE)) + GC_thr_init(); + +@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) + GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); + } + if (altstack_lo) { ++ // When a thread goes into a coroutine, we lose its original sp until ++ // control flow returns to the thread. ++ // While in the coroutine, the sp points outside the thread stack, ++ // so we can detect this and push the entire thread stack instead, ++ // as an approximation. ++ // We assume that the coroutine has similarly added its entire stack. ++ // This could be made accurate by cooperating with the application ++ // via new functions and/or callbacks. ++ stack_limit = pthread_get_stacksize_np(p->id); ++ if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack ++ altstack_lo = altstack_hi - stack_limit; ++ } ++ + total_size += altstack_hi - altstack_lo; + GC_push_all_stack(altstack_lo, altstack_hi); + } diff --git a/pthread_stop_world.c b/pthread_stop_world.c index 4b2c429..1fb4c52 100644 --- a/pthread_stop_world.c From 520587b9a0327194371146f4dd25c9227a2c79e2 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 13:38:07 -0700 Subject: [PATCH 253/371] glossary: local store: clarify --- doc/manual/src/glossary.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 77de58965..d653a2ae4 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -15,10 +15,10 @@ invoked, the Nix store can be referred to as a "_local_" or a "_remote_" one: - + A *local store* exists on the local filesystem of + + A *local store* exists on the filesystem of the machine where Nix is invoked. You can use other local stores by passing the `--store` flag to the - `nix` command. + `nix` command. Local stores can be used for building derivations. + A *remote store* exists anywhere other than the local filesystem. One example is the `/nix/store` From 2812682ebee9d4419ba89690177b31564ce5ba77 Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+amjoseph-nixpkgs@users.noreply.github.com> Date: Thu, 1 Sep 2022 20:40:39 +0000 Subject: [PATCH 254/371] Update doc/manual/src/glossary.md Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d653a2ae4..f072f35e1 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -52,7 +52,7 @@ from some server. - [substituter]{#gloss-substituter}\ - A *substituter* is a store other than `/nix/store` from which Nix will + A *substituter* is an additional store from which Nix will copy a store path instead of building it. Nix will not copy a store path from a remote store unless one of the following is true: From 9cb84121435e8ca6a51950b9d96a3d3be47c809e Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+amjoseph-nixpkgs@users.noreply.github.com> Date: Thu, 1 Sep 2022 20:41:04 +0000 Subject: [PATCH 255/371] Update doc/manual/src/glossary.md Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index f072f35e1..91865c807 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -53,7 +53,7 @@ - [substituter]{#gloss-substituter}\ A *substituter* is an additional store from which Nix will - copy a store path instead of building it. Nix will not copy a store + copy store objects it doesn't have. Nix will not copy a store path from a remote store unless one of the following is true: - the store object is signed by one of the `trusted-public-keys` From 41153f30bd5ca1bd9fa10d18da7a6b5b78a94087 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 13:54:09 -0700 Subject: [PATCH 256/371] glossary: substituter: merge output-addressed cases --- doc/manual/src/glossary.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 91865c807..608beb8f8 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -59,8 +59,9 @@ - the store object is signed by one of the `trusted-public-keys` - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking - - the store object is a derivation - - the store object is the realisation of a fixed-output derivation + - the store object is *output-addressed*; this includes + derivations, the outputs of content-addressed derivations, and + the outputs of fixed-output derivations. - [purity]{#gloss-purity}\ The assumption that equal Nix derivations when run always produce From 1f56b5d77247d89a15a2c16ba2f5d1d672c835e8 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 14:09:06 -0700 Subject: [PATCH 257/371] doc/manual: un-inline definitions from `substitute` --- doc/manual/src/glossary.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 608beb8f8..8dff4646c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -7,6 +7,14 @@ translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). + - [content-addressed derivation]{#gloss-fixed-output-derivation} + FIXME + + - [fixed-output derivation]{#gloss-fixed-output-derivation} + A derivation which includes the `__outHash` attribute; the output + of such derivations must exactly match the hash. All fixed-output + derivations are [content-addressed derivations](#gloss-content-addressed-derivation). + - [store]{#gloss-store}\ The location in the file system where store objects live. Typically `/nix/store`. @@ -44,6 +52,16 @@ derivation outputs (objects produced by running a build action), or derivations (files describing a build action). + - [input-addressed store object]{#gloss-input-addressed-store-object}\ + Store objects produced by building a + non-[content-addressed](#gloss-content-addressed-derivation) + derivation. + + - [output-addressed store object]{#gloss-output-addressed-store-object}\ + A store object whose store path hashes its content. This + includes derivations and the outputs of + [content-addressed derivations](#gloss-content-addressed-derivation) + - [substitute]{#gloss-substitute}\ A substitute is a command invocation stored in the Nix database that describes how to build a store object, bypassing the normal build @@ -59,9 +77,7 @@ - the store object is signed by one of the `trusted-public-keys` - the substituter is in the `trusted-substituters` list - the `no-require-sigs` option has been set to disable signature checking - - the store object is *output-addressed*; this includes - derivations, the outputs of content-addressed derivations, and - the outputs of fixed-output derivations. + - the store object is [output-addressed](#gloss-output-addressed-store-object) - [purity]{#gloss-purity}\ The assumption that equal Nix derivations when run always produce From 0a98d564b3d8d195c023429e5f7faf63e20b5d93 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 14:41:27 -0700 Subject: [PATCH 258/371] glossary: resolve FIXME in #gloss-fixed-output-derivation --- doc/manual/src/glossary.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 8dff4646c..473aef03c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -7,13 +7,14 @@ translated into low-level *store derivations* (implicitly by `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). - - [content-addressed derivation]{#gloss-fixed-output-derivation} - FIXME + - [content-addressed derivation]{#gloss-content-addressed-derivation}\ + A derivation which has the + [`__contentAddressed`](language/advanced-attributes.md#contentAddressed) + attribute set to `true`. - - [fixed-output derivation]{#gloss-fixed-output-derivation} - A derivation which includes the `__outHash` attribute; the output - of such derivations must exactly match the hash. All fixed-output - derivations are [content-addressed derivations](#gloss-content-addressed-derivation). + - [fixed-output derivation]{#gloss-fixed-output-derivation}\ + A derivation which includes the `__outputHash` attribute; the output + of such derivations must exactly match the hash. - [store]{#gloss-store}\ The location in the file system where store objects live. Typically @@ -59,8 +60,10 @@ - [output-addressed store object]{#gloss-output-addressed-store-object}\ A store object whose store path hashes its content. This - includes derivations and the outputs of - [content-addressed derivations](#gloss-content-addressed-derivation) + includes derivations, the outputs of + [content-addressed derivations](#gloss-content-addressed-derivation), + and the outputs of + [fixed-output derivations](#gloss-fixed-output-derivation). - [substitute]{#gloss-substitute}\ A substitute is a command invocation stored in the Nix database that From 1b2b8c39fd64871b3df261c05ed001fcd1057a6c Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 14:47:17 -0700 Subject: [PATCH 259/371] fix link to language/advanced-attributes.md#adv-attr-contentAddressed --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 473aef03c..bf1d0cf05 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -9,7 +9,7 @@ - [content-addressed derivation]{#gloss-content-addressed-derivation}\ A derivation which has the - [`__contentAddressed`](language/advanced-attributes.md#contentAddressed) + [`__contentAddressed`](language/advanced-attributes.md#adv-attr-contentAddressed) attribute set to `true`. - [fixed-output derivation]{#gloss-fixed-output-derivation}\ From def4fb9a0f73046efbf9fdb4f1e35898fb27ca34 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 14:47:33 -0700 Subject: [PATCH 260/371] __outputHash: add link --- doc/manual/src/glossary.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index bf1d0cf05..d61cfc823 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -13,8 +13,8 @@ attribute set to `true`. - [fixed-output derivation]{#gloss-fixed-output-derivation}\ - A derivation which includes the `__outputHash` attribute; the output - of such derivations must exactly match the hash. + A derivation which includes the + [`__outputHash`](language/advanced-attributes.md#adv-attr-outputHash) attribute. - [store]{#gloss-store}\ The location in the file system where store objects live. Typically From 8139bbe2ba767458cba4158627ee3d58f4a35d7d Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 15:01:07 -0700 Subject: [PATCH 261/371] implement https://github.com/NixOS/nix/pull/6870#pullrequestreview-1093700220 --- doc/manual/src/glossary.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d61cfc823..b30633833 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -74,8 +74,8 @@ - [substituter]{#gloss-substituter}\ A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. Nix will not copy a store - path from a remote store unless one of the following is true: + copy store objects it doesn't have. Nix will copy a store + path from a remote store only if one of the following is true: - the store object is signed by one of the `trusted-public-keys` - the substituter is in the `trusted-substituters` list From 57f12df5e4cde436566d3c4f0226d329e6eedf1a Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 15:09:10 -0700 Subject: [PATCH 262/371] input-addressed store object: include FODOs --- doc/manual/src/glossary.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b30633833..89a9b94a1 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -54,8 +54,9 @@ derivations (files describing a build action). - [input-addressed store object]{#gloss-input-addressed-store-object}\ - Store objects produced by building a - non-[content-addressed](#gloss-content-addressed-derivation) + A store object produced by building a + non-[content-addressed](#gloss-content-addressed-derivation), + non-[fixed-output](#gloss-fixed-output-derivation), derivation. - [output-addressed store object]{#gloss-output-addressed-store-object}\ From d5e064d8162e377556dc9daba99868085561a080 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:46:31 -0700 Subject: [PATCH 263/371] glossary: fix broken link --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 89a9b94a1..1aebdaa67 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -9,7 +9,7 @@ - [content-addressed derivation]{#gloss-content-addressed-derivation}\ A derivation which has the - [`__contentAddressed`](language/advanced-attributes.md#adv-attr-contentAddressed) + [`__contentAddressed`](language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. - [fixed-output derivation]{#gloss-fixed-output-derivation}\ From 887e922be29d37d377ef766c7fe7a2103f43ca21 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:47:13 -0700 Subject: [PATCH 264/371] glossary: outputHash, not __outputHash --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 1aebdaa67..73e209103 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -14,7 +14,7 @@ - [fixed-output derivation]{#gloss-fixed-output-derivation}\ A derivation which includes the - [`__outputHash`](language/advanced-attributes.md#adv-attr-outputHash) attribute. + [`outputHash`](language/advanced-attributes.md#adv-attr-outputHash) attribute. - [store]{#gloss-store}\ The location in the file system where store objects live. Typically From f6c750e8b2f299e9876fd8f2578f9093682f6d7f Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:48:34 -0700 Subject: [PATCH 265/371] glossary: remove extraneous comma --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 73e209103..7ba595ba0 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -56,7 +56,7 @@ - [input-addressed store object]{#gloss-input-addressed-store-object}\ A store object produced by building a non-[content-addressed](#gloss-content-addressed-derivation), - non-[fixed-output](#gloss-fixed-output-derivation), + non-[fixed-output](#gloss-fixed-output-derivation) derivation. - [output-addressed store object]{#gloss-output-addressed-store-object}\ From 59dc8346ca53f49ccdbbd6709b12a479376d1464 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:51:56 -0700 Subject: [PATCH 266/371] move substituter signature-checking conditions to configuration file documentation --- doc/manual/src/glossary.md | 9 ++------- src/libstore/globals.hh | 8 ++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 7ba595ba0..6bf041e7c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -75,13 +75,8 @@ - [substituter]{#gloss-substituter}\ A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. Nix will copy a store - path from a remote store only if one of the following is true: - - - the store object is signed by one of the `trusted-public-keys` - - the substituter is in the `trusted-substituters` list - - the `no-require-sigs` option has been set to disable signature checking - - the store object is [output-addressed](#gloss-output-addressed-store-object) + copy store objects it doesn't have. For details, see the + [`substituters` option](command-ref/conf-file.html#conf-substituters). - [purity]{#gloss-purity}\ The assumption that equal Nix derivations when run always produce diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..a659036e2 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -613,6 +613,14 @@ public: are tried based on their Priority value, which each substituter can set independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. + + Nix will copy a store path from a remote store only if one + of the following is true: + + - the store object is signed by one of the `trusted-public-keys` + - the substituter is in the `trusted-substituters` list + - the `no-require-sigs` option has been set to disable signature checking + - the store object is [output-addressed](#gloss-output-addressed-store-object) )", {"binary-caches"}}; From 1ab913467ef8e9ff946e64bd31841775d743b2d6 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:54:23 -0700 Subject: [PATCH 267/371] linkify mention of other options --- src/libstore/globals.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a659036e2..a4db3bf08 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -617,10 +617,10 @@ public: Nix will copy a store path from a remote store only if one of the following is true: - - the store object is signed by one of the `trusted-public-keys` - - the substituter is in the `trusted-substituters` list - - the `no-require-sigs` option has been set to disable signature checking - - the store object is [output-addressed](#gloss-output-addressed-store-object) + - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) + - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) )", {"binary-caches"}}; From e6f5352e71a1811eb2eb3bfb989e109de590c7a7 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 18:27:00 -0700 Subject: [PATCH 268/371] #binary-cache -> #gloss-binary-cache --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 6bf041e7c..a34b8a60c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -34,7 +34,7 @@ directory on another machine, accessed via `ssh` or served by the `nix-serve` Perl script. - - [binary cache]{#binary-cache}\ + - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its metadata and signatures are kept in `.narinfo` files rather than in a Nix database. This different format simplifies serving store objects From e90f2fcfc71ca997f254c86f8ed12fc143374752 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 18:28:05 -0700 Subject: [PATCH 269/371] glossary: add entry for `chroot store` (used 11 times in nix) --- doc/manual/src/glossary.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a34b8a60c..70a0eb994 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -34,6 +34,9 @@ directory on another machine, accessed via `ssh` or served by the `nix-serve` Perl script. + - [chroot store]{#gloss-chroot-store}\ + A local store whose canonical path is anything other than `/nix/store`. + - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its metadata and signatures are kept in `.narinfo` files rather than in a From bd63ae7e18f649e22ace799e6d366efe4029dc80 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 1 Sep 2022 23:13:09 -0500 Subject: [PATCH 270/371] =?UTF-8?q?Don=E2=80=99t=20add=20a=20space=20after?= =?UTF-8?q?=20attrs=20completion=20in=20zsh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This matches the behavior of bash. We don’t want to add a space after completion on attrs. Uses -S. Switches to new compadd style comppletions instead of _describe. Shouldn’t have any negative issues from what I can tell. --- misc/zsh/completion.zsh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh index e702c721e..e86984168 100644 --- a/misc/zsh/completion.zsh +++ b/misc/zsh/completion.zsh @@ -10,14 +10,15 @@ function _nix() { local -a suggestions declare -a suggestions for suggestion in ${res:1}; do - # FIXME: This doesn't work properly if the suggestion word contains a `:` - # itself - suggestions+="${suggestion/ /:}" + suggestions+=("${suggestion%% *}") done + local -a args if [[ "$tpe" == filenames ]]; then - compadd -f + args+=('-f') + elif [[ "$tpe" == attrs ]]; then + args+=('-S' '') fi - _describe 'nix' suggestions + compadd -J nix "${args[@]}" -a suggestions } -_nix "$@" +# _nix "$@" From dd3cd1a16740cb20ba00a36f8b3cf8f813e7480f Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 2 Sep 2022 10:50:02 -0500 Subject: [PATCH 271/371] Apply suggestions from code review --- misc/zsh/completion.zsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh index e86984168..f9b3dca74 100644 --- a/misc/zsh/completion.zsh +++ b/misc/zsh/completion.zsh @@ -21,4 +21,4 @@ function _nix() { compadd -J nix "${args[@]}" -a suggestions } -# _nix "$@" +_nix "$@" From 4894e567fb27c02866abd75f59da740efa6c33a5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 2 Sep 2022 11:46:34 -0500 Subject: [PATCH 272/371] =?UTF-8?q?Don=E2=80=99t=20readDerivation=20if=20i?= =?UTF-8?q?mpure=20derivations=20feature=20is=20disabled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readDerivation is pretty slow, and while it may not be significant for some use cases, on things like ghc-nix where we have thousands of derivations is really slows things down. So, this just doesn’t do the impure derivation check if the impure derivation experimental feature is disabled. Perhaps we could cache the result of isPure() and keep the check, but this is a quick fix to for the slowdown introduced with impure derivations features in 2.8.0. --- src/libstore/build/derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83da657f0..41d2e2a1c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -344,7 +344,7 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast(drv.get())->inputDrvs) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (drv->type().isPure() && !drv->type().isFixed()) { + if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(i.first); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", From 1f041ac54f43093e4f4df1caa630d491ff51c3f8 Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Fri, 2 Sep 2022 18:32:35 -0500 Subject: [PATCH 273/371] Prevent tempdir from being GC-ed before addToStoreFromDump has renamed it This fixes issue 6823 by placing the tempdir used in LocalStore::addToStoreFromDump outside the Nix store, where automatic GC is no longer a concern. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a272e4301..6abd52683 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1388,7 +1388,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; - auto tempDir = createTempDir(realStoreDir, "add"); + auto tempDir = createTempDir("", "add"); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; From 102434e4cbb8ed1b4075f99fb999f092fa068d5b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 3 Sep 2022 00:27:16 -0500 Subject: [PATCH 274/371] Disable SA_RESTART for some signals on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disables the SA_RESTART behavior on macOS which causes: > Restarting of pending calls is requested by setting the SA_RESTART bit > in sa_flags. The affected system calls include read(2), write(2), > sendto(2), recvfrom(2), sendmsg(2) and recvmsg(2) on a communications > channel or a slow device (such as a terminal, but not a regular file) > and during a wait(2) or ioctl(2). From: https://man.openbsd.org/sigaction#SA_RESTART This being set on macOS caused a bug where read() calls to the daemon socket were blocking after a SIGINT was received. As a result, checkInterrupt was never reached even though the signal was received by the signal handler thread. On Linux, SA_RESTART is disabled by default. This probably effects other BSDs but I don’t have the ability to test it there right now. --- src/libmain/shared.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 31454e49d..0ee3fe772 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -194,9 +194,16 @@ void initNix() /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. * Instead, add a dummy sigaction handler, and signalHandlerThread * can handle the rest. */ - struct sigaction sa; - sa.sa_handler = sigHandler; - if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); + act.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); + + // Disable SA_RESTART for interrupts, so that system calls on this thread + // error with EINTR like they do on Linux, and we don’t hang forever. + act.sa_handler = SIG_DFL; + if (sigaction(SIGINT, &act, 0)) throw SysError("handling SIGINT"); + if (sigaction(SIGTERM, &act, 0)) throw SysError("handling SIGTERM"); + if (sigaction(SIGHUP, &act, 0)) throw SysError("handling SIGHUP"); + if (sigaction(SIGPIPE, &act, 0)) throw SysError("handling SIGPIPE"); #endif /* Register a SIGSEGV handler to detect stack overflows. */ From a47b5476e1efb5c571f5a032a0b63ac8a2dfae82 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 3 Sep 2022 16:06:33 -0500 Subject: [PATCH 275/371] Add more signals --- src/libmain/shared.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0ee3fe772..9769c993e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -181,8 +181,9 @@ void initNix() /* Reset SIGCHLD to its default. */ struct sigaction act; sigemptyset(&act.sa_mask); - act.sa_handler = SIG_DFL; act.sa_flags = 0; + + act.sa_handler = SIG_DFL; if (sigaction(SIGCHLD, &act, 0)) throw SysError("resetting SIGCHLD"); @@ -197,13 +198,23 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); - // Disable SA_RESTART for interrupts, so that system calls on this thread - // error with EINTR like they do on Linux, and we don’t hang forever. + /* Disable SA_RESTART for interrupts, so that system calls on this thread + * error with EINTR like they do on Linux. + * Most signals on BSD systems default to SA_RESTART on, but Nix + * expects EINTR from syscalls to properly exit. */ act.sa_handler = SIG_DFL; if (sigaction(SIGINT, &act, 0)) throw SysError("handling SIGINT"); if (sigaction(SIGTERM, &act, 0)) throw SysError("handling SIGTERM"); if (sigaction(SIGHUP, &act, 0)) throw SysError("handling SIGHUP"); if (sigaction(SIGPIPE, &act, 0)) throw SysError("handling SIGPIPE"); + if (sigaction(SIGQUIT, &act, 0)) throw SysError("handling SIGQUIT"); + if (sigaction(SIGILL, &act, 0)) throw SysError("handling SIGILL"); + if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); + if (sigaction(SIGABRT, &act, 0)) throw SysError("handling SIGABRT"); + if (sigaction(SIGFPE, &act, 0)) throw SysError("handling SIGFPE"); + if (sigaction(SIGBUS, &act, 0)) throw SysError("handling SIGBUS"); + if (sigaction(SIGXCPU, &act, 0)) throw SysError("handling SIGXCPU"); + if (sigaction(SIGXFSZ, &act, 0)) throw SysError("handling SIGXFSZ"); #endif /* Register a SIGSEGV handler to detect stack overflows. */ From 3fca5f6c693179a2b49add54166b1a126c755264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 5 Sep 2022 14:44:01 +0200 Subject: [PATCH 276/371] Installer: Reset the timestamps in the tarball Otherwise it isn't reproducible. Fix https://github.com/NixOS/nix/issues/7001 --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 1b26460e7..cdb81179a 100644 --- a/flake.nix +++ b/flake.nix @@ -260,6 +260,7 @@ echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ From 77d3d3d18d181f4abe63d48361a96f4aac30668d Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 6 Sep 2022 09:47:53 -0500 Subject: [PATCH 277/371] Remove some signals --- src/libmain/shared.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 9769c993e..52b75f757 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -208,13 +208,7 @@ void initNix() if (sigaction(SIGHUP, &act, 0)) throw SysError("handling SIGHUP"); if (sigaction(SIGPIPE, &act, 0)) throw SysError("handling SIGPIPE"); if (sigaction(SIGQUIT, &act, 0)) throw SysError("handling SIGQUIT"); - if (sigaction(SIGILL, &act, 0)) throw SysError("handling SIGILL"); if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); - if (sigaction(SIGABRT, &act, 0)) throw SysError("handling SIGABRT"); - if (sigaction(SIGFPE, &act, 0)) throw SysError("handling SIGFPE"); - if (sigaction(SIGBUS, &act, 0)) throw SysError("handling SIGBUS"); - if (sigaction(SIGXCPU, &act, 0)) throw SysError("handling SIGXCPU"); - if (sigaction(SIGXFSZ, &act, 0)) throw SysError("handling SIGXFSZ"); #endif /* Register a SIGSEGV handler to detect stack overflows. */ From 6ce2e96c88c71e40303980eb5793aa6ae4a5a333 Mon Sep 17 00:00:00 2001 From: Jonathan Ringer Date: Tue, 6 Sep 2022 08:18:13 -0700 Subject: [PATCH 278/371] Docs: Add nix develop --command entry Add example of nix develop being used to execuate a series of script commands. This is common when doing things like CI/CD, and should be represented in the official documentation. Also useful for people looking for the 'nix develop' equivalent of 'nix-shell --run'. Related: - https://github.com/NixOS/nix/issues/6908 - https://github.com/NixOS/nix/issues/6908#issuecomment-1229266853 --- src/nix/develop.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/develop.md b/src/nix/develop.md index e036ec6b9..4e8542d1b 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -66,6 +66,12 @@ R""( `nixpkgs#glibc` in `~/my-glibc` and want to compile another package against it. +* Run a series of script commands: + + ```console + # nix develop --command bash -c "mkdir build && cmake .. && make" + ``` + # Description `nix develop` starts a `bash` shell that provides an interactive build From 27be54ca533933db8c3e0cde4b213abf10dd5237 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Sep 2022 18:27:39 +0200 Subject: [PATCH 279/371] nix develop: Ignore stdenv's $SHELL Stdenv sets this to a bash that doesn't have readline/completion support, so running 'nix (develop|shell)' inside a 'nix develop' gives you a crippled shell. So let's just ignore the derivation's $SHELL. This could break interactive use of build phases that use $SHELL, but they appear to be fairly rare. --- src/nix/develop.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ba7ba7c25..4de109754 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -246,6 +246,7 @@ struct Common : InstallableCommand, MixProfile "NIX_LOG_FD", "NIX_REMOTE", "PPID", + "SHELL", "SHELLOPTS", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", From ece12a97d9c7e0024ebddb9e5eb0c919a9efb694 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Sep 2022 19:20:31 +0200 Subject: [PATCH 280/371] lockfile -> lock file for consistency --- src/libexpr/flake/flake.cc | 4 ++-- src/libexpr/flake/lockfile.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 105e76bc6..119c556ac 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -483,12 +483,12 @@ LockedFlake lockFlake( } else if (auto follows = std::get_if<1>(&i.second)) { if (! trustLock) { // It is possible that the flake has changed, - // so we must confirm all the follows that are in the lockfile are also in the flake. + // so we must confirm all the follows that are in the lock file are also in the flake. auto overridePath(inputPath); overridePath.push_back(i.first); auto o = overrides.find(overridePath); // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lockfile. + // since some of the inputs may not be present in the lock file. if (o == overrides.end()) { mustRefetch = true; // There's no point populating the rest of the fake inputs, diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 60b52d578..629d2e669 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -36,7 +36,7 @@ LockedNode::LockedNode(const nlohmann::json & json) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) - throw Error("lockfile contains mutable lock '%s'", + throw Error("lock file contains mutable lock '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } From 84fe75a12a085c6b4b8d4ac65a048f569de1252b Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Tue, 6 Sep 2022 17:48:00 -0500 Subject: [PATCH 281/371] Keep created temp dirs inside store, but protect from GC Implements the approach suggested by feedback on PR #6994, where tempdir paths are created in the store (now with an exclusive lock). As part of this work, the currently-broken and unused `createTempDirInStore` function is updated to create an exclusive lock on the temp directory in the store. The GC now makes a non-blocking attempt to lock any store directories that "look like" the temp directories created by this function, and if it can't acquire one, ignores the directory. --- src/libstore/gc.cc | 12 ++++++++++++ src/libstore/local-store.cc | 29 +++++++++++++++++++---------- src/libstore/local-store.hh | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 4c1a82279..6cd7efbc9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -619,6 +619,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Path path = storeDir + "/" + std::string(baseName); Path realPath = realStoreDir + "/" + std::string(baseName); + /* There may be temp directories in the store that are still in use + by another process. We need to be sure that we can acquire an + exclusive lock before deleting them. */ + AutoCloseFD tmpDirFd; + if (baseName.rfind("add-", 0) == 0) { + tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { + debug("skipping locked tempdir '%s'", realPath); + return; + } + } + printInfo("deleting '%1%'", path); results.paths.insert(path); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6abd52683..5ee451da3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1382,13 +1382,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name std::unique_ptr delTempDir; Path tempPath; + Path tempDir; + AutoCloseFD tempDirFd; if (!inMemory) { /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; - auto tempDir = createTempDir("", "add"); + std::tie(tempDir, tempDirFd) = createTempDirInStore(); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; @@ -1431,6 +1433,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); + tempDirFd.close(); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1507,18 +1510,24 @@ StorePath LocalStore::addTextToStore( /* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() + garbage-collected until the returned FD is closed. */ +std::pair LocalStore::createTempDirInStore() { - Path tmpDir; + Path tmpDirFn; + AutoCloseFD tmpDirFd; + bool lockedByUs = false; do { /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(parseStorePath(tmpDir)); - } while (!pathExists(tmpDir)); - return tmpDir; + the GC between createTempDir() and when we acquire a lock on it. + We'll repeat until 'tmpDir' exists and we've locked it. */ + tmpDirFn = createTempDir(realStoreDir, "add"); + tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() < 0) { + continue; + } + lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); + } while (!pathExists(tmpDirFn) || !lockedByUs); + return {tmpDirFn, std::move(tmpDirFd)}; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 70d225be3..bd0ce1fe6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -256,7 +256,7 @@ private: void findRuntimeRoots(Roots & roots, bool censor); - Path createTempDirInStore(); + std::pair createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); From 7852609999cdcbc056ef47b530dcb253bd2c0697 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 12 Sep 2022 11:27:25 +0200 Subject: [PATCH 282/371] issue template: add feature label each change should be an improvement, a label for that is redundant. --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 392ed30c6..4fe86d5ec 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: improvement +labels: feature assignees: '' --- From 565d888e0f6a2c66ee7b10f6fe6a97f79fa51732 Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Mon, 12 Sep 2022 11:33:23 -0500 Subject: [PATCH 283/371] Address PR feedback on #6694 --- src/libstore/gc.cc | 5 ++--- src/libstore/local-store.cc | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6cd7efbc9..9ef8972f3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -622,9 +622,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ - AutoCloseFD tmpDirFd; - if (baseName.rfind("add-", 0) == 0) { - tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); + if (baseName.find("tmp-", 0) == 0) { + AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir '%s'", realPath); return; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5ee451da3..0b07cde34 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1433,7 +1433,6 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); - tempDirFd.close(); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1520,7 +1519,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "add"); + tmpDirFn = createTempDir(realStoreDir, "tmp"); tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); if (tmpDirFd.get() < 0) { continue; From c6ff33ff5c83a546fc6e82055aa04abfe41011dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 13 Sep 2022 15:29:13 +0200 Subject: [PATCH 284/371] RunPager: Stop the progress bar In particular, the progress bar was interfering with 'less' rendering in '--help' (e.g. run 'nix --help' and hit '/' to search). --- src/libmain/shared.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 52b75f757..c1cf38565 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -4,6 +4,7 @@ #include "gc-store.hh" #include "util.hh" #include "loggers.hh" +#include "progress-bar.hh" #include #include @@ -422,6 +423,8 @@ RunPager::RunPager() if (!pager) pager = getenv("PAGER"); if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return; + stopProgressBar(); + Pipe toPager; toPager.create(); From d365cced4fadbbc63f0c39902a7091e1a34c34de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 13 Sep 2022 16:58:32 +0200 Subject: [PATCH 285/371] Trim option descriptions This removes unintended blank lines in Markdown when the description is a multiline string literal. --- src/libutil/args.cc | 6 +++--- src/nix/main.cc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 44b63f0f6..753980fd4 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -216,7 +216,7 @@ nlohmann::json Args::toJSON() if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") - j["description"] = flag->description; + j["description"] = trim(flag->description); j["category"] = flag->category; if (flag->handler.arity != ArityAny) j["arity"] = flag->handler.arity; @@ -237,7 +237,7 @@ nlohmann::json Args::toJSON() } auto res = nlohmann::json::object(); - res["description"] = description(); + res["description"] = trim(description()); res["flags"] = std::move(flags); res["args"] = std::move(args); auto s = doc(); @@ -379,7 +379,7 @@ nlohmann::json MultiCommand::toJSON() auto j = command->toJSON(); auto cat = nlohmann::json::object(); cat["id"] = command->category(); - cat["description"] = categories[command->category()]; + cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); cmds[name] = std::move(j); } diff --git a/src/nix/main.cc b/src/nix/main.cc index f434e9655..e0155cd5d 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -325,7 +325,7 @@ void mainWrapped(int argc, char * * argv) std::cout << "attrs\n"; break; } for (auto & s : *completions) - std::cout << s.completion << "\t" << s.description << "\n"; + std::cout << s.completion << "\t" << trim(s.description) << "\n"; } }); From 8ebdbeb2574ab3a8b6dbd9826451d9f26ca5ad3e Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Wed, 7 Sep 2022 11:58:25 -0700 Subject: [PATCH 286/371] Add fish suport to installer Before this patch, installing Nix using the Fish shell did not work because Fish wasn't configured to add Nix to the PATH. Some options in #1512 offered workarounds, but they typically involve extra plugins or packages. This patch adds native, out-of-the-box support for the Fish shell. Note that Fish supports a `conf.d` directory, which is intended for exactly use cases like this: software projects distributing shell snippets. This patch takes advantage of it. The installer doesn't append any Nix loader behavior to any Fish config file. Because of that, the uninstall process is smooth and a reinstall obliterates the existing nix.fish files that we place instead of bothering the user with a backup / manual removal. Both single-user and multi-user cases are covered. It has been tested on Ubuntu, and a Mac with MacPorts, homebrew, and the Fish installer pkg. Closes #1512 Co-authored-by: Graham Christensen --- .gitignore | 2 ++ scripts/install-multi-user.sh | 47 +++++++++++++++++++++++++++++ scripts/install-nix-from-closure.sh | 29 +++++++++++++++--- scripts/local.mk | 2 ++ scripts/nix-profile-daemon.fish.in | 35 +++++++++++++++++++++ scripts/nix-profile.fish.in | 35 +++++++++++++++++++++ scripts/nix-profile.sh.in | 1 - 7 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 scripts/nix-profile-daemon.fish.in create mode 100644 scripts/nix-profile.fish.in diff --git a/.gitignore b/.gitignore index 0c1b89ace..8e0db013f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh /scripts/nix-profile-daemon.sh +/scripts/nix-profile.fish +/scripts/nix-profile-daemon.fish # /src/libexpr/ /src/libexpr/lexer-tab.cc diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9a990275c..a39339050 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -37,6 +37,19 @@ readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/e readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" +# Fish has different syntax than zsh/bash, treat it separate +readonly PROFILE_FISH_SUFFIX="conf.d/nix.fish" +readonly PROFILE_FISH_PREFIXES=( + # each of these are common values of $__fish_sysconf_dir, + # under which Fish will look for a file named + # $PROFILE_FISH_SUFFIX. + "/etc/fish" # standard + "/usr/local/etc/fish" # their installer .pkg for macOS + "/opt/homebrew/etc/fish" # homebrew + "/opt/local/etc/fish" # macports +) +readonly PROFILE_NIX_FILE_FISH="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.fish" + readonly NIX_INSTALLED_NIX="@nix@" readonly NIX_INSTALLED_CACERT="@cacert@" #readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" @@ -828,6 +841,19 @@ fi EOF } +# Fish has differing syntax +fish_source_lines() { + cat <&2 - printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p_sh" "$p_sh" >> "$fn" fi added=1 + p=${p_sh} break fi done for i in .zshenv .zshrc; do fn="$HOME/$i" if [ -w "$fn" ]; then - if ! grep -q "$p" "$fn"; then + if ! grep -q "$p_sh" "$fn"; then echo "modifying $fn..." >&2 - printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p_sh" "$p_sh" >> "$fn" fi added=1 + p=${p_sh} break fi done + + if [ -d "$HOME/.config/fish" ]; then + fishdir=$HOME/.config/fish/conf.d + if [ ! -d "$fishdir" ]; then + mkdir -p "$fishdir" + fi + + fn="$fishdir/nix.fish" + echo "placing $fn..." >&2 + printf '\nif test -e %s; . %s; end # added by Nix installer\n' "$p_fish" "$p_fish" > "$fn" + added=1 + p=${p_fish} + fi +else + p=${p_sh} fi if [ -z "$added" ]; then diff --git a/scripts/local.mk b/scripts/local.mk index b8477178e..46255e432 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -6,6 +6,8 @@ noinst-scripts += $(nix_noinst_scripts) profiledir = $(sysconfdir)/profile.d $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile.fish, $(profiledir)/nix.fish, 0644)) $(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile-daemon.fish, $(profiledir)/nix-daemon.fish, 0644)) clean-files += $(nix_noinst_scripts) diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in new file mode 100644 index 000000000..56d851a9c --- /dev/null +++ b/scripts/nix-profile-daemon.fish.in @@ -0,0 +1,35 @@ +# Only execute this file once per shell. +if test -n "$__ETC_PROFILE_NIX_SOURCED" + return +end + +set __ETC_PROFILE_NIX_SOURCED 1 + +set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if test -n "$NIX_SSH_CERT_FILE" + : # Allow users to override the NIX_SSL_CERT_FILE +else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt +else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed + set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem +else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt +else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS + set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt +else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" +else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES + if test -e "$i/etc/ssl/certs/ca-bundle.crt" + set --export NIX_SSL_CERT_FILE "$i/etc/ssl/certs/ca-bundle.crt" + end + end +end + +fish_add_path --prepend --global "@localstatedir@/nix/profiles/default/bin" +fish_add_path --prepend --global "$HOME/.nix-profile/bin" diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in new file mode 100644 index 000000000..59d247771 --- /dev/null +++ b/scripts/nix-profile.fish.in @@ -0,0 +1,35 @@ +if test -n "$HOME" && test -n "$USER" + + # Set up the per-user profile. + + set NIX_LINK $HOME/.nix-profile + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. + if test -n "$NIX_SSH_CERT_FILE" + : # Allow users to override the NIX_SSL_CERT_FILE + else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt + else if test -e /etc/ssl/ca-bundle.pem # openSUSE Tumbleweed + set --export NIX_SSL_CERT_FILE /etc/ssl/ca-bundle.pem + else if test -e /etc/ssl/certs/ca-bundle.crt # Old NixOS + set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt + else if test -e /etc/pki/tls/certs/ca-bundle.crt # Fedora, CentOS + set --export NIX_SSL_CERT_FILE /etc/pki/tls/certs/ca-bundle.crt + else if test -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" # fall back to cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" + else if test -e "$NIX_LINK/etc/ca-bundle.crt" # old cacert in Nix profile + set --export NIX_SSL_CERT_FILE "$NIX_LINK/etc/ca-bundle.crt" + end + + # Only use MANPATH if it is already set. In general `man` will just simply + # pick up `.nix-profile/share/man` because is it close to `.nix-profile/bin` + # which is in the $PATH. For more info, run `manpath -d`. + set --export --prepend --path MANPATH "$NIX_LINK/share/man" + + fish_add_path --prepend --global "$NIX_LINK/bin" + set --erase NIX_LINK +end diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 45cbcbe74..5636085d4 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,7 +1,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up the per-user profile. - # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix NIX_LINK=$HOME/.nix-profile From 7194c87dce39d89868b3bc25790fefb56f7fefae Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 12 Sep 2022 09:46:06 -0700 Subject: [PATCH 287/371] Add installer_test matrix for shells Signed-off-by: Ana Hobden --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86b5dfd2e..628d1d192 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,14 @@ jobs: with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - - run: nix-instantiate -E 'builtins.currentTime' --eval + - run: sudo apt install fish zsh + if: matrix.os == 'ubuntu-latest' + - run: brew install fish + if: matrix.os == 'macos-latest' + - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" + - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" docker_push_image: needs: [check_secrets, tests] From fae3b4fe8abc2b307a583e396a24d7899bb21451 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Sep 2022 15:40:43 +0200 Subject: [PATCH 288/371] Add an installer test This runs the installer in a QEMU VM. Unlike the old installer test that ran inside a declaratively built RedHat/Debian image, this uses an image from Vagrant. --- flake.nix | 5 + tests/installer/default.nix | 136 +++++++++++++++++++++++++++ tests/installer/vagrant_insecure_key | 27 ++++++ 3 files changed, 168 insertions(+) create mode 100644 tests/installer/default.nix create mode 100644 tests/installer/vagrant_insecure_key diff --git a/flake.nix b/flake.nix index cdb81179a..ec64719d1 100644 --- a/flake.nix +++ b/flake.nix @@ -546,6 +546,11 @@ # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; } "touch $out"); + installerTests = import ./tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; + }; checks = forAllSystems (system: { diff --git a/tests/installer/default.nix b/tests/installer/default.nix new file mode 100644 index 000000000..39911aeb2 --- /dev/null +++ b/tests/installer/default.nix @@ -0,0 +1,136 @@ +{ binaryTarballs +, nixpkgsFor +}: + +let + + installScripts = { + install-default = { + script = '' + set -eux + + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --no-channel-add + ''; + }; + + install-force-no-daemon = { + script = '' + set -eux + + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --no-daemon + ''; + }; + + install-force-daemon = { + script = '' + set -eux + + tar -xf ./nix.tar.xz + mv ./nix-* nix + ./nix/install --daemon + ''; + }; + }; + + images = { + + "ubuntu-14-04" = { + image = import { + url = https://app.vagrantup.com/ubuntu/boxes/trusty64/versions/20190514.0.0/providers/virtualbox.box; + hash = "sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="; + }; + rootDisk = "box-disk1.vmdk"; + system = "x86_64-linux"; + }; + + "ubuntu-16-04" = { + image = import { + url = https://app.vagrantup.com/ubuntu/boxes/xenial64/versions/20211001.0.0/providers/virtualbox.box; + hash = "sha256-JCc0wd9vaSzCU8coByVtb/oDTAXYBPnORwEShS4oj4U="; + }; + rootDisk = "ubuntu-xenial-16.04-cloudimg.vmdk"; + system = "x86_64-linux"; + }; + + "ubuntu-22-10" = { + image = import { + url = https://app.vagrantup.com/ubuntu/boxes/kinetic64/versions/20220910.0.0/providers/virtualbox.box; + hash = "sha256-/IXr+Apyx2dqX6Gj4SoNtQ/5v1eKKopwzFgozAq6GFY="; + }; + rootDisk = "ubuntu-kinetic-22.10-cloudimg.vmdk"; + system = "x86_64-linux"; + }; + + }; + + makeTest = imageName: testName: + let image = images.${imageName}; in + with nixpkgsFor.${image.system}; + runCommand + "installer-test-${imageName}-${testName}" + { buildInputs = [ qemu_kvm openssh ]; + image = image.image; + installScript = installScripts.${testName}.script; + binaryTarball = binaryTarballs.${system}; + } + '' + echo "Unpacking Vagrant box..." + tar xvf $image + + qemu-img create -b ./${image.rootDisk} -F vmdk -f qcow2 ./disk.qcow2 + + echo "Starting qemu..." + qemu-kvm -m 4096 -nographic \ + -drive id=disk1,file=./disk.qcow2,if=virtio \ + -netdev user,id=net0,restrict=yes,hostfwd=tcp::20022-:22 -device virtio-net-pci,netdev=net0 & + qemu_pid=$! + trap "kill $qemu_pid" EXIT + + if ! [ -e ./vagrant_insecure_key ]; then + cp ${./vagrant_insecure_key} vagrant_insecure_key + fi + + chmod 0400 ./vagrant_insecure_key + + ssh_opts="-o StrictHostKeyChecking=no -o PubkeyAcceptedKeyTypes=+ssh-rsa -i ./vagrant_insecure_key" + ssh="ssh -p 20022 -q $ssh_opts vagrant@localhost" + + echo "Waiting for SSH..." + for ((i = 0; i < 120; i++)); do + echo "[ssh] Trying to connect..." + if $ssh -- true; then + echo "[ssh] Connected!" + break + fi + if ! kill -0 $qemu_pid; then + echo "qemu died unexpectedly" + exit 1 + fi + sleep 1 + done + + echo "Copying installer..." + scp -P 20022 $ssh_opts $binaryTarball/nix-*.tar.xz vagrant@localhost:nix.tar.xz + + echo "Running installer..." + $ssh "$installScript" + + echo "Testing Nix installation..." + # FIXME: should update ~/.bashrc. + $ssh "source ~/.profile; nix-env --version" + + echo "Done!" + touch $out + ''; + +in + +{ + ubuntu-14-04.install-default = makeTest "ubuntu-14-04" "install-default"; + #ubuntu-16-04.install-default = makeTest "ubuntu-16-04" "install-default"; + #ubuntu-22-10.install-default = makeTest "ubuntu-22-10" "install-default"; +} diff --git a/tests/installer/vagrant_insecure_key b/tests/installer/vagrant_insecure_key new file mode 100644 index 000000000..7d6a08390 --- /dev/null +++ b/tests/installer/vagrant_insecure_key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- From 0a8e666dd6d18ede4b5cd648e19d5950ee19f095 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Sep 2022 18:40:16 +0200 Subject: [PATCH 289/371] Add Fedora 36 --- tests/installer/default.nix | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 39911aeb2..ab3ef62f0 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -65,6 +65,16 @@ let system = "x86_64-linux"; }; + + "fedora-36" = { + image = import { + url = https://app.vagrantup.com/generic/boxes/fedora36/versions/4.1.12/providers/libvirt.box; + hash = "sha256-rxPgnDnFkTDwvdqn2CV3ZUo3re9AdPtSZ9SvOHNvaks="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + }; makeTest = imageName: testName: @@ -78,10 +88,12 @@ let binaryTarball = binaryTarballs.${system}; } '' - echo "Unpacking Vagrant box..." + echo "Unpacking Vagrant box $image..." tar xvf $image - qemu-img create -b ./${image.rootDisk} -F vmdk -f qcow2 ./disk.qcow2 + image_type=$(qemu-img info ${image.rootDisk} | sed 's/file format: \(.*\)/\1/; t; d') + + qemu-img create -b ./${image.rootDisk} -F "$image_type" -f qcow2 ./disk.qcow2 echo "Starting qemu..." qemu-kvm -m 4096 -nographic \ @@ -121,7 +133,7 @@ let echo "Testing Nix installation..." # FIXME: should update ~/.bashrc. - $ssh "source ~/.profile; nix-env --version" + $ssh "source ~/.bash_profile || source ~/.bash_login || source ~/.profile || true; nix-env --version" echo "Done!" touch $out @@ -133,4 +145,5 @@ in ubuntu-14-04.install-default = makeTest "ubuntu-14-04" "install-default"; #ubuntu-16-04.install-default = makeTest "ubuntu-16-04" "install-default"; #ubuntu-22-10.install-default = makeTest "ubuntu-22-10" "install-default"; + fedora-36.install-default = makeTest "fedora-36" "install-default"; } From 906c947ee8f2478d27e5eda649f44716e952d8a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Sep 2022 18:53:30 +0200 Subject: [PATCH 290/371] Enable daemon installation test on Fedora --- tests/installer/default.nix | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index ab3ef62f0..8b6bde73f 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -7,8 +7,6 @@ let installScripts = { install-default = { script = '' - set -eux - tar -xf ./nix.tar.xz mv ./nix-* nix ./nix/install --no-channel-add @@ -17,8 +15,6 @@ let install-force-no-daemon = { script = '' - set -eux - tar -xf ./nix.tar.xz mv ./nix-* nix ./nix/install --no-daemon @@ -27,15 +23,15 @@ let install-force-daemon = { script = '' - set -eux - tar -xf ./nix.tar.xz mv ./nix-* nix - ./nix/install --daemon + ./nix/install --daemon --no-channel-add ''; }; }; + disableSELinux = "sudo setenforce 0"; + images = { "ubuntu-14-04" = { @@ -65,7 +61,6 @@ let system = "x86_64-linux"; }; - "fedora-36" = { image = import { url = https://app.vagrantup.com/generic/boxes/fedora36/versions/4.1.12/providers/libvirt.box; @@ -73,6 +68,7 @@ let }; rootDisk = "box.img"; system = "x86_64-linux"; + postBoot = disableSELinux; }; }; @@ -84,6 +80,7 @@ let "installer-test-${imageName}-${testName}" { buildInputs = [ qemu_kvm openssh ]; image = image.image; + postBoot = image.postBoot or ""; installScript = installScripts.${testName}.script; binaryTarball = binaryTarballs.${system}; } @@ -125,15 +122,25 @@ let sleep 1 done + if [[ -n $postBoot ]]; then + echo "Running post-boot commands..." + $ssh "set -ex; $postBoot" + fi + echo "Copying installer..." scp -P 20022 $ssh_opts $binaryTarball/nix-*.tar.xz vagrant@localhost:nix.tar.xz echo "Running installer..." - $ssh "$installScript" + $ssh "set -eux; $installScript" echo "Testing Nix installation..." # FIXME: should update ~/.bashrc. - $ssh "source ~/.bash_profile || source ~/.bash_login || source ~/.profile || true; nix-env --version" + $ssh " + set -ex + source ~/.bash_profile || source ~/.bash_login || source ~/.profile || true + nix-env --version + nix --extra-experimental-features nix-command store ping + " echo "Done!" touch $out @@ -146,4 +153,5 @@ in #ubuntu-16-04.install-default = makeTest "ubuntu-16-04" "install-default"; #ubuntu-22-10.install-default = makeTest "ubuntu-22-10" "install-default"; fedora-36.install-default = makeTest "fedora-36" "install-default"; + fedora-36.install-force-daemon = makeTest "fedora-36" "install-force-daemon"; } From cc6e31231547fc64c89c6682316f2bab03db6879 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Sep 2022 19:44:41 +0200 Subject: [PATCH 291/371] Get Ubuntu 22.10 to work --- tests/installer/default.nix | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 8b6bde73f..9b2a34bdb 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -85,6 +85,8 @@ let binaryTarball = binaryTarballs.${system}; } '' + shopt -s nullglob + echo "Unpacking Vagrant box $image..." tar xvf $image @@ -92,10 +94,19 @@ let qemu-img create -b ./${image.rootDisk} -F "$image_type" -f qcow2 ./disk.qcow2 + extra_qemu_opts= + + # Add the config disk, required by the Ubuntu images. + config_drive=$(echo *configdrive.vmdk || true) + if [[ -n $config_drive ]]; then + extra_qemu_opts+=" -drive id=disk2,file=$config_drive,if=virtio" + fi + echo "Starting qemu..." qemu-kvm -m 4096 -nographic \ -drive id=disk1,file=./disk.qcow2,if=virtio \ - -netdev user,id=net0,restrict=yes,hostfwd=tcp::20022-:22 -device virtio-net-pci,netdev=net0 & + -netdev user,id=net0,restrict=yes,hostfwd=tcp::20022-:22 -device virtio-net-pci,netdev=net0 \ + $extra_qemu_opts & qemu_pid=$! trap "kill $qemu_pid" EXIT @@ -137,7 +148,13 @@ let # FIXME: should update ~/.bashrc. $ssh " set -ex - source ~/.bash_profile || source ~/.bash_login || source ~/.profile || true + + # FIXME: get rid of this; ideally ssh should just work. + source ~/.bash_profile || true + source ~/.bash_login || true + source ~/.profile || true + source /etc/bashrc || true + nix-env --version nix --extra-experimental-features nix-command store ping " @@ -151,7 +168,8 @@ in { ubuntu-14-04.install-default = makeTest "ubuntu-14-04" "install-default"; #ubuntu-16-04.install-default = makeTest "ubuntu-16-04" "install-default"; - #ubuntu-22-10.install-default = makeTest "ubuntu-22-10" "install-default"; + ubuntu-22-10.install-default = makeTest "ubuntu-22-10" "install-default"; + ubuntu-22-10.install-force-daemon = makeTest "ubuntu-22-10" "install-force-daemon"; fedora-36.install-default = makeTest "fedora-36" "install-default"; fedora-36.install-force-daemon = makeTest "fedora-36" "install-force-daemon"; } From 02af02854d41b390957300bac778139bc1c6b5c2 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 14 Sep 2022 15:35:56 -0600 Subject: [PATCH 292/371] dockerImage: fix root shell Currently root's shell is set to a path that does not exist; this change sets it to the correct path to bash --- docker.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker.nix b/docker.nix index e95caf274..bb2b4e7ff 100644 --- a/docker.nix +++ b/docker.nix @@ -33,7 +33,7 @@ let root = { uid = 0; - shell = "/bin/bash"; + shell = "${pkgs.bashInteractive}/bin/bash"; home = "/root"; gid = 0; }; From fe958a682d293dec5f27d0c161833b453370d755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 11:42:10 +0200 Subject: [PATCH 293/371] Test building --- tests/installer/default.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 9b2a34bdb..d0707018f 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -146,7 +146,7 @@ let echo "Testing Nix installation..." # FIXME: should update ~/.bashrc. - $ssh " + $ssh < \$out"]; }') + [[ \$(cat \$out) = foobar ]] + EOF echo "Done!" touch $out From 29aaec1e593f1837a73779f243ed0ec4220f7ea8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 13:15:26 +0200 Subject: [PATCH 294/371] Make cross product of images and tests --- tests/installer/default.nix | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index d0707018f..72ad764cc 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -168,11 +168,9 @@ let in -{ - ubuntu-14-04.install-default = makeTest "ubuntu-14-04" "install-default"; - #ubuntu-16-04.install-default = makeTest "ubuntu-16-04" "install-default"; - ubuntu-22-10.install-default = makeTest "ubuntu-22-10" "install-default"; - ubuntu-22-10.install-force-daemon = makeTest "ubuntu-22-10" "install-force-daemon"; - fedora-36.install-default = makeTest "fedora-36" "install-default"; - fedora-36.install-force-daemon = makeTest "fedora-36" "install-force-daemon"; -} +builtins.mapAttrs (imageName: image: + { ${image.system} = builtins.mapAttrs (testName: test: + makeTest imageName testName + ) installScripts; + } +) images From 5c8cdb9b60e0e8d24458a15577e4be3aaa16b600 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 13:19:46 +0200 Subject: [PATCH 295/371] Add Ubuntu 22.04 LTS --- tests/installer/default.nix | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 72ad764cc..17b0fd4ea 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -52,6 +52,15 @@ let system = "x86_64-linux"; }; + "ubuntu-22-04" = { + image = import { + url = https://app.vagrantup.com/generic/boxes/ubuntu2204/versions/4.1.12/providers/libvirt.box; + hash = "sha256-HNll0Qikw/xGIcogni5lz01vUv+R3o8xowP2EtqjuUQ="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + "ubuntu-22-10" = { image = import { url = https://app.vagrantup.com/ubuntu/boxes/kinetic64/versions/20220910.0.0/providers/virtualbox.box; From ef714aa8a566bbdb30919ffd45b8a1fd8e2bc484 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 13:25:26 +0200 Subject: [PATCH 296/371] Remove pre-release Ubuntu 22.10 --- tests/installer/default.nix | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 17b0fd4ea..a2cbbcbb8 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -61,15 +61,6 @@ let system = "x86_64-linux"; }; - "ubuntu-22-10" = { - image = import { - url = https://app.vagrantup.com/ubuntu/boxes/kinetic64/versions/20220910.0.0/providers/virtualbox.box; - hash = "sha256-/IXr+Apyx2dqX6Gj4SoNtQ/5v1eKKopwzFgozAq6GFY="; - }; - rootDisk = "ubuntu-kinetic-22.10-cloudimg.vmdk"; - system = "x86_64-linux"; - }; - "fedora-36" = { image = import { url = https://app.vagrantup.com/generic/boxes/fedora36/versions/4.1.12/providers/libvirt.box; From 503f31e2a0de3192ccc572cc17a6dd02863ebec8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 13:28:03 +0200 Subject: [PATCH 297/371] Use libvirt image --- tests/installer/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index a2cbbcbb8..39e3c8d26 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -45,10 +45,10 @@ let "ubuntu-16-04" = { image = import { - url = https://app.vagrantup.com/ubuntu/boxes/xenial64/versions/20211001.0.0/providers/virtualbox.box; - hash = "sha256-JCc0wd9vaSzCU8coByVtb/oDTAXYBPnORwEShS4oj4U="; + url = https://app.vagrantup.com/generic/boxes/ubuntu1604/versions/4.1.12/providers/libvirt.box; + hash = "sha256-lO4oYQR2tCh5auxAYe6bPOgEqOgv3Y3GC1QM1tEEEU8="; }; - rootDisk = "ubuntu-xenial-16.04-cloudimg.vmdk"; + rootDisk = "box.img"; system = "x86_64-linux"; }; From a96ad2ab25ea054b9d1c473ce2f692bd1f83402b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 14:51:10 +0200 Subject: [PATCH 298/371] Add RHEL 7/8 --- tests/installer/default.nix | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 39e3c8d26..d31e2a949 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -71,6 +71,38 @@ let postBoot = disableSELinux; }; + # Currently fails with 'error while loading shared libraries: + # libsodium.so.23: cannot stat shared object: Invalid argument'. + /* + "rhel-6" = { + image = import { + url = https://app.vagrantup.com/generic/boxes/rhel6/versions/4.1.12/providers/libvirt.box; + hash = "sha256-QwzbvRoRRGqUCQptM7X/InRWFSP2sqwRt2HaaO6zBGM="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + */ + + "rhel-7" = { + image = import { + url = https://app.vagrantup.com/generic/boxes/rhel7/versions/4.1.12/providers/libvirt.box; + hash = "sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + }; + + "rhel-8" = { + image = import { + url = https://app.vagrantup.com/generic/boxes/rhel8/versions/4.1.12/providers/libvirt.box; + hash = "sha256-zFOPjSputy1dPgrQRixBXmlyN88cAKjJ21VvjSWUCUY="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + }; + }; makeTest = imageName: testName: @@ -116,7 +148,7 @@ let chmod 0400 ./vagrant_insecure_key - ssh_opts="-o StrictHostKeyChecking=no -o PubkeyAcceptedKeyTypes=+ssh-rsa -i ./vagrant_insecure_key" + ssh_opts="-o StrictHostKeyChecking=no -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedKeyTypes=+ssh-rsa -i ./vagrant_insecure_key" ssh="ssh -p 20022 -q $ssh_opts vagrant@localhost" echo "Waiting for SSH..." From ba04b5b1d74c285e12fc3d24524cb8f30f108767 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 14:51:44 +0200 Subject: [PATCH 299/371] Disable Ubuntu 14.04 --- tests/installer/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index d31e2a949..a3a7f85f9 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -34,6 +34,7 @@ let images = { + /* "ubuntu-14-04" = { image = import { url = https://app.vagrantup.com/ubuntu/boxes/trusty64/versions/20190514.0.0/providers/virtualbox.box; @@ -42,6 +43,7 @@ let rootDisk = "box-disk1.vmdk"; system = "x86_64-linux"; }; + */ "ubuntu-16-04" = { image = import { From 3dd313a7c20772ef34af4a43fb3673df1e7d00cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 15:50:52 +0200 Subject: [PATCH 300/371] Add RHEL 9 --- tests/installer/default.nix | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index a3a7f85f9..eab103562 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -105,6 +105,17 @@ let postBoot = disableSELinux; }; + "rhel-9" = { + image = import { + url = https://app.vagrantup.com/generic/boxes/rhel9/versions/4.1.12/providers/libvirt.box; + hash = "sha256-vL/FbB3kK1rcSaR627nWmScYGKGk4seSmAdq6N5diMg="; + }; + rootDisk = "box.img"; + system = "x86_64-linux"; + postBoot = disableSELinux; + extraQemuOpts = "-cpu Westmere-v2"; + }; + }; makeTest = imageName: testName: @@ -128,7 +139,7 @@ let qemu-img create -b ./${image.rootDisk} -F "$image_type" -f qcow2 ./disk.qcow2 - extra_qemu_opts= + extra_qemu_opts="${image.extraQemuOpts}" # Add the config disk, required by the Ubuntu images. config_drive=$(echo *configdrive.vmdk || true) From 0d4bf9c4d836f8e8570ad1d39245a5835ef4aaf1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Sep 2022 15:56:46 +0200 Subject: [PATCH 301/371] Fix evaluation --- tests/installer/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index eab103562..c118937a6 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -139,7 +139,7 @@ let qemu-img create -b ./${image.rootDisk} -F "$image_type" -f qcow2 ./disk.qcow2 - extra_qemu_opts="${image.extraQemuOpts}" + extra_qemu_opts="${image.extraQemuOpts or ""}" # Add the config disk, required by the Ubuntu images. config_drive=$(echo *configdrive.vmdk || true) From 84fb036062b879c454188a2a4f7123720a6eb9be Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Sep 2022 18:16:17 +0200 Subject: [PATCH 302/371] add issue template for missing or incorrect documentation this allows anyone to create labelled issues for easy filtering. --- .../ISSUE_TEMPLATE/missing_documentation.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/missing_documentation.md diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md new file mode 100644 index 000000000..84868814f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -0,0 +1,28 @@ +--- +name: Missing or incorrect documentation +about: +title: '' +labels: 'documentation' +assignees: '' + +--- + +## Problem + + + +## Checklist + + + +- [ ] checked [latest Nix manual]\ ([source]) +- [ ] checked [open documentation issues and pull requests] for possible duplicates + +[latest Nix manual]: https://nixos.org/manual/nix/unstable/ +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/src +[open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation + +## Proposal + + + From 47fa1087c8864654f6cfae84e25ac6db318ed1d4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Sep 2022 09:36:20 +0200 Subject: [PATCH 303/371] Update doc/manual/src/contributing/hacking.md --- doc/manual/src/contributing/hacking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index d8a8c8591..628744bf2 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -142,8 +142,8 @@ After the CI run completes, you can check the output to extract the installer ur 1. Click into the detailed view of the CI run. 2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). 3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the install_url -5. To generate an install command, plug this install_url and your github username into this template: +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: ```console sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve From 0a4bd9fe88807cbae51b8b8b51c4897a76991d20 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Sep 2022 09:36:30 +0200 Subject: [PATCH 304/371] Update doc/manual/src/contributing/hacking.md --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 628744bf2..9e4e679e6 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -138,7 +138,7 @@ If you've already pushed to a fork of Nix on GitHub before, you may have noticed ### Using the CI-generated installer for manual testing -After the CI run completes, you can check the output to extract the installer url: +After the CI run completes, you can check the output to extract the installer URL: 1. Click into the detailed view of the CI run. 2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). 3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) From 1ae974120a24f70eba12e073dbba4b7bac73eedf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Sep 2022 09:36:37 +0200 Subject: [PATCH 305/371] Update doc/manual/src/contributing/hacking.md --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9e4e679e6..7f3905d38 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -121,7 +121,7 @@ If you've already pushed to a fork of Nix on GitHub before, you may have noticed - `x86_64-darwin`. While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. -- the `installer_test` job will try to use this installer and run a trivial Nix command on `ubuntu-latest` and `macos-latest`. +- The `installer_test` job will try to use this installer and run a trivial Nix command on `ubuntu-latest` and `macos-latest`. ### One-time setup From dc8c0b173c2cc5bd0fe4273f741fda5591ba4133 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Sep 2022 09:36:55 +0200 Subject: [PATCH 306/371] Update doc/manual/src/contributing/hacking.md --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 7f3905d38..f4aeda871 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -130,7 +130,7 @@ If you've already pushed to a fork of Nix on GitHub before, you may have noticed - Create or log in to an account. - Create a Cachix cache using the format `-nix-install-tests`. - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new cachix auth token and copy the generated value. + - Generate a new Cachix auth token and copy the generated value. 4. At github.com: - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - Name the secret `CACHIX_AUTH_TOKEN` From 875a99eaa483850e7794a495102ce0c97658d89f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 16 Sep 2022 09:41:27 +0200 Subject: [PATCH 307/371] fix markdown rendering quirk markdown would interpret parentheses as belonging to the first link without escaping. --- .github/ISSUE_TEMPLATE/missing_documentation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 84868814f..8ded9f063 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -15,7 +15,7 @@ assignees: '' -- [ ] checked [latest Nix manual]\ ([source]) +- [ ] checked [latest Nix manual] \([source]) - [ ] checked [open documentation issues and pull requests] for possible duplicates [latest Nix manual]: https://nixos.org/manual/nix/unstable/ From b3550d9179611692a4e27fbe4e5f493f4e8713e3 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 16 Sep 2022 00:47:54 -0700 Subject: [PATCH 308/371] libexpr/fetchurl.nix: allow __impure fetch This commit adds an optional `__impure` parameter to fetchurl.nix, which allows the caller to use `libfetcher`'s fetcher in an impure derivation. This allows nixpkgs' patch-normalizing fetcher (fetchpatch) to be rewritten to use nix's internal fetchurl, thereby eliminating the awkward "you can't use fetchpatch here" banners scattered all over the place. See also: https://github.com/NixOS/nixpkgs/pull/188587 --- src/libexpr/fetchurl.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix index 02531103b..38815fcc4 100644 --- a/src/libexpr/fetchurl.nix +++ b/src/libexpr/fetchurl.nix @@ -12,13 +12,13 @@ , executable ? false , unpack ? false , name ? baseNameOf (toString url) +, __impure ? false }: -derivation { +derivation ({ builder = "builtin:fetchurl"; # New-style output content requirements. - inherit outputHashAlgo outputHash; outputHashMode = if unpack || executable then "recursive" else "flat"; inherit name url executable unpack; @@ -38,4 +38,6 @@ derivation { # To make "nix-prefetch-url" work. urls = [ url ]; -} +} // (if __impure + then { inherit __impure; } + else { inherit outputHashAlgo outputHash; })) From 673fd21b7c12b3b0a7fd7e0c9c78caefd8906836 Mon Sep 17 00:00:00 2001 From: Adam Joseph <54836058+amjoseph-nixpkgs@users.noreply.github.com> Date: Fri, 16 Sep 2022 08:51:14 +0000 Subject: [PATCH 309/371] Update src/libexpr/fetchurl.nix Co-authored-by: Eelco Dolstra --- src/libexpr/fetchurl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix index 38815fcc4..b487e959a 100644 --- a/src/libexpr/fetchurl.nix +++ b/src/libexpr/fetchurl.nix @@ -12,7 +12,7 @@ , executable ? false , unpack ? false , name ? baseNameOf (toString url) -, __impure ? false +, impure ? false }: derivation ({ From fb985f855c3bba09703bfb0ad7618ab881c2b0c4 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 16 Sep 2022 01:52:20 -0700 Subject: [PATCH 310/371] fetchurl.nix: change other use of __impure --- src/libexpr/fetchurl.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix index b487e959a..9d1b61d7f 100644 --- a/src/libexpr/fetchurl.nix +++ b/src/libexpr/fetchurl.nix @@ -38,6 +38,6 @@ derivation ({ # To make "nix-prefetch-url" work. urls = [ url ]; -} // (if __impure - then { inherit __impure; } +} // (if impure + then { __impure = true; } else { inherit outputHashAlgo outputHash; })) From ad5b09423aa7e5112de06667e28e78a9e4d1ff33 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Fri, 16 Sep 2022 01:59:24 -0700 Subject: [PATCH 311/371] release-notes/rl-next.md: note new argument to fetchurl.nix --- doc/manual/src/release-notes/rl-next.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 78ae99f4b..68f7d1a9d 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,2 +1,7 @@ # Release X.Y (202?-??-??) +* `` now accepts an additional argument `impure` which + defaults to `false`. If it is set to `true`, the `hash` and `sha256` + arguments will be ignored and the resulting derivation will have + `__impure` set to `true`, making it an impure derivation. + From 4bd52bf6c4c88e2f8a9b703d75c3db5ad062353c Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sat, 17 Sep 2022 13:20:11 -0500 Subject: [PATCH 312/371] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index f4aeda871..5fad34763 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -85,8 +85,6 @@ $ nix develop ## Testing Nix -Nix comes with three different flavors of tests: unit, functional and integration. - ### Unit-tests The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined From 84bdb0e3ade70be722087b95beb7f460e0d3da8d Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sun, 18 Sep 2022 12:58:28 -0500 Subject: [PATCH 313/371] address review feedback Mainly: - Try to triangulate between narrative that framed this as a new/easy process and the need for a reference that will not quickly grow stale. - Fix a ~continuity issue where the text was talking about "your Cachix cache" before saying that you'd need to make a Cachix cache to enable the installer tests. - Adopt suggestion on titling, and nest subtitles in the installer test section. --- doc/manual/src/contributing/hacking.md | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 5fad34763..f67660ab2 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -83,7 +83,7 @@ by: $ nix develop ``` -## Testing Nix +## Running tests ### Unit-tests @@ -107,21 +107,21 @@ Because these tests are expensive and require more than what the standard github You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` -## Testing the install scripts +### Installer tests -Testing the install scripts has traditionally been tedious, but you can now do this much more easily via the GitHub Actions CI runs (at least for platforms that Github Actions supports). +With just a little one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can easily test the installer each time you push to a branch. -If you've already pushed to a fork of Nix on GitHub before, you may have noticed that the CI workflows in your fork list skipped `installer` and `installer_test` jobs. Once your Nix fork is set up correctly, pushing to it will also run these jobs. -- The `installer` job will generate installers for these platforms: +Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - `x86_64-linux` - `armv6l-linux` - `armv7l-linux` - - `x86_64-darwin`. - - While this installer is in your Cachix cache, you can use it for manual testing on any of these platforms. -- The `installer_test` job will try to use this installer and run a trivial Nix command on `ubuntu-latest` and `macos-latest`. + - `x86_64-darwin` -### One-time setup +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +#### One-time setup 1. Have a GitHub account with a fork of the Nix repo. 2. At cachix.org: @@ -129,12 +129,12 @@ If you've already pushed to a fork of Nix on GitHub before, you may have noticed - Create a Cachix cache using the format `-nix-install-tests`. - Navigate to the new cache > Settings > Auth Tokens. - Generate a new Cachix auth token and copy the generated value. -4. At github.com: +3. At github.com: - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN` + - Name the secret `CACHIX_AUTH_TOKEN`. - Paste the copied value of the Cachix cache auth token. -### Using the CI-generated installer for manual testing +#### Using the CI-generated installer for manual testing After the CI run completes, you can check the output to extract the installer URL: 1. Click into the detailed view of the CI run. @@ -147,7 +147,7 @@ After the CI run completes, you can check the output to extract the installer UR sh <(curl -L ) --tarball-url-prefix https://-nix-install-tests.cachix.org/serve ``` -