From 523359d133ea8f048f69d8e8b3e5f1b674baf17a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 08:38:12 -0500 Subject: [PATCH 001/123] WIP: Document the design of Nix The current docs are all "how to do things" and no "what is Nix" or "why are things the way they are". I see lots of misconception on the wider internet, and I also think we would benefit from a "living document" to answer some questions people currently turn to the thesis for. I think a new section of the manual can address all these issues. --- doc/manual/local.mk | 6 +++- doc/manual/src/SUMMARY.md.in | 5 +++ doc/manual/src/design/design.md | 5 +++ doc/manual/src/design/overview.md | 13 ++++++++ doc/manual/src/design/store/entries.md | 46 ++++++++++++++++++++++++++ doc/manual/src/design/store/paths.md | 30 +++++++++++++++++ doc/manual/src/design/store/store.md | 5 +++ 7 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 doc/manual/src/design/design.md create mode 100644 doc/manual/src/design/overview.md create mode 100644 doc/manual/src/design/store/entries.md create mode 100644 doc/manual/src/design/store/paths.md create mode 100644 doc/manual/src/design/store/store.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 371ed6f21..a26fb8fcf 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,9 @@ ifeq ($(doc_generate),yes) +MANUAL_SRCS := \ + $(call rwildcard, $(d)/src, *.md) \ + $(call rwildcard, $(d)/src, */*.md) + # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ @@ -97,7 +101,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(call rwildcard, $(d)/src, *.md) +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/expressions/builtins.md $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual endif diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c8cb72fc0..c320bde3f 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,6 +15,11 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) +- [Design and Data Model](design/design.md) + - [Overview](design/overview.md) + - [The Store Layer](design/store/store.md) + - [Store Entries](design/store/entries.md) + - [Store Paths](design/store/paths.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md new file mode 100644 index 000000000..7d4211764 --- /dev/null +++ b/doc/manual/src/design/design.md @@ -0,0 +1,5 @@ +# Design and Data Model + +Most of the manual is about how to use Nix. +This chapter is about what Nix actually is. +The hope is that it can serve as a reference for key concepts, and also shed light on why things are the way they are. diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md new file mode 100644 index 000000000..05f4a70b2 --- /dev/null +++ b/doc/manual/src/design/overview.md @@ -0,0 +1,13 @@ +# Overview + +Nix is broken into layers that operate fairly independently. + +At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. + +Below that is the Nix *expression language*, in which packages and configurations are written. +These are the layers which users interact with most. + +Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. +The store layer may not be as visible, but this is the heart of Nix. + +This chapter will start there and work up towards the more user-facing interfaces described in the rest of the manual. diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md new file mode 100644 index 000000000..c16c98a36 --- /dev/null +++ b/doc/manual/src/design/store/entries.md @@ -0,0 +1,46 @@ +# Store Entries + +File system data in Nix is organized into *store entries*. +A store entry is the combination of + + - some file system data + - references to store entries + +## File system data + +Nix supports the a similar model of the file system as Git. +Namely, every file system object falls into these three cases: + + - File: arbitrary data + + - Directory: mapping of names to child file system objects. + File children additionally have an executable flag. + + - Symlink: may point anywhere. + In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. + +A bare file as the "root" file system object is allowed. +Note that there is no containing directory object to store its executable bit; it's deemed non-executable by default. + +## References + +Store entries can refer to both other store entries and themselves. + +Store references are normally calculated by scanning the file system data for store paths when a new store entry is created, but this isn't mandatory, as store entries are allowed to have references that don't correspond to contained store paths, and contained store paths that don't correspond to references. + +The references themselves need not be store paths per-se (this is an implementation detail of the store). +But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. +That said, all the references of the store entry must agree on a store dir. +Also the store directory of the references must equal that of any store which contains the store entry doing the referencing. + +## Relocatability + +The two final restrictions of the previous section yield an alternative of view of the same information. +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least once reference. + +This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. + +Lastly, this illustrates the purpose of tracking self references. +Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. +\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md new file mode 100644 index 000000000..1e4369625 --- /dev/null +++ b/doc/manual/src/design/store/paths.md @@ -0,0 +1,30 @@ +# Store Paths + +A store path is a pair of a 20-byte digest and a name. + +Historically it is the triple of those two and also the store directory, but the modern implementation's internal representation is just the pair. +This change is because in the vast majority of cases, the store dir is fully determined by the context in which the store path occurs. + +## String representation + +A store path is rendered as the concatenation of + + - the store directory + + - a path-separator (`/`) + + - the digest rendered as Base-32 (20 bytes becomes 32 bytes) + + - a hyphen (`-`) + + - the name + +Let's take the store path from the very beginning of this manual as an example: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + +This parses like so: + + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ + store dir digest name diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md new file mode 100644 index 000000000..c2431f1c5 --- /dev/null +++ b/doc/manual/src/design/store/store.md @@ -0,0 +1,5 @@ +A Nix store is a collection of *store entries* referred to by *store paths*. +Every store also has a "store directory path", which is a path prefix used for various purposes. + +There are many types of stores, but all of them at least respect this model. +Some however offer additional functionality. From a2b3160f2898c2009d458cc27d5e8ea925cf583b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 09:09:42 -0500 Subject: [PATCH 002/123] Briefly describe the digest of a store path --- doc/manual/src/design/store/paths.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 1e4369625..18ba27ecb 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -28,3 +28,24 @@ This parses like so: /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ store dir digest name + +## The digest + +The calculation of the digest is quite complicated for historical reasons. +Some of the details will be saved for later. + +> Historical note. The 20 byte restriction is because originally a digests were SHA-1 hashes. +> This is no longer true, but longer hashes and other information is still boiled down to 20 bytes. + +Store paths are either content-addressed or "input-addressed". + +Content addressing means that the digest ultimately derives from referred store entry's file system data and references, and thus can be verified (if one knows how it was calculated). + +Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. +Store paths of this sort can not be validated from the content of the store entry. +Rather, the store entry might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. +The signature indicates that someone is vouching for the store entry really being the results of a plan with that digest. + +While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. +That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. +In the parlance of referencing schemes, this means that store paths are not "self-describing". From e64633f98f290cb571df72a667a0a267722fc397 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 24 Nov 2020 10:18:39 -0500 Subject: [PATCH 003/123] Flesh out TOC --- doc/manual/src/SUMMARY.md.in | 6 ++++++ doc/manual/src/design/store/building.md | 0 doc/manual/src/design/store/drvs/ca.md | 0 doc/manual/src/design/store/drvs/drvs.md | 0 doc/manual/src/design/store/drvs/ia.md | 0 doc/manual/src/design/store/input-addressing.md | 1 + doc/manual/src/design/store/nar.md | 1 + doc/manual/src/design/store/netry-ca.md | 1 + 8 files changed, 9 insertions(+) create mode 100644 doc/manual/src/design/store/building.md create mode 100644 doc/manual/src/design/store/drvs/ca.md create mode 100644 doc/manual/src/design/store/drvs/drvs.md create mode 100644 doc/manual/src/design/store/drvs/ia.md create mode 100644 doc/manual/src/design/store/input-addressing.md create mode 100644 doc/manual/src/design/store/nar.md create mode 100644 doc/manual/src/design/store/netry-ca.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c320bde3f..8c4dbcb18 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,6 +20,12 @@ - [The Store Layer](design/store/store.md) - [Store Entries](design/store/entries.md) - [Store Paths](design/store/paths.md) + - [Nix Archives](design/store/nar.md) + - [Content-Addressing Store Entries](design/store/entry-ca.md) + - [Derivations](design/store/drvs/drvs.md) + - [Input-Addressing](design/store/drvs/ia.md) + - [Content-Addressing (Experimental)](design/store/drvs/ca.md) + - [Building](design/store/building.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/ca.md b/doc/manual/src/design/store/drvs/ca.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/drvs/ia.md b/doc/manual/src/design/store/drvs/ia.md new file mode 100644 index 000000000..e69de29bb diff --git a/doc/manual/src/design/store/input-addressing.md b/doc/manual/src/design/store/input-addressing.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/input-addressing.md @@ -0,0 +1 @@ +TODO diff --git a/doc/manual/src/design/store/nar.md b/doc/manual/src/design/store/nar.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/nar.md @@ -0,0 +1 @@ +TODO diff --git a/doc/manual/src/design/store/netry-ca.md b/doc/manual/src/design/store/netry-ca.md new file mode 100644 index 000000000..1333ed77b --- /dev/null +++ b/doc/manual/src/design/store/netry-ca.md @@ -0,0 +1 @@ +TODO From a210504bc7b67defe16fda97a6d2a844ecc9bd2b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 11:41:15 -0400 Subject: [PATCH 004/123] Apply suggestions from code review --- doc/manual/src/design/overview.md | 2 +- doc/manual/src/design/store/entries.md | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 05f4a70b2..792cfefbf 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -10,4 +10,4 @@ These are the layers which users interact with most. Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. The store layer may not be as visible, but this is the heart of Nix. -This chapter will start there and work up towards the more user-facing interfaces described in the rest of the manual. +This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index c16c98a36..2eb4bf90c 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -8,8 +8,8 @@ A store entry is the combination of ## File system data -Nix supports the a similar model of the file system as Git. -Namely, every file system object falls into these three cases: +The nix store uses a simple filesystem model, similar to the one Git uses. +In particular, every file system object falls into these three cases: - File: arbitrary data @@ -20,13 +20,25 @@ Namely, every file system object falls into these three cases: In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. -Note that there is no containing directory object to store its executable bit; it's deemed non-executable by default. +Note that it cannot be executable, though. +This is a consequence of the executable flags being part of the child entries of the directory, rather than the child files themselves. +A root file has no parent directory; so there is no child entry about the root file, and therefore no executable flag for it. +Without a flag saying which, whether root files are executable or non-executable by default must be decided by convention, and the choice of Nix (and git) is to make them non-executable. ## References Store entries can refer to both other store entries and themselves. -Store references are normally calculated by scanning the file system data for store paths when a new store entry is created, but this isn't mandatory, as store entries are allowed to have references that don't correspond to contained store paths, and contained store paths that don't correspond to references. +References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store entries. +For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store entries by searching in all their files. +When we get to building in a future section, this process will be described in precise detail. + +However, scanning for references is not mandatory. +Store entries are allowed to have official references that *don't* correspond to store paths contained in their contents, +and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. +Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. + +This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. The references themselves need not be store paths per-se (this is an implementation detail of the store). But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. @@ -35,8 +47,8 @@ Also the store directory of the references must equal that of any store which co ## Relocatability -The two final restrictions of the previous section yield an alternative of view of the same information. -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least once reference. +The two final restrictions of the previous section yield an alternative view of the same information. +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. From e3a0209a9e279a30ea9381743c77b2737e8ed95e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 12:35:01 -0400 Subject: [PATCH 005/123] Move the bits on relocating store entires to the end They are too advanced for up front. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/design/store/building.md | 15 +++++++++++++++ doc/manual/src/design/store/entries.md | 17 ----------------- doc/manual/src/design/store/relocatability.md | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 17 deletions(-) create mode 100644 doc/manual/src/design/store/relocatability.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8c4dbcb18..1e8b35483 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,6 +26,7 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) + - [Advanced Topic: store dir relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index e69de29bb..5adccbc0b 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -0,0 +1,15 @@ +# Building + +## Scanning for references + +Before in the section on [store entries](../entries.md), we talked abstractly about scanning for references. +Now we can make this concrete. + +After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store entries. +There is a few steps of this, but let's start with the simple case of one input-addressed output first. + +\[Overview of things that need to happen.] + +For example, if Nix thinks `/nix/store/asdfasdfasdf-foo` and `/nix/store/qwerqwerqwer-bar` are paths the data might plausibly reference, Nix will scan all the contents of all files recursively for the "hash parts" `asdfasdfasdf`` and `qwerqwerqwer`. + +\[Explain why whitelist.] diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index 2eb4bf90c..74f429e78 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -39,20 +39,3 @@ and they are also allowed to *not* have references that *do* correspond to store Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. - -The references themselves need not be store paths per-se (this is an implementation detail of the store). -But, like rendered store paths (see next section) in addition to identifying store entries they must also identify the store directory of the store(s) that contain those store entries. -That said, all the references of the store entry must agree on a store dir. -Also the store directory of the references must equal that of any store which contains the store entry doing the referencing. - -## Relocatability - -The two final restrictions of the previous section yield an alternative view of the same information. -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. - -This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. - -Lastly, this illustrates the purpose of tracking self references. -Store entries without self-references or other references are relocatable, while store paths with self-references aren't. -This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. -\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md new file mode 100644 index 000000000..04b581459 --- /dev/null +++ b/doc/manual/src/design/store/relocatability.md @@ -0,0 +1,15 @@ +## Advanced Topic: store entry relocatability + +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store entry to a store with a different mount point. + +Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. + +~~The two final restrictions of the previous section yield an alternative view of the same information.~~ +Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. + +This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. + +Lastly, this illustrates the purpose of tracking self references. +Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. +\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] From 678d75baeaca049042330e543f610d630e8c43b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 13:13:55 -0400 Subject: [PATCH 006/123] Start on the derivations section --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/design/store/drvs/drvs.md | 59 ++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1e8b35483..a0efdb9cd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,7 +26,7 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store dir relocatability](design/store/relocatability.md) + - [Advanced Topic: store entry relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index e69de29bb..cedb05985 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -0,0 +1,59 @@ +# Derivations + +Derivations are recipes to create store entries. + +Derivations are the heart of Nix. +Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. +This is where Nix comes in. + +Derivations produce data by running arbitrary commands, like Make or Ninja rules. +Unlike those systems, derivations do not produce arbitrary files, but only specific store entries. +They cannot modify the store in any way, other than creating those store entries. +This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. + +Based on the above, we can conceptually break derivations down into 3 parts: + +1. What command will be run? + +2. What existing store entries are needed as inputs? + +3. What store entries will be produced as outputs? + +## What command will be run? + +The original core of Nix was very simple about this, in the mold of traditional Unix. +Commands consist of 3 parts: + +1. Path to executable + +2. Arguments (Excecpt for `argv[0]`, which is taken from the path in the usual way) + +3. Environment variables. + +## What existing store entries are needed as inputs? + +The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" +It's a good questions! + +## What store entries will be produced as outputs? + +## Extra extensions + +### `__structuredAttrs` + +Historically speaking, most users of Nix made GNU Bash with a script the command run, regardless of what they were doing. +Bash variable are automatically created from env vars, but bash also supports array and string-keyed map variables in addition to string variables. +People also usually create derivations using language which also support these richer data types. +It was thus desired a way to get this data from the language "planning" the derivation to language to bash, the language evaluated at "run time". + +`__structuredAttrs` does this by smuggling inside the core derivation format a map of named richer data. +At run time, this becomes two things: + +1. A JSON file containing that map. +2. A bash script setting those variables. + +The bash command can be passed a script which will "source" that Nix-created bash script, setting those variables with the richer data. +The outer script can then do whatever it likes with those richer variables as input. + +However, since derivations can already contain arbitary input sources, the vast majority of `__structuredAttrs` can be handled by upper layers. +We might consider implementing `__structuredAttrs` in higher layers in the future, and simplifying the store layer. From f5386d7059ad2c63b32356082d1f6ecacfc1e93b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 21 Mar 2022 13:15:35 -0400 Subject: [PATCH 007/123] Fix stub file's name --- doc/manual/src/design/store/{netry-ca.md => entry-ca.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/manual/src/design/store/{netry-ca.md => entry-ca.md} (100%) diff --git a/doc/manual/src/design/store/netry-ca.md b/doc/manual/src/design/store/entry-ca.md similarity index 100% rename from doc/manual/src/design/store/netry-ca.md rename to doc/manual/src/design/store/entry-ca.md From a04340f9a1ce3c107b21a11395b4a2348fd19571 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 10:55:30 -0400 Subject: [PATCH 008/123] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 792cfefbf..7790e701e 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -7,7 +7,8 @@ At the top is the *command line interface*, i.e. the argument parsing of the var Below that is the Nix *expression language*, in which packages and configurations are written. These are the layers which users interact with most. -Below that is the *store layer*, Nix's machinery for presenting and files and fully elaborated build plans, and also executing those build plans. +Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. +It is also used for executing those build plans. The store layer may not be as visible, but this is the heart of Nix. This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." From 75c5191a1f8cf70a11098e89ed33edc203f52dab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 11:04:55 -0400 Subject: [PATCH 009/123] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 7790e701e..bd3d935e6 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -1,6 +1,6 @@ # Overview -Nix is broken into layers that operate fairly independently. +Nix consists of layers that operate fairly independently. At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. From cdb0bf3b65172fd0e366d44f17392b4261c3d925 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 22 Mar 2022 11:15:56 -0400 Subject: [PATCH 010/123] Update doc/manual/src/design/overview.md Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index bd3d935e6..2d3e9226b 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,7 +5,8 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. Below that is the Nix *expression language*, in which packages and configurations are written. -These are the layers which users interact with most. + +The command line and expression language are what users interact with most. Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. It is also used for executing those build plans. From e308602fdfabf0ff1f0cd95baf1d90ba54a02281 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Apr 2022 20:48:03 -0400 Subject: [PATCH 011/123] Update doc/manual/src/design/store/drvs/drvs.md Co-authored-by: Matthieu Coudron --- doc/manual/src/design/store/drvs/drvs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index cedb05985..98e95809b 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -26,7 +26,7 @@ Commands consist of 3 parts: 1. Path to executable -2. Arguments (Excecpt for `argv[0]`, which is taken from the path in the usual way) +2. Arguments (Except for `argv[0]`, which is taken from the path in the usual way) 3. Environment variables. From 4e2d5ae2027a19ab12d6d024cdf5125c3200a7a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 21:53:39 -0400 Subject: [PATCH 012/123] doc: Store entry -> store object This matches the terminology in Eelco's thesis. --- doc/manual/src/SUMMARY.md.in | 6 +++--- doc/manual/src/design/overview.md | 2 +- doc/manual/src/design/store/building.md | 4 ++-- doc/manual/src/design/store/drvs/drvs.md | 14 ++++++------- doc/manual/src/design/store/entries.md | 20 +++++++++---------- doc/manual/src/design/store/paths.md | 8 ++++---- doc/manual/src/design/store/relocatability.md | 10 +++++----- doc/manual/src/design/store/store.md | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a0efdb9cd..5c4cfc920 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,15 +18,15 @@ - [Design and Data Model](design/design.md) - [Overview](design/overview.md) - [The Store Layer](design/store/store.md) - - [Store Entries](design/store/entries.md) + - [Store Objects](design/store/entries.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Entries](design/store/entry-ca.md) + - [Content-Addressing Store Objects](design/store/entry-ca.md) - [Derivations](design/store/drvs/drvs.md) - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store entry relocatability](design/store/relocatability.md) + - [Advanced Topic: store object relocatability](design/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 2d3e9226b..9329f2ad7 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -2,7 +2,7 @@ Nix consists of layers that operate fairly independently. -At the top is the *command line interface*, i.e. the argument parsing of the various Nix executables. +At the top is the *command line interface*. Below that is the Nix *expression language*, in which packages and configurations are written. diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index 5adccbc0b..f97b74952 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -2,10 +2,10 @@ ## Scanning for references -Before in the section on [store entries](../entries.md), we talked abstractly about scanning for references. +Before in the section on [store objects](../entries.md), we talked abstractly about scanning for references. Now we can make this concrete. -After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store entries. +After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. There is a few steps of this, but let's start with the simple case of one input-addressed output first. \[Overview of things that need to happen.] diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/design/store/drvs/drvs.md index 98e95809b..766a7b47f 100644 --- a/doc/manual/src/design/store/drvs/drvs.md +++ b/doc/manual/src/design/store/drvs/drvs.md @@ -1,23 +1,23 @@ # Derivations -Derivations are recipes to create store entries. +Derivations are recipes to create store objects. Derivations are the heart of Nix. Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. This is where Nix comes in. Derivations produce data by running arbitrary commands, like Make or Ninja rules. -Unlike those systems, derivations do not produce arbitrary files, but only specific store entries. -They cannot modify the store in any way, other than creating those store entries. +Unlike those systems, derivations do not produce arbitrary files, but only specific store objects. +They cannot modify the store in any way, other than creating those store objects. This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. Based on the above, we can conceptually break derivations down into 3 parts: 1. What command will be run? -2. What existing store entries are needed as inputs? +2. What existing store objects are needed as inputs? -3. What store entries will be produced as outputs? +3. What store objects will be produced as outputs? ## What command will be run? @@ -30,12 +30,12 @@ Commands consist of 3 parts: 3. Environment variables. -## What existing store entries are needed as inputs? +## What existing store objects are needed as inputs? The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" It's a good questions! -## What store entries will be produced as outputs? +## What store objects will be produced as outputs? ## Extra extensions diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/entries.md index 74f429e78..ddfefa166 100644 --- a/doc/manual/src/design/store/entries.md +++ b/doc/manual/src/design/store/entries.md @@ -1,10 +1,10 @@ -# Store Entries +# Store Objects -File system data in Nix is organized into *store entries*. -A store entry is the combination of +File system data in Nix is organized into *store objects*. +A store object is the combination of - some file system data - - references to store entries + - references to store objects ## File system data @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: File children additionally have an executable flag. - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing file system data or that of another store entry referenced by the containing store entry are allowed, but might not function as intended. + In particular, Symlinks that do not point within the containing file system data or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. Note that it cannot be executable, though. @@ -27,15 +27,15 @@ Without a flag saying which, whether root files are executable or non-executable ## References -Store entries can refer to both other store entries and themselves. +Store objects can refer to both other store objects and themselves. -References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store entries. -For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store entries by searching in all their files. +References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store objects. +For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. However, scanning for references is not mandatory. -Store entries are allowed to have official references that *don't* correspond to store paths contained in their contents, +Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. -This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store entry, rather than try to recompute them on the fly by scanning their contents. +This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store object, rather than try to recompute them on the fly by scanning their contents. diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 18ba27ecb..d7d2ef960 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -39,12 +39,12 @@ Some of the details will be saved for later. Store paths are either content-addressed or "input-addressed". -Content addressing means that the digest ultimately derives from referred store entry's file system data and references, and thus can be verified (if one knows how it was calculated). +Content addressing means that the digest ultimately derives from referred store object's file system data and references, and thus can be verified (if one knows how it was calculated). Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. -Store paths of this sort can not be validated from the content of the store entry. -Rather, the store entry might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. -The signature indicates that someone is vouching for the store entry really being the results of a plan with that digest. +Store paths of this sort can not be validated from the content of the store object. +Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. +The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md index 04b581459..5cdbe9486 100644 --- a/doc/manual/src/design/store/relocatability.md +++ b/doc/manual/src/design/store/relocatability.md @@ -1,15 +1,15 @@ -## Advanced Topic: store entry relocatability +## Advanced Topic: store object relocatability -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store entry to a store with a different mount point. +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store object to a store with a different mount point. Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. ~~The two final restrictions of the previous section yield an alternative view of the same information.~~ -Rather than associating store dirs with the references, we can say a store entry itself has a store dir if and only if it has at least one reference. +Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. -This corresponds to the observation that a store entry with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store entry without any references, i.e. thus without a store directory, can exist in any store. +This corresponds to the observation that a store object with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store object without any references, i.e. thus without a store directory, can exist in any store. Lastly, this illustrates the purpose of tracking self references. -Store entries without self-references or other references are relocatable, while store paths with self-references aren't. +Store objects without self-references or other references are relocatable, while store paths with self-references aren't. This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. \[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index c2431f1c5..e91f0f6e4 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,4 +1,4 @@ -A Nix store is a collection of *store entries* referred to by *store paths*. +A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. There are many types of stores, but all of them at least respect this model. From 838ba26fda23649545cc61a274e20076a2e27892 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 21:57:10 -0400 Subject: [PATCH 013/123] Rename files after store entry -> store object rename --- doc/manual/src/SUMMARY.md.in | 4 ++-- doc/manual/src/design/store/building.md | 2 +- doc/manual/src/design/store/{entry-ca.md => object-ca.md} | 0 doc/manual/src/design/store/{entries.md => objects.md} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename doc/manual/src/design/store/{entry-ca.md => object-ca.md} (100%) rename doc/manual/src/design/store/{entries.md => objects.md} (100%) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5c4cfc920..323a569d1 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -18,10 +18,10 @@ - [Design and Data Model](design/design.md) - [Overview](design/overview.md) - [The Store Layer](design/store/store.md) - - [Store Objects](design/store/entries.md) + - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Objects](design/store/entry-ca.md) + - [Content-Addressing Store Objects](design/store/object-ca.md) - [Derivations](design/store/drvs/drvs.md) - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/design/store/building.md index f97b74952..f4f2649a3 100644 --- a/doc/manual/src/design/store/building.md +++ b/doc/manual/src/design/store/building.md @@ -2,7 +2,7 @@ ## Scanning for references -Before in the section on [store objects](../entries.md), we talked abstractly about scanning for references. +Before in the section on [store objects](../objects.md), we talked abstractly about scanning for references. Now we can make this concrete. After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. diff --git a/doc/manual/src/design/store/entry-ca.md b/doc/manual/src/design/store/object-ca.md similarity index 100% rename from doc/manual/src/design/store/entry-ca.md rename to doc/manual/src/design/store/object-ca.md diff --git a/doc/manual/src/design/store/entries.md b/doc/manual/src/design/store/objects.md similarity index 100% rename from doc/manual/src/design/store/entries.md rename to doc/manual/src/design/store/objects.md From 1bbad62c7d5f3b0d18c3fdb5a8e947ae5232139f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:02:21 -0400 Subject: [PATCH 014/123] doc: File system data -> file system object, to match Nix --- doc/manual/src/design/store/objects.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index ddfefa166..8783a1648 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -3,10 +3,10 @@ File system data in Nix is organized into *store objects*. A store object is the combination of - - some file system data + - A (root) file system object - references to store objects -## File system data +## File system objects The nix store uses a simple filesystem model, similar to the one Git uses. In particular, every file system object falls into these three cases: @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: File children additionally have an executable flag. - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing file system data or that of another store object referenced by the containing store object are allowed, but might not function as intended. + In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file as the "root" file system object is allowed. Note that it cannot be executable, though. @@ -29,7 +29,7 @@ Without a flag saying which, whether root files are executable or non-executable Store objects can refer to both other store objects and themselves. -References are normally calculated by scanning the file system data for store paths (which we describe in the next section) referring to store objects. +References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. From 5f4d2ac091342c26d1a3f447961932139ed568b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:21:29 -0400 Subject: [PATCH 015/123] Improve store object section In particular, Nix is *not* like Git, so that needs to be fixed. --- doc/manual/src/design/store/objects.md | 32 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 8783a1648..2c2ed2bdb 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -1,34 +1,46 @@ # Store Objects File system data in Nix is organized into *store objects*. -A store object is the combination of +A store object is the pair of - A (root) file system object - - references to store objects + - A set of references to store objects ## File system objects -The nix store uses a simple filesystem model, similar to the one Git uses. +The nix store uses a simple filesystem model. In particular, every file system object falls into these three cases: - - File: arbitrary data + - File: an executable flag, and arbitrary data - Directory: mapping of names to child file system objects. - File children additionally have an executable flag. - Symlink: may point anywhere. + In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. -A bare file as the "root" file system object is allowed. -Note that it cannot be executable, though. -This is a consequence of the executable flags being part of the child entries of the directory, rather than the child files themselves. -A root file has no parent directory; so there is no child entry about the root file, and therefore no executable flag for it. -Without a flag saying which, whether root files are executable or non-executable by default must be decided by convention, and the choice of Nix (and git) is to make them non-executable. +A bare file or symlink as the "root" file system object is allowed. + +### Comparison with Git + +This is close to Git's model, but with one crucial difference: +Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + +So long as the root object is a directory, the representations are isomorphic. +There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. + +However, if the root object is a file, there is loss of fidelity. +Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. + +Git's model matches Unix tradition, but Nix's model is more natural. ## References Store objects can refer to both other store objects and themselves. +Self-reference may seem pointless, but tracking them is in fact useful. +We can best explain why later after more concepts have been established. + References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. When we get to building in a future section, this process will be described in precise detail. From b4df351880bb019f6b3176f739059440f128dc43 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:28:26 -0400 Subject: [PATCH 016/123] Relocability -> relocation in store object title --- doc/manual/src/design/store/relocatability.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/design/store/relocatability.md index 5cdbe9486..c7f869135 100644 --- a/doc/manual/src/design/store/relocatability.md +++ b/doc/manual/src/design/store/relocatability.md @@ -1,8 +1,8 @@ -## Advanced Topic: store object relocatability +## Advanced Topic: Store object relocation -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissable to relocate a store object to a store with a different mount point. +Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissible to relocate a store object to a store with a different mount point. -Recall from the section on [store paths](./store-paths.md), concrete store paths look like `/-`. +Recall from the section on [store paths](./store-paths.md) that concrete store paths look like `/-`. ~~The two final restrictions of the previous section yield an alternative view of the same information.~~ Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. From 55b437b5516dfdcd4358de48e7496b08d7be39c6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 22:56:21 -0400 Subject: [PATCH 017/123] Improve store path section --- doc/manual/src/design/store/objects.md | 2 +- doc/manual/src/design/store/paths.md | 37 +++++++++++++++++++------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 2c2ed2bdb..8431958a5 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -17,7 +17,7 @@ In particular, every file system object falls into these three cases: - Symlink: may point anywhere. - In particular, Symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. + In particular, symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. A bare file or symlink as the "root" file system object is allowed. diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index d7d2ef960..1a00de5cb 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -2,18 +2,15 @@ A store path is a pair of a 20-byte digest and a name. -Historically it is the triple of those two and also the store directory, but the modern implementation's internal representation is just the pair. -This change is because in the vast majority of cases, the store dir is fully determined by the context in which the store path occurs. - ## String representation A store path is rendered as the concatenation of - - the store directory + - a store directory - a path-separator (`/`) - - the digest rendered as Base-32 (20 bytes becomes 32 bytes) + - the digest rendered as Base-32 (20 arbitrary bytes becomes 32 ASCII chars) - a hyphen (`-`) @@ -21,14 +18,32 @@ A store path is rendered as the concatenation of Let's take the store path from the very beginning of this manual as an example: - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 This parses like so: - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ store dir digest name +We then can discard the store dir to recover the conceptual pair that is a store path: + + { + digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", + name: "firefox-33.1", + } + +### Where did the "store directory" come from? + +If you notice, the above references a "store directory", but that is *not* part of the definition of a store path. +We can discard it when parsing, but what about when printing? +We need to get a store directory from *somewhere*. + +The answer is, the store directory is a property of the store that contains the store path. +The explanation for this is simple enough: a store is notionally mounted as a directory at some location, and the store object's root file system likewise mounted at this path within that directory. + +This does, however, mean the string representation of a store path is not derived just from the store path itself, but is in fact "context dependent". + ## The digest The calculation of the digest is quite complicated for historical reasons. @@ -39,10 +54,14 @@ Some of the details will be saved for later. Store paths are either content-addressed or "input-addressed". -Content addressing means that the digest ultimately derives from referred store object's file system data and references, and thus can be verified (if one knows how it was calculated). +Content addressing means that the digest ultimately derives from referred store object's file system objects and references, and thus can be verified. +There is more than one *method* of content-addressing, however. +Still, if one does know the the content addressing schema that was used, +(or guesses, there isn't that many yet!) +one can recalcuate the store path and thus verify the store object. Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. -Store paths of this sort can not be validated from the content of the store object. +Store paths of this sort can *not* be validated from the content of the store object. Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. From b98dc3b19c62abb3b8edc778cee0ac8234c5577d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 18 Apr 2022 23:03:38 -0400 Subject: [PATCH 018/123] store objects, better opining sentances --- doc/manual/src/design/store/objects.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 8431958a5..38c73bc7a 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -1,6 +1,6 @@ # Store Objects -File system data in Nix is organized into *store objects*. +Data in Nix is chiefly organized into *store objects*. A store object is the pair of - A (root) file system object @@ -8,7 +8,7 @@ A store object is the pair of ## File system objects -The nix store uses a simple filesystem model. +The Nix store uses a simple filesystem model. In particular, every file system object falls into these three cases: - File: an executable flag, and arbitrary data From e4eea5e84e78cc57859cca9295da450258e8f55b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 01:44:02 -0400 Subject: [PATCH 019/123] Include abstract syntax based on the thesis for FSOs See https://edolstra.github.io/pubs/phd-thesis.pdf, page 91. --- doc/manual/src/design/store/objects.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 38c73bc7a..0f3d2a499 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -9,6 +9,16 @@ A store object is the pair of ## File system objects The Nix store uses a simple filesystem model. + + data FileSystemObject + = Regular Executable ByteString + | Directory (Map FileName FSO) + | SymLink ByteString + + data Executable + = Executable + | NonExecutable + In particular, every file system object falls into these three cases: - File: an executable flag, and arbitrary data @@ -26,10 +36,21 @@ A bare file or symlink as the "root" file system object is allowed. This is close to Git's model, but with one crucial difference: Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + data GitObject + = Blob ByteString + | Tree (Map FileName (Persission, FSO)) + + data Persission + = Directory -- IFF paired with tree + -- Iff paired with blob, one of: + | RegFile + | ExecutableFile + | Symlink + So long as the root object is a directory, the representations are isomorphic. There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. -However, if the root object is a file, there is loss of fidelity. +However, if the root object is a blob, there is loss of fidelity. Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. Git's model matches Unix tradition, but Nix's model is more natural. From 4e4bbd9e832e810b4e4cf33aced0ec0322cf55b6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 02:02:11 -0400 Subject: [PATCH 020/123] Improve store objects session more --- doc/manual/src/design/store/paths.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/design/store/paths.md index 1a00de5cb..cf51eb866 100644 --- a/doc/manual/src/design/store/paths.md +++ b/doc/manual/src/design/store/paths.md @@ -47,20 +47,28 @@ This does, however, mean the string representation of a store path is not derive ## The digest The calculation of the digest is quite complicated for historical reasons. -Some of the details will be saved for later. +The details of the algorithms will be discussed later once more concepts have been introduced. +For now, we just concern ourselves with the *key properties* of those algorithms. -> Historical note. The 20 byte restriction is because originally a digests were SHA-1 hashes. -> This is no longer true, but longer hashes and other information is still boiled down to 20 bytes. +::: {.note} +**Historical note** The 20 byte restriction is because originally a digests were SHA-1 hashes. +This is no longer true, but longer hashes and other information are still boiled down to 20 bytes. +::: -Store paths are either content-addressed or "input-addressed". +Store paths are either *content-addressed* or *input-addressed*. -Content addressing means that the digest ultimately derives from referred store object's file system objects and references, and thus can be verified. +::: {.note} +The former is a standard term used elsewhere. +The later is our own creation to evoke a contrast with content addressing. +::: + +Content addressing means that the store path digest ultimately derives from referred store object's contents, namely its file system objects and references. There is more than one *method* of content-addressing, however. -Still, if one does know the the content addressing schema that was used, +Still, if one does know the content addressing schema that was used, (or guesses, there isn't that many yet!) -one can recalcuate the store path and thus verify the store object. +one can recalculate the store path and thus verify the store object. -Input addressing means that the digest derives from how the store path was produced -- the "inputs" and plan that it was built from. +Input addressing means that the store path digest derives from how the store path was produced, namely the "inputs" and plan that it was built from. Store paths of this sort can *not* be validated from the content of the store object. Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. From c86c1ec7e31c2725be20e4087845e96878134846 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 02:20:15 -0400 Subject: [PATCH 021/123] Make refernces sneak preview more concise --- doc/manual/src/design/store/objects.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/design/store/objects.md index 0f3d2a499..e4f49a170 100644 --- a/doc/manual/src/design/store/objects.md +++ b/doc/manual/src/design/store/objects.md @@ -62,9 +62,8 @@ Store objects can refer to both other store objects and themselves. Self-reference may seem pointless, but tracking them is in fact useful. We can best explain why later after more concepts have been established. -References are normally calculated by scanning the rooted file system objects for store paths (which we describe in the next section) referring to store objects. -For now, it suffices to say that a store path is a string encoding of a reference to a store paths, and therefore it is something that we can search for in the contents of files, and thus in store objects by searching in all their files. -When we get to building in a future section, this process will be described in precise detail. +References are normally calculated so as to to record the presence of textual references in store object's file systems obejcts. +This process will be described precisely in the section on [building](./building.md), once more concepts are explained, as building is the primary path new store objects with non-trivial references are created. However, scanning for references is not mandatory. Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, From 0737094161cc2b13a6ec06a58a78200cb2116b88 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Apr 2022 01:39:30 -0400 Subject: [PATCH 022/123] Add draft "Rosetta stone" by @fricklerhandwerk and stub commentary The idea and most of the execution are @fricklerhandwerk's. I changed a few things best I could based on @edolstra's corrections, and a Bazel glossary. Valentin Gagarin --- doc/manual/src/SUMMARY.md.in | 3 +- doc/manual/src/design/store/related-work.md | 37 +++++++++++++++++++++ doc/manual/src/design/store/store.md | 22 ++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 doc/manual/src/design/store/related-work.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 323a569d1..9cde44a0d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -26,7 +26,8 @@ - [Input-Addressing](design/store/drvs/ia.md) - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - [Building](design/store/building.md) - - [Advanced Topic: store object relocatability](design/store/relocatability.md) + - [Advanced Topic: Store object relocatability](design/store/relocatability.md) + - [Advanced Topic: Related work](design/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/design/store/related-work.md b/doc/manual/src/design/store/related-work.md new file mode 100644 index 000000000..b64b41988 --- /dev/null +++ b/doc/manual/src/design/store/related-work.md @@ -0,0 +1,37 @@ +# Advanced Topic: Related Work + +## Bazel + +TODO skylark and layering. + +TODO being monadic, if RFC 92. + +## Build Systems à la Carte + +TODO user-choosen keys vs keys chosen automatically? +Purity in face of dynamic tasks (no conflicts, guaranteed). + +TODO Does Nix constitute a different way to be be monadic? +Purity of keys, as mentioned. +Dynamic tasks/keys vs dynamic dependencies. +(Not sure yet.) + +## Lazy evaluation + +We clearly have thunks that produce thunks, but less clearly functions that produce functions. + +Do we have open terms? + +Do we hve thunks vs expressions distinction? +c.f. John Shutt's modern fexprs, when the syntax can "leak". + +## Machine models + +TODO +Derivations as store objects via drv files makes Nix a "Von Neumann" archicture. +Can also imagine a "Harvard" archicture where derivations are stored separately? +Can we in general imagine N heaps for N different sorts of objects? + +TODO +Also, leaning on the notion of "builtin builders" more, having multiple different sorts of user-defined builders too. +The builder is a black box as far as the Nix model is concerned. diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index e91f0f6e4..458c1e2b8 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,5 +1,27 @@ +# Nix Store + A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. There are many types of stores, but all of them at least respect this model. Some however offer additional functionality. + +## A Rosetta stone for the Nix store. + +The design of Nix is comparable to other build systems, even programming languages in general. +Here is a rough [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology. +If you are familiar with one of these columns, this might help the following sections make more sense. + +generic build system | Nix | Bazel | Build Systems à la Carte | lazy programming language +-- | -- | -- | -- | -- +data (build input, build result) | component | file (source, target) | value | value +build plan | derivation graph | action graph | `Tasks` | thunk +build step | derivation | rule | `Task` | thunk +build instructions | builder | (depends on action type) | `Task` | function +build | build | build | `Build` applied to arguments | evaluation +persistence layer | store | file system | `Store` | heap + +(n.b. Bazel terms gotten from https://docs.bazel.build/versions/main/glossary.html.) + +Plenty more could be said elaborating these comparisons. +We will save that for the end of this chapter, in the [Related Work](./related-work.md) section. From 0eae4bfad1bfa94347252b51dec8c87fa73644fb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Apr 2022 14:39:35 +0200 Subject: [PATCH 023/123] reword overview with clear terminology trying to capture alternative terms in one go here, mirroring everyday use: derivation - build plan realise - execute build there will be more of that sort. --- doc/manual/src/design/overview.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 9329f2ad7..6a5555dff 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -2,14 +2,22 @@ Nix consists of layers that operate fairly independently. -At the top is the *command line interface*. +At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the Nix *expression language*, in which packages and configurations are written. +Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +It is used to compose expressions which ultimately evaluate to *derivations* — self-contained *build plans* to derive new data from referenced input data. -The command line and expression language are what users interact with most. +::: {.note} +The Nix language itself does not have a notion of *packages* or *configurations*. +As far as we are concerned here, the result of a derivation is just data. +In practice this amounts to a set of files in a file system. +::: -Below that is the *store layer*, Nix' machinery to represent tracked files, dependencies, and fully elaborated build plans. -It is also used for executing those build plans. -The store layer may not be as visible, but this is the heart of Nix. +The command line and Nix language are what users interact with most. + +Underlying everything is the *Nix store*, a mechanism to keep track of derivations, data, and references between them. +It can also *realise derivations*, that is, *execute build instructions* to produce new data. +It uses the file system as a persistence layer, and a database to keep track of references. + +This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. -This chapter describes Nix starting with that bottom store layer, then working its way up until it reaches the more user-facing interfaces described in the rest of the manual." From 327ccd3b0787f7803a51aa5124cf0cbab694523c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Apr 2022 15:06:05 +0200 Subject: [PATCH 024/123] only use generic build system terminology we will use a translation table to introduce nix-specific terms --- doc/manual/src/design/overview.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 6a5555dff..c60121a01 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,18 +5,18 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to *derivations* — self-contained *build plans* to derive new data from referenced input data. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the result of a derivation is just data. +As far as we are concerned here, the inputs and results of a derivation are just data. In practice this amounts to a set of files in a file system. ::: The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of derivations, data, and references between them. -It can also *realise derivations*, that is, *execute build instructions* to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. +It can also execute *build instructions* captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. From 804e8bd7473516b087f39161bd34730e26480517 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:43:19 +0200 Subject: [PATCH 025/123] indicate sequence with "then" Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index c60121a01..20a3fe653 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -19,5 +19,5 @@ Underlying everything is the *Nix store*, a mechanism to keep track of build pla It can also execute *build instructions* captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. -This chapter describes Nix starting at the bottom with the store layer, working its way up to the user-facing components described in the rest of the manual. +This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 23ee0b24f7c0fd43ccf6f99b87941df4cb071689 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:44:00 +0200 Subject: [PATCH 026/123] correctly use comma for nesting Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 20a3fe653..b1adb143b 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -16,7 +16,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions* captured in the build plans, to produce new data. +It can also execute *build instructions*, captured in the build plans, to produce new data. It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 51e6bed25e073afac5a7b5e11033769a615d0473 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Apr 2022 10:54:43 +0200 Subject: [PATCH 027/123] do not mention implementation details Co-authored-by: John Ericson --- doc/manual/src/design/overview.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index b1adb143b..0cead62ab 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -17,7 +17,6 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build plans, to produce new data. -It uses the file system as a persistence layer, and a database to keep track of references. This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 89a7c956ffadbc9c17849051f3c91a1225dcfce2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Apr 2022 14:05:50 -0400 Subject: [PATCH 028/123] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- doc/manual/src/design/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 0cead62ab..15ddb562d 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -5,7 +5,7 @@ Nix consists of layers that operate fairly independently. At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. From b387d809437b0c787691b814bf65e1647fd18009 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 24 Apr 2022 21:13:45 +0200 Subject: [PATCH 029/123] remove sentence for chapter transition idea: sections could be read in different orders by linking them in different ways (e.g. depth-first or breadth-first). adding hard-coded transitions makes that confusing. --- doc/manual/src/design/overview.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md index 15ddb562d..b5f8b6aad 100644 --- a/doc/manual/src/design/overview.md +++ b/doc/manual/src/design/overview.md @@ -18,5 +18,3 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build plans, to produce new data. -This chapter describes Nix starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. - From 34ea74c9ec21641399a96386094b021d7a2b30f4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Apr 2022 11:46:15 +0200 Subject: [PATCH 030/123] reword introductory section there should be a meta section for each chapter to give motivation of the presented structure. the structure itself is visible from the table of contents. --- doc/manual/src/design/design.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index 7d4211764..f5135bc9a 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -1,5 +1,6 @@ # Design and Data Model Most of the manual is about how to use Nix. -This chapter is about what Nix actually is. -The hope is that it can serve as a reference for key concepts, and also shed light on why things are the way they are. +This chapter is about the technical principles behind Nix. + +It describes each architectural layer and its components in its own section, starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. From 75981263912805a77890bcdb63c4aa3bbb0d8e09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 21:17:44 +0200 Subject: [PATCH 031/123] remove separate meta-section, add architecture diagram the diagram is a first approximation and only covers that same section. of course there is much more going on, and other features should at some point also be illustrated. we also have to think about presentation format and technicalities behind it. the manual has to render to `man`, but we may want something more refined for web view. --- doc/manual/src/SUMMARY.md.in | 3 +-- doc/manual/src/design/design.md | 39 ++++++++++++++++++++++++++-- doc/manual/src/design/overview.md | 20 -------------- doc/manual/src/design/store/store.md | 2 +- 4 files changed, 39 insertions(+), 25 deletions(-) delete mode 100644 doc/manual/src/design/overview.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9cde44a0d..ec457126a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,8 +16,7 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Design and Data Model](design/design.md) - - [Overview](design/overview.md) - - [The Store Layer](design/store/store.md) + - [Store](design/store/store.md) - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) - [Nix Archives](design/store/nar.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index f5135bc9a..960a68f41 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -1,6 +1,41 @@ # Design and Data Model -Most of the manual is about how to use Nix. This chapter is about the technical principles behind Nix. -It describes each architectural layer and its components in its own section, starting at the bottom with the store layer, then working its way up to the user-facing components described in the rest of the manual. +## Architecture + +Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). + +``` + [ commmand line interface ] + | + | evaluates + V + [ configuration language ] + | + | evaluates to + | + reference V build +[ build inputs ] --> [ build plans ] --> [ build results ] + \ | / + \ | persisted to / + \ V / + [ store ] +``` + +At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. + +Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. + +::: {.note} +The Nix language itself does not have a notion of *packages* or *configurations*. +As far as we are concerned here, the inputs and results of a derivation are just data. +In practice this amounts to a set of files in a file system. +::: + +The command line and Nix language are what users interact with most. + +Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. +It can also execute *build instructions*, captured in the build plans, to produce new data. + diff --git a/doc/manual/src/design/overview.md b/doc/manual/src/design/overview.md deleted file mode 100644 index b5f8b6aad..000000000 --- a/doc/manual/src/design/overview.md +++ /dev/null @@ -1,20 +0,0 @@ -# Overview - -Nix consists of layers that operate fairly independently. - -At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. - -Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. -It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. - -::: {.note} -The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a derivation are just data. -In practice this amounts to a set of files in a file system. -::: - -The command line and Nix language are what users interact with most. - -Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build plans, to produce new data. - diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/design/store/store.md index 458c1e2b8..d4add52f5 100644 --- a/doc/manual/src/design/store/store.md +++ b/doc/manual/src/design/store/store.md @@ -1,4 +1,4 @@ -# Nix Store +# Store A Nix store is a collection of *store objects* referred to by *store paths*. Every store also has a "store directory path", which is a path prefix used for various purposes. From d30033759a07a0d5df7aac4c22bfa274b74baf0e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 21:21:46 +0200 Subject: [PATCH 032/123] address Nix language consistently as configuration language --- doc/manual/src/design/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/design/design.md index 960a68f41..16c51533b 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/design/design.md @@ -25,7 +25,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the *Nix language*, a [purely functional programming](https://en.m.wikipedia.org/wiki/Purely_functional_programming) language. +Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. ::: {.note} From 39f01176a713bad4e090e19c8a25c2447056640d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 27 Apr 2022 23:25:33 +0200 Subject: [PATCH 033/123] design -> architecture, add motivation following ideas found in Architecture of Gazelle[1] [1]: https://github.com/bazelbuild/bazel-gazelle/blob/56d35f8db086bb65ef876f96f7baa7b71516daf8/Design.rst --- doc/manual/src/SUMMARY.md.in | 2 +- .../src/{design/design.md => architecture/architecture.md} | 7 ++++--- doc/manual/src/{design => architecture}/store/building.md | 0 doc/manual/src/{design => architecture}/store/drvs/ca.md | 0 doc/manual/src/{design => architecture}/store/drvs/drvs.md | 0 doc/manual/src/{design => architecture}/store/drvs/ia.md | 0 .../src/{design => architecture}/store/input-addressing.md | 0 doc/manual/src/{design => architecture}/store/nar.md | 0 doc/manual/src/{design => architecture}/store/object-ca.md | 0 doc/manual/src/{design => architecture}/store/objects.md | 0 doc/manual/src/{design => architecture}/store/paths.md | 0 .../src/{design => architecture}/store/related-work.md | 0 .../src/{design => architecture}/store/relocatability.md | 0 doc/manual/src/{design => architecture}/store/store.md | 0 14 files changed, 5 insertions(+), 4 deletions(-) rename doc/manual/src/{design/design.md => architecture/architecture.md} (88%) rename doc/manual/src/{design => architecture}/store/building.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/ca.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/drvs.md (100%) rename doc/manual/src/{design => architecture}/store/drvs/ia.md (100%) rename doc/manual/src/{design => architecture}/store/input-addressing.md (100%) rename doc/manual/src/{design => architecture}/store/nar.md (100%) rename doc/manual/src/{design => architecture}/store/object-ca.md (100%) rename doc/manual/src/{design => architecture}/store/objects.md (100%) rename doc/manual/src/{design => architecture}/store/paths.md (100%) rename doc/manual/src/{design => architecture}/store/related-work.md (100%) rename doc/manual/src/{design => architecture}/store/relocatability.md (100%) rename doc/manual/src/{design => architecture}/store/store.md (100%) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index ec457126a..c029e30bf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,7 +15,7 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Design and Data Model](design/design.md) +- [Architecture](architecture/architecture.md) - [Store](design/store/store.md) - [Store Objects](design/store/objects.md) - [Store Paths](design/store/paths.md) diff --git a/doc/manual/src/design/design.md b/doc/manual/src/architecture/architecture.md similarity index 88% rename from doc/manual/src/design/design.md rename to doc/manual/src/architecture/architecture.md index 16c51533b..b17eacd2e 100644 --- a/doc/manual/src/design/design.md +++ b/doc/manual/src/architecture/architecture.md @@ -1,8 +1,9 @@ -# Design and Data Model +# Architecture -This chapter is about the technical principles behind Nix. +This chapter describes how Nix works. +It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools. -## Architecture +## Overview Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). diff --git a/doc/manual/src/design/store/building.md b/doc/manual/src/architecture/store/building.md similarity index 100% rename from doc/manual/src/design/store/building.md rename to doc/manual/src/architecture/store/building.md diff --git a/doc/manual/src/design/store/drvs/ca.md b/doc/manual/src/architecture/store/drvs/ca.md similarity index 100% rename from doc/manual/src/design/store/drvs/ca.md rename to doc/manual/src/architecture/store/drvs/ca.md diff --git a/doc/manual/src/design/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md similarity index 100% rename from doc/manual/src/design/store/drvs/drvs.md rename to doc/manual/src/architecture/store/drvs/drvs.md diff --git a/doc/manual/src/design/store/drvs/ia.md b/doc/manual/src/architecture/store/drvs/ia.md similarity index 100% rename from doc/manual/src/design/store/drvs/ia.md rename to doc/manual/src/architecture/store/drvs/ia.md diff --git a/doc/manual/src/design/store/input-addressing.md b/doc/manual/src/architecture/store/input-addressing.md similarity index 100% rename from doc/manual/src/design/store/input-addressing.md rename to doc/manual/src/architecture/store/input-addressing.md diff --git a/doc/manual/src/design/store/nar.md b/doc/manual/src/architecture/store/nar.md similarity index 100% rename from doc/manual/src/design/store/nar.md rename to doc/manual/src/architecture/store/nar.md diff --git a/doc/manual/src/design/store/object-ca.md b/doc/manual/src/architecture/store/object-ca.md similarity index 100% rename from doc/manual/src/design/store/object-ca.md rename to doc/manual/src/architecture/store/object-ca.md diff --git a/doc/manual/src/design/store/objects.md b/doc/manual/src/architecture/store/objects.md similarity index 100% rename from doc/manual/src/design/store/objects.md rename to doc/manual/src/architecture/store/objects.md diff --git a/doc/manual/src/design/store/paths.md b/doc/manual/src/architecture/store/paths.md similarity index 100% rename from doc/manual/src/design/store/paths.md rename to doc/manual/src/architecture/store/paths.md diff --git a/doc/manual/src/design/store/related-work.md b/doc/manual/src/architecture/store/related-work.md similarity index 100% rename from doc/manual/src/design/store/related-work.md rename to doc/manual/src/architecture/store/related-work.md diff --git a/doc/manual/src/design/store/relocatability.md b/doc/manual/src/architecture/store/relocatability.md similarity index 100% rename from doc/manual/src/design/store/relocatability.md rename to doc/manual/src/architecture/store/relocatability.md diff --git a/doc/manual/src/design/store/store.md b/doc/manual/src/architecture/store/store.md similarity index 100% rename from doc/manual/src/design/store/store.md rename to doc/manual/src/architecture/store/store.md From c8c1b705ad5cdf115997cef387d83c4b04fe0660 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 24 Apr 2022 21:34:33 +0200 Subject: [PATCH 034/123] reword section on Nix store --- doc/manual/src/architecture/store/store.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d4add52f5..b52fe8255 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,10 +1,14 @@ # Store -A Nix store is a collection of *store objects* referred to by *store paths*. -Every store also has a "store directory path", which is a path prefix used for various purposes. +A Nix store is a collection of [store objects](objects.md) and associated operations. -There are many types of stores, but all of them at least respect this model. -Some however offer additional functionality. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. + +Store objects are accessible in a file system through [store paths](paths.md). +Every store has a *store directory*, which contains that store’s objects and is a prefix of their store paths. +It defaults to `/nix/store`, but is in principle arbitrary. + +A Nix store can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. ## A Rosetta stone for the Nix store. From 7b5c00f67f729bb3c0d026f3f4d14e727c4cf420 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 01:15:27 +0200 Subject: [PATCH 035/123] add concrete store examples, reword note on file system --- doc/manual/src/architecture/store/store.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b52fe8255..688c7e1a2 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,11 +4,18 @@ A Nix store is a collection of [store objects](objects.md) and associated operat These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -Store objects are accessible in a file system through [store paths](paths.md). -Every store has a *store directory*, which contains that store’s objects and is a prefix of their store paths. -It defaults to `/nix/store`, but is in principle arbitrary. +A Nix store allows to add and retrieve store objects. +It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. +It also keeps track of *references* between data and can therefore garbage-collect unused store objects. -A Nix store can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. +There exist different types of stores, which all follow this model. +Examples: +- store on the local file system +- remote store accessible via SSH +- binary cache store accessible via HTTP + +Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). +The store directory defaults to `/nix/store`, but is in principle arbitrary. ## A Rosetta stone for the Nix store. From 070c85499b370bf886db1a87c04ed9c319bb6323 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 09:44:46 +0200 Subject: [PATCH 036/123] fix grammar Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 688c7e1a2..96796f26c 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,7 +4,7 @@ A Nix store is a collection of [store objects](objects.md) and associated operat These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -A Nix store allows to add and retrieve store objects. +A Nix store allows adding and retrieving store objects. It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. From 5f96a0b4e830bef59b5986c916727d07b244c22c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:18:03 +0200 Subject: [PATCH 037/123] associated operations are not collected Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 96796f26c..624439f73 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,6 +1,6 @@ # Store -A Nix store is a collection of [store objects](objects.md) and associated operations. +A Nix store is a collection of [store objects](objects.md) with associated operations. These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. From 610ddf44aa86924834bf2ac3735c7cb75a6a1f1c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 25 Apr 2022 13:25:43 +0200 Subject: [PATCH 038/123] reword introduction to rosetta stone, add links attempt to explain used and documented terminology, as well as how the declarative programming paradigm relates to building software. in the future one could highlight encouraged terms to shape future material into higher consistency. --- doc/manual/src/architecture/architecture.md | 2 +- doc/manual/src/architecture/store/store.md | 29 ++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index b17eacd2e..836b40c91 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build steps*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 624439f73..af0609d4b 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -17,22 +17,21 @@ Examples: Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). The store directory defaults to `/nix/store`, but is in principle arbitrary. -## A Rosetta stone for the Nix store. +## A [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology -The design of Nix is comparable to other build systems, even programming languages in general. -Here is a rough [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology. -If you are familiar with one of these columns, this might help the following sections make more sense. +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. -generic build system | Nix | Bazel | Build Systems à la Carte | lazy programming language +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- -data (build input, build result) | component | file (source, target) | value | value -build plan | derivation graph | action graph | `Tasks` | thunk -build step | derivation | rule | `Task` | thunk -build instructions | builder | (depends on action type) | `Task` | function -build | build | build | `Build` applied to arguments | evaluation -persistence layer | store | file system | `Store` | heap +data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value +build plan | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build graph | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) +build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build | realisation | build | application of `Build` | evaluation +persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap -(n.b. Bazel terms gotten from https://docs.bazel.build/versions/main/glossary.html.) - -Plenty more could be said elaborating these comparisons. -We will save that for the end of this chapter, in the [Related Work](./related-work.md) section. +All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software +Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf) (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) (2018). From ca5ebf63827f82ae63796d775070b2f9cf828625 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:34:53 +0200 Subject: [PATCH 039/123] revert build plan/step distinction, reorder rows --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index af0609d4b..f4f5a3e16 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -27,9 +27,9 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build plan | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) -build graph | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | realisation | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 40efe5b30b24e75b26017eb68c62acfc6b7ebf93 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:36:36 +0200 Subject: [PATCH 040/123] build instrcution: Task -> function --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f4f5a3e16..e851444c9 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -27,7 +27,7 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | `Task` | function +build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | realisation | build | application of `Build` | evaluation From a145007a577fda8a8f3b65b66f4c359410b4af82 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 23:48:59 +0200 Subject: [PATCH 041/123] component -> store object, realisation -> build --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index e851444c9..74d3d7f84 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,11 +26,11 @@ The following translation table points out similarities and equivalent terms, to generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language -- | -- | -- | -- | -- -data (build input, build result) | component | [artifact](https://bazel.build/reference/glossary#artifact) | value | value +data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) -build | realisation | build | application of `Build` | evaluation +build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software From e5e48593c85d052585ff8a275c54823caf575a9d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:16:27 +0200 Subject: [PATCH 042/123] move git comparison to related work it should be pulled out of the branch before we go for merging --- doc/manual/src/architecture/store/objects.md | 23 ------------------ .../src/architecture/store/related-work.md | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index e4f49a170..caa00d862 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -31,29 +31,6 @@ In particular, every file system object falls into these three cases: A bare file or symlink as the "root" file system object is allowed. -### Comparison with Git - -This is close to Git's model, but with one crucial difference: -Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. - - data GitObject - = Blob ByteString - | Tree (Map FileName (Persission, FSO)) - - data Persission - = Directory -- IFF paired with tree - -- Iff paired with blob, one of: - | RegFile - | ExecutableFile - | Symlink - -So long as the root object is a directory, the representations are isomorphic. -There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. - -However, if the root object is a blob, there is loss of fidelity. -Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. - -Git's model matches Unix tradition, but Nix's model is more natural. ## References diff --git a/doc/manual/src/architecture/store/related-work.md b/doc/manual/src/architecture/store/related-work.md index b64b41988..92b7d480e 100644 --- a/doc/manual/src/architecture/store/related-work.md +++ b/doc/manual/src/architecture/store/related-work.md @@ -25,6 +25,30 @@ Do we have open terms? Do we hve thunks vs expressions distinction? c.f. John Shutt's modern fexprs, when the syntax can "leak". +## Comparison with Git file system model + +This is close to Git's model, but with one crucial difference: +Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. + + data GitObject + = Blob ByteString + | Tree (Map FileName (Persission, FSO)) + + data Persission + = Directory -- IFF paired with tree + -- Iff paired with blob, one of: + | RegFile + | ExecutableFile + | Symlink + +So long as the root object is a directory, the representations are isomorphic. +There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. + +However, if the root object is a blob, there is loss of fidelity. +Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. + +Git's model matches Unix tradition, but Nix's model is more natural. + ## Machine models TODO From 90fc5b41a83575c0bd008868f989b6e1497ce3a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:18:23 +0200 Subject: [PATCH 043/123] reword file system objects - use singular for the "class" - more consistency in type definition - minor fixes in wording --- doc/manual/src/architecture/store/objects.md | 40 ++++++++------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index caa00d862..872a209c3 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -1,40 +1,32 @@ -# Store Objects +# Store Object -Data in Nix is chiefly organized into *store objects*. +Nix organizes the data it manages into *store objects*. A store object is the pair of - - A (root) file system object - - A set of references to store objects + - a [file system object](#file-system-object) + - a set of [references](#reference) to store objects. -## File system objects +## File system object {#file-system-object} -The Nix store uses a simple filesystem model. +The Nix store uses a simple file system model. data FileSystemObject - = Regular Executable ByteString - | Directory (Map FileName FSO) - | SymLink ByteString - - data Executable - = Executable - | NonExecutable - -In particular, every file system object falls into these three cases: + = File Executable Data + | Directory (Map FileName FileSystemObject) + | SymLink Path +Every file system object is one of the following: - File: an executable flag, and arbitrary data + - Directory: mapping of names to child file system objects + - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - - Directory: mapping of names to child file system objects. + In particular, symlinks pointing outside of their own root file system object, or to a store object without a matching reference, are allowed, but might not function as intended. - - Symlink: may point anywhere. +A bare file or symlink can be a root file system object. - In particular, symlinks that do not point within the containing root file system object or that of another store object referenced by the containing store object are allowed, but might not function as intended. +## Reference {#reference} -A bare file or symlink as the "root" file system object is allowed. - - -## References - -Store objects can refer to both other store objects and themselves. +A store object can refer to both other store objects and itself. Self-reference may seem pointless, but tracking them is in fact useful. We can best explain why later after more concepts have been established. From fb2ec7e4ec758e17f247218625d23ceba685e8a5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Apr 2022 10:54:51 +0200 Subject: [PATCH 044/123] reword section on references use file Contents instead of Data, as that flows more naturally in the prose. simplify explanation of the idea behind scanning for store paths remove references to unfinished sections. --- doc/manual/src/architecture/store/objects.md | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 872a209c3..d7a27b528 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -11,12 +11,12 @@ A store object is the pair of The Nix store uses a simple file system model. data FileSystemObject - = File Executable Data + = File Executable Contents | Directory (Map FileName FileSystemObject) | SymLink Path Every file system object is one of the following: - - File: an executable flag, and arbitrary data + - File: an executable flag, and arbitrary data for contents - Directory: mapping of names to child file system objects - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. @@ -26,17 +26,16 @@ A bare file or symlink can be a root file system object. ## Reference {#reference} -A store object can refer to both other store objects and itself. +A store object can refer to other store objects or itself. -Self-reference may seem pointless, but tracking them is in fact useful. -We can best explain why later after more concepts have been established. +Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. -References are normally calculated so as to to record the presence of textual references in store object's file systems obejcts. -This process will be described precisely in the section on [building](./building.md), once more concepts are explained, as building is the primary path new store objects with non-trivial references are created. +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness: +Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. -However, scanning for references is not mandatory. -Store objects are allowed to have official references that *don't* correspond to store paths contained in their contents, -and they are also allowed to *not* have references that *do* correspond to store paths contained in their store. -Taken together, this means there is no actual rule relating the store paths contained in the contents to the store paths deemed references. +However, having references match store paths in files is not enforced by the data model: +Store objects could have excess or incomplete references with respect to store paths found in their file contents. + +Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. -This is why it's its necessary for correctness, and not just performance, that Nix remember the references of each store object, rather than try to recompute them on the fly by scanning their contents. From 5fda995491f1f6b32b46b45c2b47f3e329c997f6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 00:24:23 +0200 Subject: [PATCH 045/123] formalize file system objects convention: describe every data type in prose, and illustrate with a class diagram, and a textual representation of an abstract data type. right now we save ourselves the trouble of doing class diagrams, we can add them later. but they are important. --- doc/manual/src/architecture/store/objects.md | 25 ++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index d7a27b528..7f341c87c 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -6,31 +6,42 @@ A store object is the pair of - a [file system object](#file-system-object) - a set of [references](#reference) to store objects. +We call a store object's outermost file system object the *root*. + +```haskell +data StoreOject = StoreObject { + root :: FileSystemObject +, references :: Set StoreObject +} +``` + ## File system object {#file-system-object} The Nix store uses a simple file system model. - data FileSystemObject - = File Executable Contents - | Directory (Map FileName FileSystemObject) - | SymLink Path - Every file system object is one of the following: - File: an executable flag, and arbitrary data for contents - Directory: mapping of names to child file system objects - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - In particular, symlinks pointing outside of their own root file system object, or to a store object without a matching reference, are allowed, but might not function as intended. +```haskell +data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } +``` A bare file or symlink can be a root file system object. +Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. + ## Reference {#reference} A store object can refer to other store objects or itself. Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness: +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. However, having references match store paths in files is not enforced by the data model: From 07d490fd895b09144684dd8b389ef02f0f1256c4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 00:58:40 +0200 Subject: [PATCH 046/123] stores can also delete objects --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 74d3d7f84..cbb5a4169 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -4,7 +4,7 @@ A Nix store is a collection of [store objects](objects.md) with associated opera These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. -A Nix store allows adding and retrieving store objects. +A Nix store allows adding, retrieving, and deleting store objects. It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. From e90586c0a40cf59df9d39407dcc49d5944a8b853 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 01:15:33 +0200 Subject: [PATCH 047/123] add motivation for references --- doc/manual/src/architecture/store/objects.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 7f341c87c..30683d22d 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -37,9 +37,20 @@ Symlinks pointing outside of their own root, or to a store object without a matc ## Reference {#reference} -A store object can refer to other store objects or itself. +A store object can reference other store objects. -Nix collects these references by scanning file contents for [store paths](./paths.md) when a new store object is created. +Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. + +Building, copying and deleting store objects must be done in a way that obeys this property: + +- We can only safely delete unreferenced objects. + +- When copying, to maintain correctness, either the result must be "revealed" atomically to the destination store, or objects must be copied in reference-dependency order. + +- Newly built store objects must only refer to store objects in the closure of the build inputs. + This ensures the purity of the build. + +### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. From b5ca3d12b6f3414302363c8ae362334c020448c7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 29 Apr 2022 02:10:24 +0200 Subject: [PATCH 048/123] reword details on keeping closure property --- doc/manual/src/architecture/store/objects.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index 30683d22d..f7587e112 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -43,13 +43,13 @@ Nix stores have the *closure property*: for each store object in the store, all Building, copying and deleting store objects must be done in a way that obeys this property: +- Build results must only refer to store objects in the closure of the build inputs. + +- Store objects being copied must refer to objects already in the destination store. + Recursive copying must either proceed in dependency order or be atomic. + - We can only safely delete unreferenced objects. -- When copying, to maintain correctness, either the result must be "revealed" atomically to the destination store, or objects must be copied in reference-dependency order. - -- Newly built store objects must only refer to store objects in the closure of the build inputs. - This ensures the purity of the build. - ### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. From b01bb65d3023281c601024578a9c19f32e7ad011 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Apr 2022 21:41:04 -0400 Subject: [PATCH 049/123] Fix rel path in doc --- doc/manual/src/architecture/store/building.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/building.md b/doc/manual/src/architecture/store/building.md index f4f2649a3..d43904126 100644 --- a/doc/manual/src/architecture/store/building.md +++ b/doc/manual/src/architecture/store/building.md @@ -2,7 +2,7 @@ ## Scanning for references -Before in the section on [store objects](../objects.md), we talked abstractly about scanning for references. +Before in the section on [store objects](./objects.md), we talked abstractly about scanning for references. Now we can make this concrete. After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. From 3d8f2f5cc1faa4d42138267f21d2996da99c49a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 28 Apr 2022 17:53:10 -0400 Subject: [PATCH 050/123] Fix manual TOC links --- doc/manual/src/SUMMARY.md.in | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c029e30bf..80a228741 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -16,17 +16,17 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - - [Store](design/store/store.md) - - [Store Objects](design/store/objects.md) - - [Store Paths](design/store/paths.md) - - [Nix Archives](design/store/nar.md) - - [Content-Addressing Store Objects](design/store/object-ca.md) - - [Derivations](design/store/drvs/drvs.md) - - [Input-Addressing](design/store/drvs/ia.md) - - [Content-Addressing (Experimental)](design/store/drvs/ca.md) - - [Building](design/store/building.md) - - [Advanced Topic: Store object relocatability](design/store/relocatability.md) - - [Advanced Topic: Related work](design/store/related-work.md) + - [Store](architecture/store/store.md) + - [Store Objects](architecture/store/objects.md) + - [Store Paths](architecture/store/paths.md) + - [Nix Archives](architecture/store/nar.md) + - [Content-Addressing Store Objects](architecture/store/object-ca.md) + - [Derivations](architecture/store/drvs/drvs.md) + - [Input-Addressing](architecture/store/drvs/ia.md) + - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) + - [Building](architecture/store/building.md) + - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) + - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) From 1ba6d8fb1d59f4d40825141f75098dfbfd3b0153 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:53:37 +0200 Subject: [PATCH 051/123] remove incomplete section: building --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/building.md | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 doc/manual/src/architecture/store/building.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 80a228741..596843057 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - - [Building](architecture/store/building.md) - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) diff --git a/doc/manual/src/architecture/store/building.md b/doc/manual/src/architecture/store/building.md deleted file mode 100644 index d43904126..000000000 --- a/doc/manual/src/architecture/store/building.md +++ /dev/null @@ -1,15 +0,0 @@ -# Building - -## Scanning for references - -Before in the section on [store objects](./objects.md), we talked abstractly about scanning for references. -Now we can make this concrete. - -After the derivation's command is run, Nix needs to process the "raw" output directories to turn them into legit store objects. -There is a few steps of this, but let's start with the simple case of one input-addressed output first. - -\[Overview of things that need to happen.] - -For example, if Nix thinks `/nix/store/asdfasdfasdf-foo` and `/nix/store/qwerqwerqwer-bar` are paths the data might plausibly reference, Nix will scan all the contents of all files recursively for the "hash parts" `asdfasdfasdf`` and `qwerqwerqwer`. - -\[Explain why whitelist.] From 96876b1eaeae95e3f2f338c6b3495040fc843c57 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:54:06 +0200 Subject: [PATCH 052/123] remove incomplete section: related work --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/related-work.md | 61 ------------------- 2 files changed, 62 deletions(-) delete mode 100644 doc/manual/src/architecture/store/related-work.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 596843057..ee7f5d7ac 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -25,7 +25,6 @@ - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - - [Advanced Topic: Related work](architecture/store/related-work.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/related-work.md b/doc/manual/src/architecture/store/related-work.md deleted file mode 100644 index 92b7d480e..000000000 --- a/doc/manual/src/architecture/store/related-work.md +++ /dev/null @@ -1,61 +0,0 @@ -# Advanced Topic: Related Work - -## Bazel - -TODO skylark and layering. - -TODO being monadic, if RFC 92. - -## Build Systems à la Carte - -TODO user-choosen keys vs keys chosen automatically? -Purity in face of dynamic tasks (no conflicts, guaranteed). - -TODO Does Nix constitute a different way to be be monadic? -Purity of keys, as mentioned. -Dynamic tasks/keys vs dynamic dependencies. -(Not sure yet.) - -## Lazy evaluation - -We clearly have thunks that produce thunks, but less clearly functions that produce functions. - -Do we have open terms? - -Do we hve thunks vs expressions distinction? -c.f. John Shutt's modern fexprs, when the syntax can "leak". - -## Comparison with Git file system model - -This is close to Git's model, but with one crucial difference: -Git puts the "permission" info within the directory map's values instead of making it part of the file (blob, in it's parlance) object. - - data GitObject - = Blob ByteString - | Tree (Map FileName (Persission, FSO)) - - data Persission - = Directory -- IFF paired with tree - -- Iff paired with blob, one of: - | RegFile - | ExecutableFile - | Symlink - -So long as the root object is a directory, the representations are isomorphic. -There is no "wiggle room" the git way since whenever the permission info wouldn't matter (e.g. the child object being mapped to is a directory), the permission info must be a sentinel value. - -However, if the root object is a blob, there is loss of fidelity. -Since the permission info is used to distinguish executable files, non-executable files, and symlinks, but there isn't a "parent" directory of the root to contain that info, these 3 cases cannot be distinguished. - -Git's model matches Unix tradition, but Nix's model is more natural. - -## Machine models - -TODO -Derivations as store objects via drv files makes Nix a "Von Neumann" archicture. -Can also imagine a "Harvard" archicture where derivations are stored separately? -Can we in general imagine N heaps for N different sorts of objects? - -TODO -Also, leaning on the notion of "builtin builders" more, having multiple different sorts of user-defined builders too. -The builder is a black box as far as the Nix model is concerned. From 7cec9ee3612c7255e6dc4fb94fc238f3d81ced03 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:54:24 +0200 Subject: [PATCH 053/123] remove incomplete section: relocatability --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/relocatability.md | 15 --------------- 2 files changed, 16 deletions(-) delete mode 100644 doc/manual/src/architecture/store/relocatability.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index ee7f5d7ac..c35505bbe 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,7 +24,6 @@ - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - - [Advanced Topic: Store object relocatability](architecture/store/relocatability.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/relocatability.md b/doc/manual/src/architecture/store/relocatability.md deleted file mode 100644 index c7f869135..000000000 --- a/doc/manual/src/architecture/store/relocatability.md +++ /dev/null @@ -1,15 +0,0 @@ -## Advanced Topic: Store object relocation - -Now that we know the fundamentals of the design of the Nix store, let's explore one consequence of that design: the question when it is permissible to relocate a store object to a store with a different mount point. - -Recall from the section on [store paths](./store-paths.md) that concrete store paths look like `/-`. - -~~The two final restrictions of the previous section yield an alternative view of the same information.~~ -Rather than associating store dirs with the references, we can say a store object itself has a store dir if and only if it has at least one reference. - -This corresponds to the observation that a store object with references, i.e. with a store directory under this interpretation, is confined to stores sharing that same store directory, but a store object without any references, i.e. thus without a store directory, can exist in any store. - -Lastly, this illustrates the purpose of tracking self references. -Store objects without self-references or other references are relocatable, while store paths with self-references aren't. -This is used to tell apart e.g. source code which can be stored anywhere, and pesky non-reloctable executables which assume they are installed to a certain path. -\[The default method of calculating references by scanning for store paths handles these two example cases surprisingly well.\] From b18852eb3fc1d62ec307d3b2f102c6bd8574d069 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:55:11 +0200 Subject: [PATCH 054/123] remove incomplete section: content-addressed objects --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/object-ca.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 doc/manual/src/architecture/store/object-ca.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c35505bbe..11a974b47 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,7 +20,6 @@ - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - [Nix Archives](architecture/store/nar.md) - - [Content-Addressing Store Objects](architecture/store/object-ca.md) - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) diff --git a/doc/manual/src/architecture/store/object-ca.md b/doc/manual/src/architecture/store/object-ca.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/object-ca.md +++ /dev/null @@ -1 +0,0 @@ -TODO From 3bd125ebbeae38c9efb5790e13265b7534e66c45 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:55:52 +0200 Subject: [PATCH 055/123] remove incomplete section: nix archives --- doc/manual/src/SUMMARY.md.in | 1 - doc/manual/src/architecture/store/nar.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 doc/manual/src/architecture/store/nar.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 11a974b47..38277aaf2 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,7 +19,6 @@ - [Store](architecture/store/store.md) - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - - [Nix Archives](architecture/store/nar.md) - [Derivations](architecture/store/drvs/drvs.md) - [Input-Addressing](architecture/store/drvs/ia.md) - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) diff --git a/doc/manual/src/architecture/store/nar.md b/doc/manual/src/architecture/store/nar.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/nar.md +++ /dev/null @@ -1 +0,0 @@ -TODO From ad8c2ed7f0566a5fe1b4a3591240eb06804bd958 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 13:59:03 +0200 Subject: [PATCH 056/123] remove incomplete section: input/content-addressing --- doc/manual/src/SUMMARY.md.in | 2 -- doc/manual/src/architecture/store/drvs/ca.md | 0 doc/manual/src/architecture/store/drvs/ia.md | 0 doc/manual/src/architecture/store/input-addressing.md | 1 - 4 files changed, 3 deletions(-) delete mode 100644 doc/manual/src/architecture/store/drvs/ca.md delete mode 100644 doc/manual/src/architecture/store/drvs/ia.md delete mode 100644 doc/manual/src/architecture/store/input-addressing.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 38277aaf2..257880efe 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,8 +20,6 @@ - [Store Objects](architecture/store/objects.md) - [Store Paths](architecture/store/paths.md) - [Derivations](architecture/store/drvs/drvs.md) - - [Input-Addressing](architecture/store/drvs/ia.md) - - [Content-Addressing (Experimental)](architecture/store/drvs/ca.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/ca.md b/doc/manual/src/architecture/store/drvs/ca.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/manual/src/architecture/store/drvs/ia.md b/doc/manual/src/architecture/store/drvs/ia.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/doc/manual/src/architecture/store/input-addressing.md b/doc/manual/src/architecture/store/input-addressing.md deleted file mode 100644 index 1333ed77b..000000000 --- a/doc/manual/src/architecture/store/input-addressing.md +++ /dev/null @@ -1 +0,0 @@ -TODO From d3effd014b17bc957b9af8ed35b2f25a3b54e02c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 14:05:46 +0200 Subject: [PATCH 057/123] update architecture diagram --- doc/manual/src/architecture/architecture.md | 28 ++++++++++----------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 836b40c91..25b6a9616 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -8,20 +8,20 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). ``` - [ commmand line interface ] - | - | evaluates - V - [ configuration language ] - | - | evaluates to - | - reference V build -[ build inputs ] --> [ build plans ] --> [ build results ] - \ | / - \ | persisted to / - \ V / - [ store ] + [ commmand line interface ]--------+ + | | + | evaluates | manages + V | + [ configuration language ] | + | V ++-------------------------------|---------------------------------+ +| store | | +| | evaluates to | +| | | +| references V builds | +| [ build input ] --> [ build plan ] --> [ build result ] | +| | ++-----------------------------------------------------------------+ ``` At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. From 87523f01e3dc27d4cf57354a67346f8473e93cc5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:27:44 +0200 Subject: [PATCH 058/123] match grammatical case to arrow direction --- doc/manual/src/architecture/architecture.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 25b6a9616..a0aa713f3 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -18,8 +18,8 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | store | | | | evaluates to | | | | -| references V builds | -| [ build input ] --> [ build plan ] --> [ build result ] | +| referenced by V builds | +| [ build input ] ----> [ build plan ] ----> [ build result ] | | | +-----------------------------------------------------------------+ ``` From 902638c519fb46b0f0a3f7092ccb7cd2a622b206 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 3 May 2022 14:06:51 +0200 Subject: [PATCH 059/123] build step -> build rule "step" sounds atomic, while "rule" hints at internal structure, which in our case consists of mapping inputs to outputs using build instructions. --- doc/manual/src/architecture/architecture.md | 7 ++++--- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index a0aa713f3..81348c570 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build rules*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -37,6 +37,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build plans, to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build rules, data, and references between them. +It can also execute *build instructions*, captured in the build rules, to produce new data. +A series of build rules is a *build plan*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cbb5a4169..b03aa690e 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,10 +2,10 @@ A Nix store is a collection of [store objects](objects.md) with associated operations. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build plans. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build rules. A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build plans into build outputs. +It can perform builds, that is, transform build inputs using instructions from the build rules into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. There exist different types of stores, which all follow this model. @@ -28,7 +28,7 @@ generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [B -- | -- | -- | -- | -- data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build step | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build rule | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 2a8532fb61ba2b988448e93f5074590e40df7e04 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 May 2022 16:39:26 +0200 Subject: [PATCH 060/123] build rule -> build task closer to "build systems a la carte", satisfies all other complaints --- doc/manual/src/architecture/architecture.md | 10 +++++----- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 81348c570..407e8d0f8 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -19,7 +19,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | | evaluates to | | | | | referenced by V builds | -| [ build input ] ----> [ build plan ] ----> [ build result ] | +| [ build input ] --> [ build task ] --> [ build result ] | | | +-----------------------------------------------------------------+ ``` @@ -27,7 +27,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build rules*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build tasks*, used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -37,7 +37,7 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build rules, data, and references between them. -It can also execute *build instructions*, captured in the build rules, to produce new data. +Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. +It can also execute *build instructions*, captured in the build tasks, to produce new data. -A series of build rules is a *build plan*. +A series of build tasks is a *build plan*. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b03aa690e..cc6698a40 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,10 +2,10 @@ A Nix store is a collection of [store objects](objects.md) with associated operations. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build rules. +These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build tasks. A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build rules into build outputs. +It can perform builds, that is, transform build inputs using instructions from the build tasks into build outputs. It also keeps track of *references* between data and can therefore garbage-collect unused store objects. There exist different types of stores, which all follow this model. @@ -28,7 +28,7 @@ generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [B -- | -- | -- | -- | -- data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build rule | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) +build task | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) build | build | build | application of `Build` | evaluation persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap From 689b32a543240db992c9e8240401f32bd39e6736 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 12 May 2022 12:30:28 +0200 Subject: [PATCH 061/123] clarify relation of tasks and plans --- doc/manual/src/architecture/architecture.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 407e8d0f8..e8d87f8ae 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -13,21 +13,21 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ | evaluates | manages V | [ configuration language ] | - | V -+-------------------------------|---------------------------------+ -| store | | -| | evaluates to | -| | | -| referenced by V builds | -| [ build input ] --> [ build task ] --> [ build result ] | -| | + | | ++-------------------------------|---------------------V-----------+ +| store | evaluates to | +| .............................V............................... | +| : build plan : | +| : referenced by builds : | +| : [ build input ] --> [ build task ] --> [ build result ] : | +| :...........................................................: | +-----------------------------------------------------------------+ ``` At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build tasks*, used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -40,4 +40,3 @@ The command line and Nix language are what users interact with most. Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. -A series of build tasks is a *build plan*. From 75ce32405235614235dbe343beb13bb0d17934eb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 May 2022 09:55:24 +0200 Subject: [PATCH 062/123] use singular for class names consistently --- doc/manual/src/SUMMARY.md.in | 6 +++--- doc/manual/src/architecture/store/drvs/drvs.md | 2 +- doc/manual/src/architecture/store/paths.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 257880efe..f356bd07d 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,9 +17,9 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Objects](architecture/store/objects.md) - - [Store Paths](architecture/store/paths.md) - - [Derivations](architecture/store/drvs/drvs.md) + - [Store Object](architecture/store/objects.md) + - [Store Path](architecture/store/paths.md) + - [Derivation](architecture/store/drvs/drvs.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md index 766a7b47f..5f374357d 100644 --- a/doc/manual/src/architecture/store/drvs/drvs.md +++ b/doc/manual/src/architecture/store/drvs/drvs.md @@ -1,4 +1,4 @@ -# Derivations +# Derivation Derivations are recipes to create store objects. diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index cf51eb866..402e55e69 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -1,4 +1,4 @@ -# Store Paths +# Store Path A store path is a pair of a 20-byte digest and a name. From 68d26010f6d7341b73675aa30b24b67dcd479a7b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:40:00 +0200 Subject: [PATCH 063/123] architecture overview: add link to Nix expression language reference update summary title to match file contents --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index e8d87f8ae..b818b7aa1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -26,7 +26,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the *Nix language*, a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} From ef81276cc161b0ce38cb2681867529c1cbdbd7cb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 10 May 2022 12:40:28 +0200 Subject: [PATCH 064/123] architecture overview: add link to command line reference --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index b818b7aa1..77d5ebc2c 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -24,7 +24,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ +-----------------------------------------------------------------+ ``` -At the top is the *command line interface*, translating from invocations of Nix executables to interactions with the underlying layers. +At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. From 0e63b9bf8872f3556ea8925cea0c639f329d7c6e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 4 May 2022 19:02:28 +0200 Subject: [PATCH 065/123] add link from overview to store section the overview should only link to the three main concepts presented. the store is now fairly fleshed out. others can follow later. --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 77d5ebc2c..ae977a7e1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,6 +37,6 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the *Nix store*, a mechanism to keep track of build tasks, data, and references between them. +Underlying everything is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. From 25926c5fc673264288b5d9f7f175178e5b1fdad4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 09:37:28 +0200 Subject: [PATCH 066/123] Nix store does not underly literally everything Co-authored-by: Robert Hensing --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index ae977a7e1..fa992e563 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,6 +37,6 @@ In practice this amounts to a set of files in a file system. The command line and Nix language are what users interact with most. -Underlying everything is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. +Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. From 2303f84a684e41139a549595edf5d57aada4c685 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 23:46:34 +0200 Subject: [PATCH 067/123] revert to "build plan" in overview diagram this displays correct composition again. build inputs and build results are not part of build plans in terms of data objects. also this is a much less complicated setup. this will be the first impression of architecture, and we want to get it right. --- doc/manual/src/architecture/architecture.md | 29 ++++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index fa992e563..c0fd1b0b5 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -8,19 +8,22 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). ``` - [ commmand line interface ]--------+ - | | - | evaluates | manages - V | - [ configuration language ] | - | | -+-------------------------------|---------------------V-----------+ -| store | evaluates to | -| .............................V............................... | -| : build plan : | -| : referenced by builds : | -| : [ build input ] --> [ build task ] --> [ build result ] : | -| :...........................................................: | ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | | | +| +-------------------------------------------------------------+ | +-----------------------------------------------------------------+ ``` From 4639b36b53fd369024d90e95b0178e4fae0a4203 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 23:56:47 +0200 Subject: [PATCH 068/123] use reference links for URLs --- doc/manual/src/architecture/architecture.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index c0fd1b0b5..4dfe34fd6 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -5,7 +5,7 @@ It should help users understand why Nix behaves as it does, and it should help d ## Overview -Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers). +Nix consists of [hierarchical layers][layer-architecture]. ``` +-----------------------------------------------------------------+ @@ -29,7 +29,7 @@ Nix consists of hierarchical [layers](https://en.m.wikipedia.org/wiki/Multitier_ At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. -Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional](https://en.m.wikipedia.org/wiki/Purely_functional_programming) configuration language. +Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. ::: {.note} @@ -43,3 +43,5 @@ The command line and Nix language are what users interact with most. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute *build instructions*, captured in the build tasks, to produce new data. +[layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers +[purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From 7c3bca1372aeeb5074fa4038df2984b316e99bb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 21 May 2022 00:16:24 +0200 Subject: [PATCH 069/123] revert to build plans in top-level overview do not introduce build tasks yet, that is the next level of detail. --- doc/manual/src/architecture/architecture.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 4dfe34fd6..6d4b86935 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -30,7 +30,9 @@ Nix consists of [hierarchical layers][layer-architecture]. At the top is the [command line interface](../command-ref/command-ref.md), translating from invocations of Nix executables to interactions with the underlying layers. Below that is the [Nix expression language](../expressions/expression-language.md), a [purely functional][purely-functional-programming] configuration language. -It is used to compose expressions which ultimately evaluate to self-contained *build plans*, made up *build tasks* used to derive *build results* from referenced *build inputs*. +It is used to compose expressions which ultimately evaluate to self-contained *build plans*, used to derive *build results* from referenced *build inputs*. + +The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. @@ -38,10 +40,8 @@ As far as we are concerned here, the inputs and results of a derivation are just In practice this amounts to a set of files in a file system. ::: -The command line and Nix language are what users interact with most. - Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. -It can also execute *build instructions*, captured in the build tasks, to produce new data. +It can also execute build plans to produce new data. [layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers [purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From d5eea66615266e7c341e8d4e982af12d6cb82887 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:52:10 +0200 Subject: [PATCH 070/123] introduce build tasks while it appears a bit much for the overview, this way we set the stage for going directly into data types when describing the store, instead of first having to say what build tasks are and how they relate to build plans. --- doc/manual/src/architecture/architecture.md | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 6d4b86935..8000aecd1 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -43,5 +43,37 @@ In practice this amounts to a set of files in a file system. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data. +A build plan is a series of *build tasks*. +Each build task has a special build input which is used as *build instructions*. +The result of a build task can be input to another build task. + +``` ++-----------------------------------------------------------------------------------------+ +| store | +| ................................................. | +| : build plan : | +| : : | +| [ build input ]-----instructions-, : | +| : | : | +| : v : | +| [ build input ]----------->[ build task ]--instructions-, : | +| : | : | +| : | : | +| : v : | +| : [ build task ]----->[ build result ] | +| [ build input ]-----instructions-, ^ : | +| : | | : | +| : v | : | +| [ build input ]----------->[ build task ]---------------' : | +| : ^ : | +| : | : | +| [ build input ]------------------' : | +| : : | +| : : | +| :...............................................: | +| | ++-----------------------------------------------------------------------------------------+ +``` + [layer-architecture]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers [purely-functional-programming]: https://en.m.wikipedia.org/wiki/Purely_functional_programming From b6b112b366bed363f94231410946ad82122b317c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:28:40 +0200 Subject: [PATCH 071/123] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 32 ++++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cc6698a40..d710ca4c0 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -17,21 +17,35 @@ Examples: Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). The store directory defaults to `/nix/store`, but is in principle arbitrary. -## A [Rosetta stone](https://en.m.wikipedia.org/wiki/Rosetta_Stone) for build system terminology +## A [Rosetta stone][rosetta-stone] for build system terminology The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. -generic build system | Nix | [Bazel](https://bazel.build/start/bazel-intro) | [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) | programming language +generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language -- | -- | -- | -- | -- -data (build input, build result) | store object | [artifact](https://bazel.build/reference/glossary#artifact) | value | value -build instructions | builder | ([depends on action type](https://docs.bazel.build/versions/main/skylark/lib/actions.html)) | function | function -build task | derivation | [action](https://bazel.build/reference/glossary#action) | `Task` | [thunk](https://en.m.wikipedia.org/wiki/Thunk) -build plan | derivation graph | [action graph](https://bazel.build/reference/glossary#action-graph), [build graph](https://bazel.build/reference/glossary#build-graph) | `Tasks` | [call graph](https://en.m.wikipedia.org/wiki/Call_graph) +data (build input, build result) | store object | [artifact][bazel-artifact] | value | value +build instructions | builder | ([depends on action type][bazel-actions]) | function | function +build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] +build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] build | build | build | application of `Build` | evaluation -persistence layer | store | [action cache](https://bazel.build/reference/glossary#action-cache) | `Store` | heap +persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap -All of these systems share features of [declarative programming](https://en.m.wikipedia.org/wiki/Declarative_programming) languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment](https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf) (2004), elaborated in his PhD thesis [The Purely Functional Software -Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf) (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte](https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf) (2018). +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf From e72a7874dc59e7dd9783fb2783cfaf88806bf085 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:29:59 +0200 Subject: [PATCH 072/123] beautify rosetta table while this may eventually introduce ugly diffs, the table will now render readably on the terminal (e.g. for `man nix` or `nix --help`) without further intervention. --- doc/manual/src/architecture/store/store.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d710ca4c0..808e87581 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -24,14 +24,14 @@ Usage of terms is, for historic reasons, not entirely consistent within the Nix The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. -generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language --- | -- | -- | -- | -- -data (build input, build result) | store object | [artifact][bazel-artifact] | value | value -build instructions | builder | ([depends on action type][bazel-actions]) | function | function -build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] -build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] -build | build | build | application of `Build` | evaluation -persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). From 207992a71d5d9c9ee5d09c90f30a9dd35991691d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 13:38:47 +0200 Subject: [PATCH 073/123] introduce store and store objects without file system details this leaves open implementation details, especially about store paths and file system objects, and allows explaining them together were it is more appropriate. also leaves room to carefully introduce the key insight behind Nix: applying results from programming language theory to the operating system paradigm of files and processes. --- doc/manual/src/architecture/store/store.md | 57 ++++++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 808e87581..f421a9ec3 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,12 +1,59 @@ # Store -A Nix store is a collection of [store objects](objects.md) with associated operations. +A Nix store is a collection of *store objects*. -These store objects can hold arbitrary data, and Nix makes no distinction if they are used as build inputs, build results, or build tasks. +Store objects can hold arbitrary *data* and *references* to one another. +Nix makes no distinction if they are used as build inputs, build results, or build tasks. -A Nix store allows adding, retrieving, and deleting store objects. -It can perform builds, that is, transform build inputs using instructions from the build tasks into build outputs. -It also keeps track of *references* between data and can therefore garbage-collect unused store objects. +```haskell +data Store = Set StoreObject + +data StoreObject = StoreObject { + data :: Data +, references :: Set Reference +} +``` + +A Nix store can *add*, *retrieve*, and *delete* store objects. + +It can *perform builds*, that is, create new store objects by transforming build inputs, using instructions from the build tasks, into build outputs. + +As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. + +```haskell +add :: Store -> Data -> (Store, Reference) +get :: Store -> Reference -> StoreObject +delete :: Store -> Reference -> Store + +build :: Store -> Reference -> Maybe (Store, Reference) + +collectGarbage :: Store -> Store +``` + +Store objects are [immutable](https://en.m.wikipedia.org/wiki/Immutable_object): once created, they do not change until they are deleted. + +References are [opaque](https://en.m.wikipedia.org/wiki/Opaque_data_type), [unique identifiers](https://en.m.wikipedia.org/wiki/Unique_identifier): +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +An added store object cannot have references, unless it is a build task. + +Building a store object will add appropriate references, according to provided build instructions. +These references can only come from declared build inputs, and are not known by build instructions a priori. + +```haskell +data Data = Data | Task BuildTask + +data BuildTask = BuildTask { + instructions :: Reference +, inputs :: [Reference] +} +``` + +A store object cannot be deleted as long as it is reachable from a reference still in use. +Garbage collection will delete all store objects that cannot be reached from any reference in use. + + There exist different types of stores, which all follow this model. Examples: From b84f2bdfdd21b9aa65cbfac7e0292971d52c40e4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 11 May 2022 14:20:56 +0200 Subject: [PATCH 074/123] introduce mapping to Unix files and processes --- doc/manual/src/architecture/architecture.md | 1 - doc/manual/src/architecture/store/store.md | 51 +++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 8000aecd1..3d17074cc 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -37,7 +37,6 @@ The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. As far as we are concerned here, the inputs and results of a derivation are just data. -In practice this amounts to a set of files in a file system. ::: Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f421a9ec3..d2d64dd53 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -39,7 +39,7 @@ A reference will always point to exactly one store object. An added store object cannot have references, unless it is a build task. Building a store object will add appropriate references, according to provided build instructions. -These references can only come from declared build inputs, and are not known by build instructions a priori. +These references can only come from declared build inputs, and are not known to build instructions a priori. ```haskell data Data = Data | Task BuildTask @@ -55,14 +55,59 @@ Garbage collection will delete all store objects that cannot be reached from any +## Files and Processes + +Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) on the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). + +Nix encodes immutable store objects and opaque identifiers as file system primitives: files, directories, and paths. +That allows processes to resolve references contained in files and thus access the contents of store objects. + +``` ++-----------------------------------------------------------------+ +| Nix | +| [ commmand line interface ]------, | +| | | | +| evaluates | | +| | manages | +| V | | +| [ configuration language ] | | +| | | | +| +-----------------------------|-------------------V-----------+ | +| | store evaluates to | | +| | | | | +| | referenced by V builds | | +| | [ build input ] ---> [ build plan ] ---> [ build result ] | | +| | ^ | | | +| +---------|----------------------------------------|----------+ | ++-----------|----------------------------------------|------------+ + | | + file system object store path + | | ++-----------|----------------------------------------|------------+ +| operating system +------------+ | | +| '------------ | | <-----------' | +| | file | | +| ,-- | | <-, | +| | +------------+ | | +| execute as | | read, write, execute | +| | +------------+ | | +| '-> | process | --' | +| +------------+ | ++-----------------------------------------------------------------+ +``` + +Store objects are therefore implemented as the pair of + + - a *file system object* for data + - a set of *store paths* for references. + There exist different types of stores, which all follow this model. Examples: - store on the local file system - remote store accessible via SSH - binary cache store accessible via HTTP -Every store with a file system representation has a *store directory*, which contains that store’s objects accessible through [store paths](paths.md). -The store directory defaults to `/nix/store`, but is in principle arbitrary. +Every store ultimately has to make store objects accessible to processes through the file system. ## A [Rosetta stone][rosetta-stone] for build system terminology From 4eb11d45928a412643aaf2c2ed40aa7c35cd345c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 20 May 2022 00:30:08 +0200 Subject: [PATCH 075/123] fix grammar for clarity --- doc/manual/src/architecture/store/store.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index d2d64dd53..99a92bf8e 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -16,7 +16,7 @@ data StoreObject = StoreObject { A Nix store can *add*, *retrieve*, and *delete* store objects. -It can *perform builds*, that is, create new store objects by transforming build inputs, using instructions from the build tasks, into build outputs. +It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. @@ -57,9 +57,9 @@ Garbage collection will delete all store objects that cannot be reached from any ## Files and Processes -Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) on the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). +Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). -Nix encodes immutable store objects and opaque identifiers as file system primitives: files, directories, and paths. +Nix encodes immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. ``` From 4adb6602bdf274f1b0471d8a7ab53e80eaff4854 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 02:58:25 +0200 Subject: [PATCH 076/123] clarify first sentence on store objects --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 99a92bf8e..18d25ca4d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,8 +2,8 @@ A Nix store is a collection of *store objects*. -Store objects can hold arbitrary *data* and *references* to one another. -Nix makes no distinction if they are used as build inputs, build results, or build tasks. +A store object can hold arbitrary *data* and *references* to other store objects. +Nix makes no distinction if store objects are used as build inputs, build results, or build tasks. ```haskell data Store = Set StoreObject From db8703bcac78ed2d132e39fe9bc13a2cd0fe6efc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 03:05:44 +0200 Subject: [PATCH 077/123] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 18d25ca4d..1640944ad 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -18,7 +18,7 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. -As it keeps track of references, it can [garbage-collect](https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science)) unused store objects. +As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. ```haskell add :: Store -> Data -> (Store, Reference) @@ -30,9 +30,9 @@ build :: Store -> Reference -> Maybe (Store, Reference) collectGarbage :: Store -> Store ``` -Store objects are [immutable](https://en.m.wikipedia.org/wiki/Immutable_object): once created, they do not change until they are deleted. +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -References are [opaque](https://en.m.wikipedia.org/wiki/Opaque_data_type), [unique identifiers](https://en.m.wikipedia.org/wiki/Unique_identifier): +References are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. @@ -55,6 +55,11 @@ Garbage collection will delete all store objects that cannot be reached from any +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + ## Files and Processes Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). From 445f753a820cb4b6076fec1d69d66c649618c4f0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 03:40:17 +0200 Subject: [PATCH 078/123] replace pseudo code by diagrams change prose description to visually resemble the data structure --- doc/manual/src/architecture/store/store.md | 57 ++++++++++++---------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 1640944ad..5a0f4a2df 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,33 +2,49 @@ A Nix store is a collection of *store objects*. -A store object can hold arbitrary *data* and *references* to other store objects. -Nix makes no distinction if store objects are used as build inputs, build results, or build tasks. +A store object can hold -```haskell -data Store = Set StoreObject +- arbitrary *data* +- *references* to other store objects. -data StoreObject = StoreObject { - data :: Data -, references :: Set Reference -} -``` +Nix makes no distinction if store objects are build inputs, build results, or build tasks. A Nix store can *add*, *retrieve*, and *delete* store objects. + [ data ] + | + V + [ store ] ---> add ----> [ store' ] [ reference ] + + + + [ reference ] + | + V + [ store ] ---> get ----> [ store object ] + + + + [ reference ] + | + V + [ store ] --> delete --> [ store' ] + + It can *perform builds*, that is, create new store objects by transforming build inputs into build outputs, using instructions from the build tasks. + + [ reference ] + | + V + [ store ] --> build --(maybe)--> [ store' ] [ reference' ] + + As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. -```haskell -add :: Store -> Data -> (Store, Reference) -get :: Store -> Reference -> StoreObject -delete :: Store -> Reference -> Store -build :: Store -> Reference -> Maybe (Store, Reference) + [ store ] --> collect garbage --> [ store' ] -collectGarbage :: Store -> Store -``` Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. @@ -41,15 +57,6 @@ An added store object cannot have references, unless it is a build task. Building a store object will add appropriate references, according to provided build instructions. These references can only come from declared build inputs, and are not known to build instructions a priori. -```haskell -data Data = Data | Task BuildTask - -data BuildTask = BuildTask { - instructions :: Reference -, inputs :: [Reference] -} -``` - A store object cannot be deleted as long as it is reachable from a reference still in use. Garbage collection will delete all store objects that cannot be reached from any reference in use. From 4341849193cfd9b8e1e494292516a60734dfcc53 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:15:53 +0200 Subject: [PATCH 079/123] move closure property to discussion references --- doc/manual/src/architecture/store/objects.md | 15 ------------- doc/manual/src/architecture/store/store.md | 23 +++++++++++++++----- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md index f7587e112..8ab0b9368 100644 --- a/doc/manual/src/architecture/store/objects.md +++ b/doc/manual/src/architecture/store/objects.md @@ -35,21 +35,6 @@ A bare file or symlink can be a root file system object. Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. -## Reference {#reference} - -A store object can reference other store objects. - -Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. - -Building, copying and deleting store objects must be done in a way that obeys this property: - -- Build results must only refer to store objects in the closure of the build inputs. - -- Store objects being copied must refer to objects already in the destination store. - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete unreferenced objects. - ### Reference scanning While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5a0f4a2df..a3d9e2eeb 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -52,15 +52,26 @@ References are [opaque][opaque-data-type], [unique identifiers][unique-identifie The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -An added store object cannot have references, unless it is a build task. +Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. -Building a store object will add appropriate references, according to provided build instructions. -These references can only come from declared build inputs, and are not known to build instructions a priori. +Adding, building, copying and deleting store objects must be done in a way that obeys this property: -A store object cannot be deleted as long as it is reachable from a reference still in use. -Garbage collection will delete all store objects that cannot be reached from any reference in use. +- A newly added store object cannot have references, unless it is a build task. - +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to provided build instructions. + These references can only come from declared build inputs. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + Garbage collection will delete all store objects that cannot be reached from any reference in use. + + [garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) [immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object From 843288a451461610f67798af2598a2034668153a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:16:33 +0200 Subject: [PATCH 080/123] add subsections for objects and references group description of data instead of spreading it across the section. that should help direct skimming. as it turns out, people do not actually read any of that. --- doc/manual/src/architecture/store/store.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index a3d9e2eeb..de12ac1dc 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,6 +2,8 @@ A Nix store is a collection of *store objects*. +## Store Object + A store object can hold - arbitrary *data* @@ -9,6 +11,16 @@ A store object can hold Nix makes no distinction if store objects are build inputs, build results, or build tasks. +Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. + +## Reference + +References to store objects are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: +The only way to obtain references is by adding or building store objects. +A reference will always point to exactly one store object. + +## Operations + A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] @@ -46,11 +58,7 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. - -References are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: -The only way to obtain references is by adding or building store objects. -A reference will always point to exactly one store object. +## Closure Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. From e63a768e21d9bff605e51de54f9ec1a8f74650a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:18:07 +0200 Subject: [PATCH 081/123] use reference links for URLs --- doc/manual/src/architecture/store/store.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index de12ac1dc..710d3265d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -88,11 +88,12 @@ Adding, building, copying and deleting store objects must be done in a way that ## Files and Processes -Nix provides a mapping between its store model and the [Unix paradigm](https://en.m.wikipedia.org/wiki/Everything_is_a_file) that governs the interplay of [files and processes](https://en.m.wikipedia.org/wiki/File_descriptor). - -Nix encodes immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. +Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. +[unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file +[file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor + ``` +-----------------------------------------------------------------+ | Nix | From 7b7e4c6340513efb838aa576d95c5032a90dc50d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 26 May 2022 05:20:21 +0200 Subject: [PATCH 082/123] use singular to match section heading --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 710d3265d..7367b4626 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -15,7 +15,7 @@ Store objects are [immutable][immutable-object]: once created, they do not chang ## Reference -References to store objects are [opaque][opaque-data-type], [unique identifiers][unique-identifier]: +A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. From 3794618ccecf56a468eb9befd351e3cd39595ab5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:11:58 +0200 Subject: [PATCH 083/123] add commas between output values --- doc/manual/src/architecture/store/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 7367b4626..6ac210856 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,7 +26,7 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] | V - [ store ] ---> add ----> [ store' ] [ reference ] + [ store ] ---> add ----> [ store' ], [ reference ] @@ -49,7 +49,7 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build --(maybe)--> [ store' ] [ reference' ] + [ store ] --> build --(maybe)--> [ store' ], [ reference' ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From 80de4a4804bf16fe3266618cb79ad486dc6659ae Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:20:11 +0200 Subject: [PATCH 084/123] operations diagram: store' to the right this also looks more diverse, hopefully easier to distinguish Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 6ac210856..33777dc0d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -26,14 +26,20 @@ A Nix store can *add*, *retrieve*, and *delete* store objects. [ data ] | V - [ store ] ---> add ----> [ store' ], [ reference ] + [ store ] ---> add ----> [ store' ] + | + V + [ reference ] [ reference ] | V - [ store ] ---> get ----> [ store object ] + [ store ] ---> get + | + V + [ store object ] @@ -49,7 +55,12 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build --(maybe)--> [ store' ], [ reference' ] + [ store ] --> build + \ + (maybe) --> [ store' ] + | + V + [ reference ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From 195aa28ff73ee21bfe2d398e570bf458f07f6757 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:29:32 +0200 Subject: [PATCH 085/123] references are added according to build task at this level of abstraction we do not really care about build instructions or what they are, and also build instructions including their arguments really amount to the build task. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 33777dc0d..aff41d511 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -79,7 +79,7 @@ Adding, building, copying and deleting store objects must be done in a way that - Build results must only refer to store objects in the closure of the build inputs. - Building a store object will add appropriate references, according to provided build instructions. + Building a store object will add appropriate references, according to the build task. These references can only come from declared build inputs. - Store objects being copied must refer to objects already in the destination store. From 7993ba1f388e509d7617808b4a947ed1bf1533a8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:34:24 +0200 Subject: [PATCH 086/123] constrain garbage collection scope garbage collection is now incremental, and may (in theory) never delete all unreferenced objects if it is slow enough. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index aff41d511..5b4c3072d 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -88,7 +88,7 @@ Adding, building, copying and deleting store objects must be done in a way that - We can only safely delete store objects which are not reachable from any reference still in use. - Garbage collection will delete all store objects that cannot be reached from any reference in use. + Garbage collection will delete those store objects that cannot be reached from any reference in use. From a90fc62b618f571bc5e6512e66e9c8c9a8eb6e78 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:50:51 +0200 Subject: [PATCH 087/123] make clear that file system is for processes Co-authored-by: John Ericson --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5b4c3072d..3d1ad0bad 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -150,7 +150,7 @@ Examples: - remote store accessible via SSH - binary cache store accessible via HTTP -Every store ultimately has to make store objects accessible to processes through the file system. +To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. ## A [Rosetta stone][rosetta-stone] for build system terminology From 19d8a5d83942f13c3f532b4ac21cc91c7158c21a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 2 Jun 2022 23:45:04 +0200 Subject: [PATCH 088/123] move first mention of file system object before diagram --- doc/manual/src/architecture/store/store.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 3d1ad0bad..a22735e42 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -102,6 +102,11 @@ Adding, building, copying and deleting store objects must be done in a way that Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. +Store objects are therefore implemented as the pair of + + - a *file system object* for data + - a set of *store paths* for references. + [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor @@ -139,11 +144,6 @@ That allows processes to resolve references contained in files and thus access t +-----------------------------------------------------------------+ ``` -Store objects are therefore implemented as the pair of - - - a *file system object* for data - - a set of *store paths* for references. - There exist different types of stores, which all follow this model. Examples: - store on the local file system From 93f721b0d26b6525d8eaa8a1606ebeb4b71a9b6a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:36:29 +0200 Subject: [PATCH 089/123] remove draft on derivations --- doc/manual/src/SUMMARY.md.in | 1 - .../src/architecture/store/drvs/drvs.md | 59 ------------------- 2 files changed, 60 deletions(-) delete mode 100644 doc/manual/src/architecture/store/drvs/drvs.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f356bd07d..5e2d26bf7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -19,7 +19,6 @@ - [Store](architecture/store/store.md) - [Store Object](architecture/store/objects.md) - [Store Path](architecture/store/paths.md) - - [Derivation](architecture/store/drvs/drvs.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/drvs/drvs.md b/doc/manual/src/architecture/store/drvs/drvs.md deleted file mode 100644 index 5f374357d..000000000 --- a/doc/manual/src/architecture/store/drvs/drvs.md +++ /dev/null @@ -1,59 +0,0 @@ -# Derivation - -Derivations are recipes to create store objects. - -Derivations are the heart of Nix. -Other system (like Git or IPFS) also store and transfer immutable data, but they don't concern themselves with *how* that data was created. -This is where Nix comes in. - -Derivations produce data by running arbitrary commands, like Make or Ninja rules. -Unlike those systems, derivations do not produce arbitrary files, but only specific store objects. -They cannot modify the store in any way, other than creating those store objects. -This rigid specification of what they do is what allows Nix's caching to be so simple and yet robust. - -Based on the above, we can conceptually break derivations down into 3 parts: - -1. What command will be run? - -2. What existing store objects are needed as inputs? - -3. What store objects will be produced as outputs? - -## What command will be run? - -The original core of Nix was very simple about this, in the mold of traditional Unix. -Commands consist of 3 parts: - -1. Path to executable - -2. Arguments (Except for `argv[0]`, which is taken from the path in the usual way) - -3. Environment variables. - -## What existing store objects are needed as inputs? - -The previous sub-section begs the question "how can we be sure the path to the executable points to what we think it does?" -It's a good questions! - -## What store objects will be produced as outputs? - -## Extra extensions - -### `__structuredAttrs` - -Historically speaking, most users of Nix made GNU Bash with a script the command run, regardless of what they were doing. -Bash variable are automatically created from env vars, but bash also supports array and string-keyed map variables in addition to string variables. -People also usually create derivations using language which also support these richer data types. -It was thus desired a way to get this data from the language "planning" the derivation to language to bash, the language evaluated at "run time". - -`__structuredAttrs` does this by smuggling inside the core derivation format a map of named richer data. -At run time, this becomes two things: - -1. A JSON file containing that map. -2. A bash script setting those variables. - -The bash command can be passed a script which will "source" that Nix-created bash script, setting those variables with the richer data. -The outer script can then do whatever it likes with those richer variables as input. - -However, since derivations can already contain arbitary input sources, the vast majority of `__structuredAttrs` can be handled by upper layers. -We might consider implementing `__structuredAttrs` in higher layers in the future, and simplifying the store layer. From 84ddfbf8fd9b1e77fefb3c2c11a9cedab2e530c2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:57:19 +0200 Subject: [PATCH 090/123] remove diagonal from operations diagram --- doc/manual/src/architecture/store/store.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index a22735e42..3b99982e4 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -55,12 +55,10 @@ It can *perform builds*, that is, create new store objects by transforming build [ reference ] | V - [ store ] --> build - \ - (maybe) --> [ store' ] - | - V - [ reference ] + [ store ] --> build --(maybe)--> [ store' ] + | + V + [ reference ] As it keeps track of references, it can [garbage-collect][garbage-collection] unused store objects. From f632816cbaefbba9cc27a8e0de6cffd39fa7a8dd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:09:27 +0200 Subject: [PATCH 091/123] add explanation and examples of file system objects --- doc/manual/src/architecture/store/fso.md | 64 ++++++++++++++++++++++ doc/manual/src/architecture/store/store.md | 4 +- 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 doc/manual/src/architecture/store/fso.md diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md new file mode 100644 index 000000000..958b8f62a --- /dev/null +++ b/doc/manual/src/architecture/store/fso.md @@ -0,0 +1,64 @@ +# File System Object + +The Nix store uses a simple file system model for the data it holds in [store objects](store.md#store-object). + +Every file system object is one of the following: + + - File: an executable flag, and arbitrary data for contents + - Directory: mapping of names to child file system objects + - [Symbolic link][symlink]: may point anywhere. + +We call a store object's outermost file system object the *root*. + + data FileSystemObject + = File { isExecutable :: Bool, contents :: Bytes } + | Directory { entries :: Map FileName FileSystemObject } + | SymLink { target :: Path } + +Examples: + +- a directory with contents + + /nix/store/-hello-2.10 + ├── bin + │   └── hello + └── share + ├── info + │   └── hello.info + └── man + └── man1 + └── hello.1.gz + +- a directory with relative symlink and other contents + + /nix/store/-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ + +- a directory with absolute symlink + + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. + +A bare file or symlink can be a root file system object. +Examles: + + /nix/store/-hello-2.10.tar.gz + + /nix/store/4j5...-pkg-config-wrapper-0.29.2-doc -> /nix/store/i99...-pkg-config-0.29.2-doc + +Symlinks pointing outside of their own root or to a store object without a matching reference are allowed, but might not function as intended. +Examples: + +- an arbitrarily symlinked file may change or not exist at all + + /nix/store/-foo + └── foo -> /home/foo + +- if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted + + /nix/store/-bar + └── bar -> /nix/store/abc...-foo + +[symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 3b99982e4..68bdadc4a 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -2,7 +2,7 @@ A Nix store is a collection of *store objects*. -## Store Object +## Store Object {#store-object} A store object can hold @@ -102,7 +102,7 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - - a *file system object* for data + - a [file system object](fso.md) for data - a set of *store paths* for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file From fa7ad4593d09d04afe1d215d2da09be02c2e2836 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 11:07:50 +0200 Subject: [PATCH 092/123] explain store directory --- doc/manual/src/SUMMARY.md.in | 6 +- doc/manual/src/architecture/store/objects.md | 48 ------- doc/manual/src/architecture/store/paths.md | 129 +++++++++++-------- doc/manual/src/architecture/store/store.md | 21 +-- 4 files changed, 94 insertions(+), 110 deletions(-) delete mode 100644 doc/manual/src/architecture/store/objects.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5e2d26bf7..997d75444 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,8 +17,10 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Object](architecture/store/objects.md) - - [Store Path](architecture/store/paths.md) + - [Store Path](architecture/store/path.md) + - [Digest](architecture/store/path.md#digest) + - [Input Addressing](architecture/store/path.md#input-addressing) + - [Content Addressing](architecture/store/path.md#content-addressing) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/architecture/store/objects.md b/doc/manual/src/architecture/store/objects.md deleted file mode 100644 index 8ab0b9368..000000000 --- a/doc/manual/src/architecture/store/objects.md +++ /dev/null @@ -1,48 +0,0 @@ -# Store Object - -Nix organizes the data it manages into *store objects*. -A store object is the pair of - - - a [file system object](#file-system-object) - - a set of [references](#reference) to store objects. - -We call a store object's outermost file system object the *root*. - -```haskell -data StoreOject = StoreObject { - root :: FileSystemObject -, references :: Set StoreObject -} -``` - -## File system object {#file-system-object} - -The Nix store uses a simple file system model. - -Every file system object is one of the following: - - File: an executable flag, and arbitrary data for contents - - Directory: mapping of names to child file system objects - - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link): may point anywhere. - -```haskell -data FileSystemObject - = File { isExecutable :: Bool, contents :: Bytes } - | Directory { entries :: Map FileName FileSystemObject } - | SymLink { target :: Path } -``` - -A bare file or symlink can be a root file system object. - -Symlinks pointing outside of their own root, or to a store object without a matching reference, are allowed, but might not function as intended. - -### Reference scanning - -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. -Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. - -However, having references match store paths in files is not enforced by the data model: -Store objects could have excess or incomplete references with respect to store paths found in their file contents. - -Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. -Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. - diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index 402e55e69..4867e7fd3 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -1,78 +1,103 @@ # Store Path -A store path is a pair of a 20-byte digest and a name. +Nix implements [references](store.md#reference) to [store objects](store.md#store-object) as *store paths*. -## String representation +Store paths are pairs of -A store path is rendered as the concatenation of +- a 20-byte [digest](#digest) for identification +- a symbolic name for people to read. - - a store directory - - - a path-separator (`/`) - - - the digest rendered as Base-32 (20 arbitrary bytes becomes 32 ASCII chars) - - - a hyphen (`-`) - - - the name - -Let's take the store path from the very beginning of this manual as an example: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - -This parses like so: - - /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 - ^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ - store dir digest name - -We then can discard the store dir to recover the conceptual pair that is a store path: +Example: { digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", name: "firefox-33.1", } -### Where did the "store directory" come from? +It is rendered to a file system path as the concatenation of -If you notice, the above references a "store directory", but that is *not* part of the definition of a store path. -We can discard it when parsing, but what about when printing? -We need to get a store directory from *somewhere*. + - [store directory](#store-directory) + - path-separator (`/`) + - [digest](#digest) rendered in [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - hyphen (`-`) + - name -The answer is, the store directory is a property of the store that contains the store path. -The explanation for this is simple enough: a store is notionally mounted as a directory at some location, and the store object's root file system likewise mounted at this path within that directory. +Example: -This does, however, mean the string representation of a store path is not derived just from the store path itself, but is in fact "context dependent". + /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 + |--------| |------------------------------| |----------| + store directory digest name -## The digest +## Store Directory {#store-directory} -The calculation of the digest is quite complicated for historical reasons. -The details of the algorithms will be discussed later once more concepts have been introduced. -For now, we just concern ourselves with the *key properties* of those algorithms. +Every [store](./store.md) has a store directory. + +If the store has a [file system representation](./store.md#files-and-processes), this directory contains the store’s [file system objects](#file-system-object), which can be addressed by [store paths](#store-path). + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. ::: {.note} -**Historical note** The 20 byte restriction is because originally a digests were SHA-1 hashes. -This is no longer true, but longer hashes and other information are still boiled down to 20 bytes. +The store directory defaults to `/nix/store`, but is in principle arbitrary. ::: -Store paths are either *content-addressed* or *input-addressed*. +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. + +Therefore one can only copy store objects if + +- the source and target stores' directories match + + or + +- the store object in question has no references, that is, contains no store paths. + +To move a store object to a store with a different store directory, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may break internal offsets or content hashes. + +# Digest {#digest} + +In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. + +Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). ::: {.note} -The former is a standard term used elsewhere. -The later is our own creation to evoke a contrast with content addressing. +**Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +This is no longer true, but longer hashes and other information are still truncated to 20 bytes for compatibility. ::: -Content addressing means that the store path digest ultimately derives from referred store object's contents, namely its file system objects and references. -There is more than one *method* of content-addressing, however. -Still, if one does know the content addressing schema that was used, -(or guesses, there isn't that many yet!) -one can recalculate the store path and thus verify the store object. +[digest]: https://en.m.wiktionary.org/wiki/digest#Noun +[hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function +[sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 -Input addressing means that the store path digest derives from how the store path was produced, namely the "inputs" and plan that it was built from. -Store paths of this sort can *not* be validated from the content of the store object. -Rather, the store object might come with the store path it expects to be referred to by, and a signature of that path, the contents of the store path, and other metadata. -The signature indicates that someone is vouching for the store object really being the results of a plan with that digest. -While metadata is included in the digest calculation explaining which method it was calculated by, this only serves to thwart pre-image attacks. -That metadata is scrambled with everything else so that it is difficult to tell how a given store path was produced short of a brute-force search. -In the parlance of referencing schemes, this means that store paths are not "self-describing". +### Reference scanning + +While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. +Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. + +However, having references match store paths in files is not enforced by the data model: +Store objects could have excess or incomplete references with respect to store paths found in their file contents. + +Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. + +## Input Addressing {#input-addressing} + +Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. + +To compute the hash of a store object one needs a deterministic serialisation, i.e., a binary string representation which only changes if the store object changes. + +Nix has a custom serialisation format called Nix Archive (NAR) + +Store object references of this sort can *not* be validated from the content of the store object. +Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. + +## Content Addressing {#content-addressing} + +Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. +If one knows content addressing was used, one can recalculate the reference and thus verify the store object. + +Content addressing is currently only used for the special cases of source files and "fixed-output derivations", where the contents of a store object are known in advance. +Content addressing of build results is still an [experimental feature subject to some restrictions](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md). + diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 68bdadc4a..21a876f75 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -67,18 +67,19 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -## Closure +## Closure {#closure} -Nix stores have the *closure property*: for each store object in the store, all the store objects it references must also be in the store. +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. -Adding, building, copying and deleting store objects must be done in a way that obeys this property: +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - A newly added store object cannot have references, unless it is a build task. - Build results must only refer to store objects in the closure of the build inputs. Building a store object will add appropriate references, according to the build task. - These references can only come from declared build inputs. - Store objects being copied must refer to objects already in the destination store. @@ -86,16 +87,15 @@ Adding, building, copying and deleting store objects must be done in a way that - We can only safely delete store objects which are not reachable from any reference still in use. - Garbage collection will delete those store objects that cannot be reached from any reference in use. - +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity [garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) [immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object [opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type [unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier -## Files and Processes +## Files and Processes {#files-and-processes} Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. @@ -103,11 +103,16 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - a [file system object](fso.md) for data - - a set of *store paths* for references. + - a set of [store paths](paths.md) for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor +The following diagram shows a radical simplification of how Nix interacts with the operating system: +It uses files as build inputs, and build outputs are files again. +On the operating system, files are either "dead" data, or "live" as processes, which in turn operate on files, or can bring them to life. +A build function also amounts to an operating system process (not depicted). + ``` +-----------------------------------------------------------------+ | Nix | From 1681f4e9f3608973e3247e447fd9236eb32d66f3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 13 Jun 2022 17:10:31 +0200 Subject: [PATCH 093/123] better explain reference scanning --- doc/manual/src/architecture/store/paths.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/paths.md index 4867e7fd3..956049341 100644 --- a/doc/manual/src/architecture/store/paths.md +++ b/doc/manual/src/architecture/store/paths.md @@ -70,18 +70,21 @@ This is no longer true, but longer hashes and other information are still trunca [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function [sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 - ### Reference scanning -While references could be arbitrary paths, Nix requires them to be store paths to ensure correctness. -Anything outside a given store is not under control of Nix, and therefore cannot be guaranteed to be present when needed. +When a new store object is built, Nix scans its file contents for store paths to construct its set of references. -However, having references match store paths in files is not enforced by the data model: -Store objects could have excess or incomplete references with respect to store paths found in their file contents. +The special format of a store path's [digest](#digest) allows reliably detecting it among arbitrary data. +Nix uses the [closure](store.md#closure) of build inputs to derive the list of allowed store paths, to avoid false positives. -Scanning files therefore allows reliably capturing run time dependencies without declaring them explicitly. +This way, scanning files captures run time dependencies without the user having to declare them explicitly. Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. +::: {.note} +In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +This depends on the specifics of the software to build and run. +::: + ## Input Addressing {#input-addressing} Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. From 9c544813d7883fa003d25f528f50c46faf06e1a2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:01:31 +0200 Subject: [PATCH 094/123] paths -> path use singular for terminology uniformly --- doc/manual/src/architecture/store/{paths.md => path.md} | 0 doc/manual/src/architecture/store/store.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename doc/manual/src/architecture/store/{paths.md => path.md} (100%) diff --git a/doc/manual/src/architecture/store/paths.md b/doc/manual/src/architecture/store/path.md similarity index 100% rename from doc/manual/src/architecture/store/paths.md rename to doc/manual/src/architecture/store/path.md diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 21a876f75..f662ed3e7 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -103,7 +103,7 @@ That allows processes to resolve references contained in files and thus access t Store objects are therefore implemented as the pair of - a [file system object](fso.md) for data - - a set of [store paths](paths.md) for references. + - a set of [store paths](path.md) for references. [unix-paradigm]: https://en.m.wikipedia.org/wiki/Everything_is_a_file [file-descriptor]: https://en.m.wikipedia.org/wiki/File_descriptor From c10dcccccda331499db75753ab6fed8938c31cca Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:08:13 +0200 Subject: [PATCH 095/123] make example a simple list using JSON notation is unwarranted and not explained. --- doc/manual/src/architecture/store/path.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 956049341..2cedc4234 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -9,10 +9,8 @@ Store paths are pairs of Example: - { - digest: "b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z", - name: "firefox-33.1", - } +- digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +- name: `firefox-33.1` It is rendered to a file system path as the concatenation of From 631ca18ee62502dd2e51a8f3c9e2ab656b8382ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:08:50 +0200 Subject: [PATCH 096/123] reword notes on copying clarify that we are copying between different stores. we have not introduced that notion or why it would be interesting, but for now it should be fine to keep it in context of the store directory. we could move that later to a more detailed explanation of different store types. --- doc/manual/src/architecture/store/path.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 2cedc4234..3f2a4bbff 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -42,7 +42,7 @@ It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. -Therefore one can only copy store objects if +Therefore one can only copy store objects to a different store if - the source and target stores' directories match @@ -50,8 +50,9 @@ Therefore one can only copy store objects if - the store object in question has no references, that is, contains no store paths. -To move a store object to a store with a different store directory, it has to be rebuilt, together with all its dependencies. -It is in general not enough to replace the store directory string in file contents, as this may break internal offsets or content hashes. +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. # Digest {#digest} From 7c656d90ca2c3364fd72543c0aa27115d96ac463 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 21 Jun 2022 14:25:30 +0200 Subject: [PATCH 097/123] simplify description of diagram try not to be too fancy, it's just for reading the diagram out loud. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index f662ed3e7..b3517f9b6 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -110,7 +110,7 @@ Store objects are therefore implemented as the pair of The following diagram shows a radical simplification of how Nix interacts with the operating system: It uses files as build inputs, and build outputs are files again. -On the operating system, files are either "dead" data, or "live" as processes, which in turn operate on files, or can bring them to life. +On the operating system, files can be run as processes, which in turn operate on files. A build function also amounts to an operating system process (not depicted). ``` From ec43977553e605370d160ad0f852c15d0c754e74 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:10:44 +0200 Subject: [PATCH 098/123] store: match chapter introduction to outline the various levels of detail should describe the same things. --- doc/manual/src/architecture/store/store.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index b3517f9b6..403a23bbd 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -1,6 +1,7 @@ # Store -A Nix store is a collection of *store objects*. +A Nix store is a collection of *store objects* with references between them. +It supports operations to manipulate that collection. ## Store Object {#store-object} From 348432f48ffaf084c7ba1397fe63d805891bb68d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:17:28 +0200 Subject: [PATCH 099/123] store: add concept map this should help nativate the chapter by indicating which terms should be known to understand a given concept. --- doc/manual/src/architecture/store/store.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 403a23bbd..ed250ce0c 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -3,6 +3,28 @@ A Nix store is a collection of *store objects* with references between them. It supports operations to manipulate that collection. +The following concept map is a graphical outline of this chapter. +Arrows indicate suggested reading order. + +``` + ,----------------------[ store ] + | | + v v + [ store object ] [ operations ]----------------------, + | | | + v v | + [ files and processes ] [ build ]--------->[ derivation ] | + | | | | | + v v v v | + [ file system object ] [ store path ]--->[ reference scanning ]--->[ closure ] | + | ^ | | + v | v v + [ digest ]--------------' [ garbage collection ] + / \ + v v +[ input addressing ] [ content addressing ] +``` + ## Store Object {#store-object} A store object can hold From d8b2f9f772a7fb33db17f98605f221f5254c51b4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:21:12 +0200 Subject: [PATCH 100/123] make concept map more compact invert arrows to/from derivation: - we need closures to form derivations - we need derivations to perform builds --- doc/manual/src/architecture/store/store.md | 35 ++++++++++++---------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index ed250ce0c..fba2f90fd 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -7,22 +7,25 @@ The following concept map is a graphical outline of this chapter. Arrows indicate suggested reading order. ``` - ,----------------------[ store ] - | | - v v - [ store object ] [ operations ]----------------------, - | | | - v v | - [ files and processes ] [ build ]--------->[ derivation ] | - | | | | | - v v v v | - [ file system object ] [ store path ]--->[ reference scanning ]--->[ closure ] | - | ^ | | - v | v v - [ digest ]--------------' [ garbage collection ] - / \ - v v -[ input addressing ] [ content addressing ] + ,----------[ store ]---------, + | | + | v + | [ operations ] + | / \ + v v v + [ store object ] [ garbage collection ] [ build ] + | ^ ^ | + v | | | + [ files and processes ] | [ derivation ]--' | + / \ | ^ | + v v | | | +[ file system object ] [ store path ] '--[ closure ] | + | ^ \ | | + v | v v | + [ digest ]-----' [ reference scanning ]<------' + / \ + v v +[ input addressing ] [ content addressing ] ``` ## Store Object {#store-object} From 475a332025fd9bc399206f79298d9897dde45380 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:25:19 +0200 Subject: [PATCH 101/123] make concept map even more compact --- doc/manual/src/architecture/store/store.md | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index fba2f90fd..c4eea33fa 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -11,18 +11,18 @@ Arrows indicate suggested reading order. | | | v | [ operations ] - | / \ - v v v - [ store object ] [ garbage collection ] [ build ] - | ^ ^ | - v | | | - [ files and processes ] | [ derivation ]--' | - / \ | ^ | - v v | | | -[ file system object ] [ store path ] '--[ closure ] | - | ^ \ | | - v | v v | - [ digest ]-----' [ reference scanning ]<------' + | | | + v v | + [ store object ] [ garbage collection ] | + | ^ | + v | v + [ files and processes ] [ closure ] [ build ] + / \ | | ^ | + v v | v | | +[ file system object ] [ store path ] | [ derivation ] | + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ] <----' / \ v v [ input addressing ] [ content addressing ] From a28d6878b263270c7db271c414bef2c688bb872d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:27:08 +0200 Subject: [PATCH 102/123] concept map: put closure as it is in the chapter this is not as compact any more, but it more closely resembles the chapter structure, and clearly shows that the closure property is the key idea on which most of Nix operates. --- doc/manual/src/architecture/store/store.md | 29 ++++++++++------------ 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index c4eea33fa..0f43d5a62 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -7,22 +7,19 @@ The following concept map is a graphical outline of this chapter. Arrows indicate suggested reading order. ``` - ,----------[ store ]---------, - | | - | v - | [ operations ] - | | | - v v | - [ store object ] [ garbage collection ] | - | ^ | - v | v - [ files and processes ] [ closure ] [ build ] - / \ | | ^ | - v v | v | | -[ file system object ] [ store path ] | [ derivation ] | - | ^ | | | - v | v v | - [ digest ]----' [ reference scanning ] <----' + ,--------------[ store ]----------------, + | | | + v v v + [ store object ] [ closure ]--, [ operations ] + | | | | | | + v | | v v | + [ files and processes ] | | [ garbage collection ] | + / \ | V v + v v | [ derivation ]--->[ building ] +[ file system object ] [ store path ] | | + | ^ | | | + v | v v | + [ digest ]----' [ reference scanning ] <-----------' / \ v v [ input addressing ] [ content addressing ] From c345345deeb188830e62e9b2e957cf1d09d1c6b5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 9 Jun 2022 14:33:31 +0200 Subject: [PATCH 103/123] concept map: align hights this has the weird but nice emergent property that terms at the same height are roughly at the same level of abstraction. --- doc/manual/src/architecture/store/store.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 0f43d5a62..cba875092 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -14,12 +14,12 @@ Arrows indicate suggested reading order. | | | | | | v | | v v | [ files and processes ] | | [ garbage collection ] | - / \ | V v - v v | [ derivation ]--->[ building ] -[ file system object ] [ store path ] | | + / \ | | | + v v | v v +[ file system object ] [ store path ] | [ derivation ]--->[ building ] | ^ | | | v | v v | - [ digest ]----' [ reference scanning ] <-----------' + [ digest ]----' [ reference scanning ]<------------' / \ v v [ input addressing ] [ content addressing ] From def80d5777dcd5d4b8a10e54be24d745b5d7562b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 8 Jun 2022 11:50:24 +0200 Subject: [PATCH 104/123] add subsections to table of contents --- doc/manual/src/SUMMARY.md.in | 8 +++++++- doc/manual/src/architecture/store/store.md | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 997d75444..5581bad3a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -17,7 +17,13 @@ - [Upgrading Nix](installation/upgrading.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Path](architecture/store/path.md) + - [Store Object](architecture/store/store.md#store-object) + - [Reference](architecture/store/store.md#reference) + - [Operations](architecture/store/store.md#operations) + - [Closure](architecture/store/store.md#closure) + - [Files and Processes](architecture/store/store.md#files-and-processes) + - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Store Path](architecture/store/path.md) - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) - [Content Addressing](architecture/store/path.md#content-addressing) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index cba875092..e1d983d99 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -36,13 +36,13 @@ Nix makes no distinction if store objects are build inputs, build results, or bu Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -## Reference +## Reference {#reference} A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -## Operations +## Operations {#operations} A Nix store can *add*, *retrieve*, and *delete* store objects. @@ -178,7 +178,7 @@ Examples: To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. -## A [Rosetta stone][rosetta-stone] for build system terminology +## A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. From fe4c0b8d75b9380ab64a752a7f1da680b1163731 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 12 Jul 2022 12:28:09 +0100 Subject: [PATCH 105/123] fix typo Co-authored-by: Thomas --- doc/manual/src/architecture/store/fso.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index 958b8f62a..c51996e41 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -42,7 +42,7 @@ Examples: └── nix_node -> /nix/store/f20...-nodejs-10.24. A bare file or symlink can be a root file system object. -Examles: +Examples: /nix/store/-hello-2.10.tar.gz From de5dea45f7443dd14068e6289e8844816f445c35 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 09:37:16 +0100 Subject: [PATCH 106/123] use correct Nix entity Nix itself does care a lot about what type of store object you have. --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index e1d983d99..5ac512621 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -32,7 +32,7 @@ A store object can hold - arbitrary *data* - *references* to other store objects. -Nix makes no distinction if store objects are build inputs, build results, or build tasks. +A Nix store makes no distinction if store objects are build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. From 5a5a95696f4728010486ee01b375b861acc2fddd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 10:42:21 +0100 Subject: [PATCH 107/123] note customized base32 Nix omits E O U T characters for some reason. --- doc/manual/src/architecture/store/path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 3f2a4bbff..9f47b877f 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -16,7 +16,7 @@ It is rendered to a file system path as the concatenation of - [store directory](#store-directory) - path-separator (`/`) - - [digest](#digest) rendered in [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) + - [digest](#digest) rendered in a custom variant of [base-32](https://en.m.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) - hyphen (`-`) - name From bac86231add178e4def3e3fe0bc88ace83d86c21 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 10:59:09 +0100 Subject: [PATCH 108/123] use "build plan" consistently --- doc/manual/src/architecture/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 3d17074cc..d9fb2e7e5 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -36,7 +36,7 @@ The command line and Nix language are what users interact with most. ::: {.note} The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a derivation are just data. +As far as we are concerned here, the inputs and results of a build plan are just data. ::: Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. From 9cabba1fc35c4989b0e280c12a845adf0e300dab Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:17:46 +0100 Subject: [PATCH 109/123] mention hard links Co-authored-by: Thomas --- doc/manual/src/architecture/store/fso.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index c51996e41..2ec365eb6 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -61,4 +61,9 @@ Examples: /nix/store/-bar └── bar -> /nix/store/abc...-foo +Nix file system objects do not support [hard links][hardlink]: +each file system object which is not the root has exactly one parent and one name. +However, as store objects are immutable, an underlying file system can use hard links for optimization. + [symlink]: https://en.m.wikipedia.org/wiki/Symbolic_link +[hardlink]: https://en.m.wikipedia.org/wiki/Hard_link From 29c062537d37fc10997767edd3edb6af41f1fb00 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:42:16 +0100 Subject: [PATCH 110/123] hashes: truncate -> reduce, mention SHA-256 the longer SHA-256 hashes are not truncated, but in fact processed. Co-authored-by: Thomas --- doc/manual/src/architecture/store/path.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 9f47b877f..64bc4a522 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -62,12 +62,13 @@ Store objects are therefore said to be either [input-addressed](#input-addressin ::: {.note} **Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -This is no longer true, but longer hashes and other information are still truncated to 20 bytes for compatibility. +Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. ::: [digest]: https://en.m.wiktionary.org/wiki/digest#Noun [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function [sha-1]: https://en.m.wikipedia.org/wiki/SHA-1 +[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Reference scanning From 0228eb8ae76d048b16caffff9628aec347ff1485 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 13 Jul 2022 11:53:59 +0100 Subject: [PATCH 111/123] add Java example on manual dependency declaration --- doc/manual/src/architecture/store/path.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 64bc4a522..7d793749b 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -83,6 +83,8 @@ Doing it at build time and persisting references in the store object avoids repe ::: {.note} In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. + +For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. ::: ## Input Addressing {#input-addressing} From db6faf44a9befeaabbbd1bb025b3f5a7d743dd16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 28 Jul 2022 23:23:43 +0200 Subject: [PATCH 112/123] clarify what store objects can be the store of course makes a distinction, but that is not relevant here --- doc/manual/src/architecture/store/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 5ac512621..920d483d5 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -32,7 +32,7 @@ A store object can hold - arbitrary *data* - *references* to other store objects. -A Nix store makes no distinction if store objects are build inputs, build results, or build tasks. +Store objects can be build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. From 00a7eaed6759c284444c161c69a3a585579a7d5f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:30:27 +0200 Subject: [PATCH 113/123] add file system object to table of contents --- doc/manual/src/SUMMARY.md.in | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5581bad3a..d589156ec 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -27,6 +27,7 @@ - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) - [Content Addressing](architecture/store/path.md#content-addressing) + - [File System Object](architecture/store/fso.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) From b7309ce624d17a2dbe97f63700d59830c35b074a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:31:24 +0200 Subject: [PATCH 114/123] move architecture to the end --- doc/manual/src/SUMMARY.md.in | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index d589156ec..9c2ae2ad3 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,19 +15,6 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) -- [Architecture](architecture/architecture.md) - - [Store](architecture/store/store.md) - - [Store Object](architecture/store/store.md#store-object) - - [Reference](architecture/store/store.md#reference) - - [Operations](architecture/store/store.md#operations) - - [Closure](architecture/store/store.md#closure) - - [Files and Processes](architecture/store/store.md#files-and-processes) - - [Build system terminology](architecture/store/store.md#build-system-terminology) - - [Store Path](architecture/store/path.md) - - [Digest](architecture/store/path.md#digest) - - [Input Addressing](architecture/store/path.md#input-addressing) - - [Content Addressing](architecture/store/path.md#content-addressing) - - [File System Object](architecture/store/fso.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) @@ -79,6 +66,19 @@ @manpages@ - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) +- [Architecture](architecture/architecture.md) + - [Store](architecture/store/store.md) + - [Store Object](architecture/store/store.md#store-object) + - [Reference](architecture/store/store.md#reference) + - [Operations](architecture/store/store.md#operations) + - [Closure](architecture/store/store.md#closure) + - [Files and Processes](architecture/store/store.md#files-and-processes) + - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Store Path](architecture/store/path.md) + - [Digest](architecture/store/path.md#digest) + - [Input Addressing](architecture/store/path.md#input-addressing) + - [Content Addressing](architecture/store/path.md#content-addressing) + - [File System Object](architecture/store/fso.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) From 3df1ee2ba573a999aebb87cc9c5cac21df158120 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 12:32:35 +0200 Subject: [PATCH 115/123] clarify what explicitly declaring certain dependencies means --- doc/manual/src/architecture/store/path.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 7d793749b..3b14b2f3c 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -81,7 +81,7 @@ This way, scanning files captures run time dependencies without the user having Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. ::: {.note} -In practice, it is sometimes still necessary to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. From 8cec32e7f5e073946ef7bbf5ec1839e40b203bbc Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 13:50:44 +0200 Subject: [PATCH 116/123] fix directory tree renderings --- doc/manual/src/architecture/store/fso.md | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/manual/src/architecture/store/fso.md b/doc/manual/src/architecture/store/fso.md index 2ec365eb6..e0eb69f60 100644 --- a/doc/manual/src/architecture/store/fso.md +++ b/doc/manual/src/architecture/store/fso.md @@ -19,27 +19,27 @@ Examples: - a directory with contents - /nix/store/-hello-2.10 - ├── bin - │   └── hello - └── share - ├── info - │   └── hello.info - └── man - └── man1 - └── hello.1.gz + /nix/store/-hello-2.10 + ├── bin + │   └── hello + └── share + ├── info + │   └── hello.info + └── man + └── man1 + └── hello.1.gz - a directory with relative symlink and other contents - /nix/store/-go-1.16.9 - ├── bin -> share/go/bin - ├── nix-support/ - └── share/ + /nix/store/-go-1.16.9 + ├── bin -> share/go/bin + ├── nix-support/ + └── share/ - a directory with absolute symlink - /nix/store/d3k...-nodejs - └── nix_node -> /nix/store/f20...-nodejs-10.24. + /nix/store/d3k...-nodejs + └── nix_node -> /nix/store/f20...-nodejs-10.24. A bare file or symlink can be a root file system object. Examples: @@ -53,13 +53,13 @@ Examples: - an arbitrarily symlinked file may change or not exist at all - /nix/store/-foo - └── foo -> /home/foo + /nix/store/-foo + └── foo -> /home/foo - if a symlink to a store path was not automatically created by Nix, it may be invalid or get invalidated when the store object is deleted - /nix/store/-bar - └── bar -> /nix/store/abc...-foo + /nix/store/-bar + └── bar -> /nix/store/abc...-foo Nix file system objects do not support [hard links][hardlink]: each file system object which is not the root has exactly one parent and one name. From cc3a5f4ba2cb6859e0f324a5e74a42668b96aff0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 14:04:02 +0200 Subject: [PATCH 117/123] use correct mdBook syntax for callouts --- doc/manual/src/architecture/architecture.md | 7 +++---- doc/manual/src/architecture/store/path.md | 21 +++++++++------------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index d9fb2e7e5..96371056b 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -34,10 +34,9 @@ It is used to compose expressions which ultimately evaluate to self-contained *b The command line and Nix language are what users interact with most. -::: {.note} -The Nix language itself does not have a notion of *packages* or *configurations*. -As far as we are concerned here, the inputs and results of a build plan are just data. -::: +> **Note** +> The Nix language itself does not have a notion of *packages* or *configurations*. +> As far as we are concerned here, the inputs and results of a build plan are just data. Underlying these is the [Nix store](./store/store.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data. diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 3b14b2f3c..5a18df67a 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -34,9 +34,8 @@ If the store has a [file system representation](./store.md#files-and-processes), This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. -::: {.note} -The store directory defaults to `/nix/store`, but is in principle arbitrary. -::: +> **Note** +> The store directory defaults to `/nix/store`, but is in principle arbitrary. It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. @@ -60,10 +59,9 @@ In a [store path](#store-path), the [digest][digest] is the output of a [cryptog Store objects are therefore said to be either [input-addressed](#input-addressing) or [content-addressed](#content-addressing). -::: {.note} -**Historical note**: The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. -Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. -::: +> **Historical Note** +> The 20 byte restriction is because originally digests were [SHA-1][sha-1] hashes. +> Nix now uses [SHA-256][sha-256], and longer hashes are still reduced to 20 bytes for compatibility. [digest]: https://en.m.wiktionary.org/wiki/digest#Noun [hash]: https://en.m.wikipedia.org/wiki/Cryptographic_hash_function @@ -80,12 +78,11 @@ Nix uses the [closure](store.md#closure) of build inputs to derive the list of a This way, scanning files captures run time dependencies without the user having to declare them explicitly. Doing it at build time and persisting references in the store object avoids repeating this time-consuming operation. -::: {.note} -In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. +> **Note** +> In practice, it is sometimes still necessary for users to declare certain dependencies explicitly, if they are to be preserved in the build result's closure. This depends on the specifics of the software to build and run. - -For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. -::: +> +> For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. ## Input Addressing {#input-addressing} From b63174227704c2a5fc11e815aaf3b4a6fd6f4576 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 4 Aug 2022 14:31:33 +0200 Subject: [PATCH 118/123] fix page rendering apparently it is not possible to link to page anchors with `mdBook`[1] [1]: https://github.com/rust-lang/mdBook/issues/167 --- doc/manual/src/SUMMARY.md.in | 8 +-- doc/manual/src/architecture/store/path.md | 2 +- doc/manual/src/architecture/store/store.md | 61 ------------------- .../src/architecture/store/store/closure.md | 29 +++++++++ .../architecture/store/store/terminology.md | 32 ++++++++++ 5 files changed, 64 insertions(+), 68 deletions(-) create mode 100644 doc/manual/src/architecture/store/store/closure.md create mode 100644 doc/manual/src/architecture/store/store/terminology.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 9c2ae2ad3..a95c385e4 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -68,12 +68,8 @@ - [nix.conf](command-ref/conf-file.md) - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - - [Store Object](architecture/store/store.md#store-object) - - [Reference](architecture/store/store.md#reference) - - [Operations](architecture/store/store.md#operations) - - [Closure](architecture/store/store.md#closure) - - [Files and Processes](architecture/store/store.md#files-and-processes) - - [Build system terminology](architecture/store/store.md#build-system-terminology) + - [Closure](architecture/store/store/closure.md) + - [Build system terminology](architecture/store/build-system-terminology.md) - [Store Path](architecture/store/path.md) - [Digest](architecture/store/path.md#digest) - [Input Addressing](architecture/store/path.md#input-addressing) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 5a18df67a..039d8b1c6 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -39,7 +39,7 @@ This means a store path is not just derived from the referenced store object its It is important which store a given store object belongs to: Files in the store object can contain store paths, and processes may read these paths. -Nix can only guarantee [referential integrity](store.md#closure) if store paths do not cross store boundaries. +Nix can only guarantee [referential integrity](store/closure.md) if store paths do not cross store boundaries. Therefore one can only copy store objects to a different store if diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 920d483d5..9f580f9da 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -89,35 +89,6 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] - -## Closure {#closure} - -Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. - -The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. - -Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: - -- A newly added store object cannot have references, unless it is a build task. - -- Build results must only refer to store objects in the closure of the build inputs. - - Building a store object will add appropriate references, according to the build task. - -- Store objects being copied must refer to objects already in the destination store. - - Recursive copying must either proceed in dependency order or be atomic. - -- We can only safely delete store objects which are not reachable from any reference still in use. - - - -[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity -[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) -[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object -[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type -[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier - ## Files and Processes {#files-and-processes} Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. @@ -178,35 +149,3 @@ Examples: To make store objects accessible to processes, stores ultimately have to expose store objects through the file system. -## A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} - -The Nix store's design is comparable to other build systems. -Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. - -The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. - -| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | -| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | -| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | -| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | -| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | -| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | -| build | build | build | application of `Build` | evaluation | -| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | - -All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). - -[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone -[bazel]: https://bazel.build/start/bazel-intro -[bazel-artifact]: https://bazel.build/reference/glossary#artifact -[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html -[bazel-action]: https://bazel.build/reference/glossary#action -[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph -[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph -[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache -[thunk]: https://en.m.wikipedia.org/wiki/Thunk -[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph -[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming -[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf -[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf -[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf diff --git a/doc/manual/src/architecture/store/store/closure.md b/doc/manual/src/architecture/store/store/closure.md new file mode 100644 index 000000000..065b95ffc --- /dev/null +++ b/doc/manual/src/architecture/store/store/closure.md @@ -0,0 +1,29 @@ +# Closure + +Nix stores ensure [referential integrity][referential-integrity]: for each store object in the store, all the store objects it references must also be in the store. + +The set of all store objects reachable by following references from a given initial set of store objects is called a *closure*. + +Adding, building, copying and deleting store objects must be done in a way that preserves referential integrity: + +- A newly added store object cannot have references, unless it is a build task. + +- Build results must only refer to store objects in the closure of the build inputs. + + Building a store object will add appropriate references, according to the build task. + +- Store objects being copied must refer to objects already in the destination store. + + Recursive copying must either proceed in dependency order or be atomic. + +- We can only safely delete store objects which are not reachable from any reference still in use. + + + +[referential-integrity]: https://en.m.wikipedia.org/wiki/Referential_integrity +[garbage-collection]: https://en.m.wikipedia.org/wiki/Garbage_collection_(computer_science) +[immutable-object]: https://en.m.wikipedia.org/wiki/Immutable_object +[opaque-data-type]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique-identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + + diff --git a/doc/manual/src/architecture/store/store/terminology.md b/doc/manual/src/architecture/store/store/terminology.md new file mode 100644 index 000000000..3dd7aa756 --- /dev/null +++ b/doc/manual/src/architecture/store/store/terminology.md @@ -0,0 +1,32 @@ +# A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} + +The Nix store's design is comparable to other build systems. +Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. + +The following translation table points out similarities and equivalent terms, to help clarify their meaning and inform consistent use in the future. + +| generic build system | Nix | [Bazel][bazel] | [Build Systems à la Carte][bsalc] | programming language | +| -------------------------------- | ---------------- | -------------------------------------------------------------------- | --------------------------------- | ------------------------ | +| data (build input, build result) | store object | [artifact][bazel-artifact] | value | value | +| build instructions | builder | ([depends on action type][bazel-actions]) | function | function | +| build task | derivation | [action][bazel-action] | `Task` | [thunk][thunk] | +| build plan | derivation graph | [action graph][bazel-action-graph], [build graph][bazel-build-graph] | `Tasks` | [call graph][call-graph] | +| build | build | build | application of `Build` | evaluation | +| persistence layer | store | [action cache][bazel-action-cache] | `Store` | heap | + +All of these systems share features of [declarative programming][declarative-programming] languages, a key insight first put forward by Eelco Dolstra et al. in [Imposing a Memory Management Discipline on Software Deployment][immdsd] (2004), elaborated in his PhD thesis [The Purely Functional Software Deployment Model][phd-thesis] (2006), and further refined by Andrey Mokhov et al. in [Build Systems à la Carte][bsalc] (2018). + +[rosetta-stone]: https://en.m.wikipedia.org/wiki/Rosetta_Stone +[bazel]: https://bazel.build/start/bazel-intro +[bazel-artifact]: https://bazel.build/reference/glossary#artifact +[bazel-actions]: https://docs.bazel.build/versions/main/skylark/lib/actions.html +[bazel-action]: https://bazel.build/reference/glossary#action +[bazel-action-graph]: https://bazel.build/reference/glossary#action-graph +[bazel-build-graph]: https://bazel.build/reference/glossary#build-graph +[bazel-action-cache]: https://bazel.build/reference/glossary#action-cache +[thunk]: https://en.m.wikipedia.org/wiki/Thunk +[call-graph]: https://en.m.wikipedia.org/wiki/Call_graph +[declarative-programming]: https://en.m.wikipedia.org/wiki/Declarative_programming +[immdsd]: https://edolstra.github.io/pubs/immdsd-icse2004-final.pdf +[phd-thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[bsalc]: https://www.microsoft.com/en-us/research/uploads/prod/2018/03/build-systems.pdf From b74a3f51c2ecb200ceb877bf5dda528664ef7e10 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:01:27 -0400 Subject: [PATCH 119/123] Fix gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ba8e95191..0c1b89ace 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ perl/Makefile.config /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/expressions/builtins.md +/doc/manual/src/language/builtins.md # /scripts/ /scripts/nix-profile.sh From b430a6743c59aebc08968da3cf01e476edbc36c2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:01:55 -0400 Subject: [PATCH 120/123] Remove sections within from SUMMARY --- doc/manual/src/SUMMARY.md.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 812a3d732..a47d39f31 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -62,11 +62,8 @@ - [Architecture](architecture/architecture.md) - [Store](architecture/store/store.md) - [Closure](architecture/store/store/closure.md) - - [Build system terminology](architecture/store/build-system-terminology.md) + - [Build system terminology](architecture/store/store/build-system-terminology.md) - [Store Path](architecture/store/path.md) - - [Digest](architecture/store/path.md#digest) - - [Input Addressing](architecture/store/path.md#input-addressing) - - [Content Addressing](architecture/store/path.md#content-addressing) - [File System Object](architecture/store/fso.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) From 016d7a8f78bee2181d1d60bcf9a8a59a4f18e91f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:19 -0400 Subject: [PATCH 121/123] Fix rosetta stone file name --- .../store/store/{terminology.md => build-system-terminology.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename doc/manual/src/architecture/store/store/{terminology.md => build-system-terminology.md} (97%) diff --git a/doc/manual/src/architecture/store/store/terminology.md b/doc/manual/src/architecture/store/store/build-system-terminology.md similarity index 97% rename from doc/manual/src/architecture/store/store/terminology.md rename to doc/manual/src/architecture/store/store/build-system-terminology.md index 3dd7aa756..eefbaa630 100644 --- a/doc/manual/src/architecture/store/store/terminology.md +++ b/doc/manual/src/architecture/store/store/build-system-terminology.md @@ -1,4 +1,4 @@ -# A [Rosetta stone][rosetta-stone] for build system terminology {#build-system-terminology} +# A [Rosetta stone][rosetta-stone] for build system terminology The Nix store's design is comparable to other build systems. Usage of terms is, for historic reasons, not entirely consistent within the Nix ecosystem, and still subject to slow change. From 6f6498f59c9263d3d4bdc6d2c3776e6abad2e516 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:38 -0400 Subject: [PATCH 122/123] Remove header fragments which is not needd --- doc/manual/src/architecture/store/path.md | 8 ++++---- doc/manual/src/architecture/store/store.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/architecture/store/path.md b/doc/manual/src/architecture/store/path.md index 039d8b1c6..663f04f46 100644 --- a/doc/manual/src/architecture/store/path.md +++ b/doc/manual/src/architecture/store/path.md @@ -26,7 +26,7 @@ Example: |--------| |------------------------------| |----------| store directory digest name -## Store Directory {#store-directory} +## Store Directory Every [store](./store.md) has a store directory. @@ -53,7 +53,7 @@ One cannot copy a store object to a store with a different store directory. Instead, it has to be rebuilt, together with all its dependencies. It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. -# Digest {#digest} +# Digest In a [store path](#store-path), the [digest][digest] is the output of a [cryptographic hash function][hash] of either all *inputs* involved in building the referenced store object or its actual *contents*. @@ -84,7 +84,7 @@ This depends on the specifics of the software to build and run. > > For example, Java programs are compressed after compilation, which obfuscates any store paths they may refer to and prevents Nix from automatically detecting them. -## Input Addressing {#input-addressing} +## Input Addressing Input addressing means that the digest derives from how the store object was produced, namely its build inputs and build plan. @@ -95,7 +95,7 @@ Nix has a custom serialisation format called Nix Archive (NAR) Store object references of this sort can *not* be validated from the content of the store object. Rather, a cryptographic signature has to be used to indicate that someone is vouching for the store object really being produced from a build plan with that digest. -## Content Addressing {#content-addressing} +## Content Addressing Content addressing means that the digest derives from the store object's contents, namely its file system objects and references. If one knows content addressing was used, one can recalculate the reference and thus verify the store object. diff --git a/doc/manual/src/architecture/store/store.md b/doc/manual/src/architecture/store/store.md index 9f580f9da..08b6701d5 100644 --- a/doc/manual/src/architecture/store/store.md +++ b/doc/manual/src/architecture/store/store.md @@ -25,7 +25,7 @@ Arrows indicate suggested reading order. [ input addressing ] [ content addressing ] ``` -## Store Object {#store-object} +## Store Object A store object can hold @@ -36,13 +36,13 @@ Store objects can be build inputs, build results, or build tasks. Store objects are [immutable][immutable-object]: once created, they do not change until they are deleted. -## Reference {#reference} +## Reference A store object reference is an [opaque][opaque-data-type], [unique identifier][unique-identifier]: The only way to obtain references is by adding or building store objects. A reference will always point to exactly one store object. -## Operations {#operations} +## Operations A Nix store can *add*, *retrieve*, and *delete* store objects. @@ -89,7 +89,7 @@ As it keeps track of references, it can [garbage-collect][garbage-collection] un [ store ] --> collect garbage --> [ store' ] -## Files and Processes {#files-and-processes} +## Files and Processes Nix maps between its store model and the [Unix paradigm][unix-paradigm] of [files and processes][file-descriptor], by encoding immutable store objects and opaque identifiers as file system primitives: files and directories, and paths. That allows processes to resolve references contained in files and thus access the contents of store objects. From 39d32ac4c63f4aa3784d114b19c0eca83e306ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Aug 2022 14:02:58 -0400 Subject: [PATCH 123/123] Add disclaimer that arch section is WIP and links may rot So we can iterate without worrying so much. --- doc/manual/src/architecture/architecture.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 96371056b..41deb07af 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -1,5 +1,7 @@ # Architecture +*(This chapter is unstable and a work in progress. Incoming links may rot.)* + This chapter describes how Nix works. It should help users understand why Nix behaves as it does, and it should help developers understand how to modify Nix and how to write similar tools.