diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
new file mode 100644
index 000000000..d0f1b09ca
--- /dev/null
+++ b/doc/manual/command-ref/conf-file.xml
@@ -0,0 +1,1236 @@
+
+
+
+
+ nix.conf
+ 5
+ Nix
+
+
+
+
+ nix.conf
+ Nix configuration file
+
+
+Description
+
+By default Nix reads settings from the following places:
+
+The system-wide configuration file
+sysconfdir/nix/nix.conf
+(i.e. /etc/nix/nix.conf on most systems), or
+$NIX_CONF_DIR/nix.conf if
+NIX_CONF_DIR is set. Values loaded in this file are not forwarded to the Nix daemon. The
+client assumes that the daemon has already loaded them.
+
+
+User-specific configuration files:
+
+
+ If NIX_USER_CONF_FILES is set, then each path separated by
+ : will be loaded in reverse order.
+
+
+
+ Otherwise it will look for nix/nix.conf files in
+ XDG_CONFIG_DIRS and XDG_CONFIG_HOME.
+
+ The default location is $HOME/.config/nix.conf if
+ those environment variables are unset.
+
+
+The configuration files consist of
+name =
+value pairs, one per line. Other
+files can be included with a line like include
+path, where
+path is interpreted relative to the current
+conf file and a missing file is an error unless
+!include is used instead.
+Comments start with a # character. Here is an
+example configuration file:
+
+
+keep-outputs = true # Nice for developers
+keep-derivations = true # Idem
+
+
+You can override settings on the command line using the
+ flag, e.g. --option keep-outputs
+false.
+
+The following settings are currently available:
+
+
+
+
+ allowed-uris
+
+
+
+ A list of URI prefixes to which access is allowed in
+ restricted evaluation mode. For example, when set to
+ https://github.com/NixOS, builtin functions
+ such as fetchGit are allowed to access
+ https://github.com/NixOS/patchelf.git.
+
+
+
+
+
+
+ allow-import-from-derivation
+
+ By default, Nix allows you to import from a derivation,
+ allowing building at evaluation time. With this option set to false, Nix will throw an error
+ when evaluating an expression that uses this feature, allowing users to ensure their evaluation
+ will not require any builds to take place.
+
+
+
+
+ allow-new-privileges
+
+ (Linux-specific.) By default, builders on Linux
+ cannot acquire new privileges by calling setuid/setgid programs or
+ programs that have file capabilities. For example, programs such
+ as sudo or ping will
+ fail. (Note that in sandbox builds, no such programs are available
+ unless you bind-mount them into the sandbox via the
+ option.) You can allow the
+ use of such programs by enabling this option. This is impure and
+ usually undesirable, but may be useful in certain scenarios
+ (e.g. to spin up containers or set up userspace network interfaces
+ in tests).
+
+
+
+
+ allowed-users
+
+
+
+ A list of names of users (separated by whitespace) that
+ are allowed to connect to the Nix daemon. As with the
+ option, you can specify groups by
+ prefixing them with @. Also, you can allow
+ all users by specifying *. The default is
+ *.
+
+ Note that trusted users are always allowed to connect.
+
+
+
+
+
+
+ auto-optimise-store
+
+ If set to true, Nix
+ automatically detects files in the store that have identical
+ contents, and replaces them with hard links to a single copy.
+ This saves disk space. If set to false (the
+ default), you can still run nix-store
+ --optimise to get rid of duplicate
+ files.
+
+
+
+
+ builders
+
+ A list of machines on which to perform builds. See for details.
+
+
+
+
+ builders-use-substitutes
+
+ If set to true, Nix will instruct
+ remote build machines to use their own binary substitutes if available. In
+ practical terms, this means that remote hosts will fetch as many build
+ dependencies as possible from their own substitutes (e.g, from
+ cache.nixos.org), instead of waiting for this host to
+ upload them all. This can drastically reduce build times if the network
+ connection between this computer and the remote build host is slow. Defaults
+ to false.
+
+
+
+ build-users-group
+
+ This options specifies the Unix group containing
+ the Nix build user accounts. In multi-user Nix installations,
+ builds should not be performed by the Nix account since that would
+ allow users to arbitrarily modify the Nix store and database by
+ supplying specially crafted builders; and they cannot be performed
+ by the calling user since that would allow him/her to influence
+ the build result.
+
+ Therefore, if this option is non-empty and specifies a valid
+ group, builds will be performed under the user accounts that are a
+ member of the group specified here (as listed in
+ /etc/group). Those user accounts should not
+ be used for any other purpose!
+
+ Nix will never run two builds under the same user account at
+ the same time. This is to prevent an obvious security hole: a
+ malicious user writing a Nix expression that modifies the build
+ result of a legitimate Nix expression being built by another user.
+ Therefore it is good to have as many Nix build user accounts as
+ you can spare. (Remember: uids are cheap.)
+
+ The build users should have permission to create files in
+ the Nix store, but not delete them. Therefore,
+ /nix/store should be owned by the Nix
+ account, its group should be the group specified here, and its
+ mode should be 1775.
+
+ If the build users group is empty, builds will be performed
+ under the uid of the Nix process (that is, the uid of the caller
+ if NIX_REMOTE is empty, the uid under which the Nix
+ daemon runs if NIX_REMOTE is
+ daemon). Obviously, this should not be used in
+ multi-user settings with untrusted users.
+
+
+
+
+
+
+ compress-build-log
+
+ If set to true (the default),
+ build logs written to /nix/var/log/nix/drvs
+ will be compressed on the fly using bzip2. Otherwise, they will
+ not be compressed.
+
+
+
+ connect-timeout
+
+
+
+ The timeout (in seconds) for establishing connections in
+ the binary cache substituter. It corresponds to
+ curl’s
+ option.
+
+
+
+
+
+
+ cores
+
+ Sets the value of the
+ NIX_BUILD_CORES environment variable in the
+ invocation of builders. Builders can use this variable at their
+ discretion to control the maximum amount of parallelism. For
+ instance, in Nixpkgs, if the derivation attribute
+ enableParallelBuilding is set to
+ true, the builder passes the
+ flag to GNU Make.
+ It can be overridden using the command line switch and
+ defaults to 1. The value 0
+ means that the builder should use all available CPU cores in the
+ system.
+
+ See also .
+
+
+ diff-hook
+
+
+ Absolute path to an executable capable of diffing build results.
+ The hook executes if is
+ true, and the output of a build is known to not be the same.
+ This program is not executed to determine if two results are the
+ same.
+
+
+
+ The diff hook is executed by the same user and group who ran the
+ build. However, the diff hook does not have write access to the
+ store path just built.
+
+
+ The diff hook program receives three parameters:
+
+
+
+
+ A path to the previous build's results
+
+
+
+
+
+ A path to the current build's results
+
+
+
+
+
+ The path to the build's derivation
+
+
+
+
+
+ The path to the build's scratch directory. This directory
+ will exist only if the build was run with
+ .
+
+
+
+
+
+ The stderr and stdout output from the diff hook will not be
+ displayed to the user. Instead, it will print to the nix-daemon's
+ log.
+
+
+ When using the Nix daemon, diff-hook must
+ be set in the nix.conf configuration file, and
+ cannot be passed at the command line.
+
+
+
+
+
+ enforce-determinism
+
+ See .
+
+
+
+ extra-sandbox-paths
+
+ A list of additional paths appended to
+ . Useful if you want to extend
+ its default value.
+
+
+
+
+ extra-platforms
+
+ Platforms other than the native one which
+ this machine is capable of building for. This can be useful for
+ supporting additional architectures on compatible machines:
+ i686-linux can be built on x86_64-linux machines (and the default
+ for this setting reflects this); armv7 is backwards-compatible with
+ armv6 and armv5tel; some aarch64 machines can also natively run
+ 32-bit ARM code; and qemu-user may be used to support non-native
+ platforms (though this may be slow and buggy). Most values for this
+ are not enabled by default because build systems will often
+ misdetect the target platform and generate incompatible code, so you
+ may wish to cross-check the results of using this option against
+ proper natively-built versions of your
+ derivations.
+
+
+
+
+ extra-substituters
+
+ Additional binary caches appended to those
+ specified in . When used by
+ unprivileged users, untrusted substituters (i.e. those not listed
+ in ) are silently
+ ignored.
+
+
+
+ fallback
+
+ If set to true, Nix will fall
+ back to building from source if a binary substitute fails. This
+ is equivalent to the flag. The
+ default is false.
+
+
+
+ fsync-metadata
+
+ If set to true, changes to the
+ Nix store metadata (in /nix/var/nix/db) are
+ synchronously flushed to disk. This improves robustness in case
+ of system crashes, but reduces performance. The default is
+ true.
+
+
+
+ hashed-mirrors
+
+ A list of web servers used by
+ builtins.fetchurl to obtain files by hash.
+ Given a hash type ht and a base-16 hash
+ h, Nix will try to download the file
+ from
+ hashed-mirror/ht/h.
+ This allows files to be downloaded even if they have disappeared
+ from their original URI. For example, given the hashed mirror
+ http://tarballs.example.com/, when building the
+ derivation
+
+
+builtins.fetchurl {
+ url = "https://example.org/foo-1.2.3.tar.xz";
+ sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+}
+
+
+ Nix will attempt to download this file from
+ http://tarballs.example.com/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae
+ first. If it is not available there, if will try the original URI.
+
+
+
+
+ http-connections
+
+ The maximum number of parallel TCP connections
+ used to fetch files from binary caches and by other downloads. It
+ defaults to 25. 0 means no limit.
+
+
+
+
+ keep-build-log
+
+ If set to true (the default),
+ Nix will write the build log of a derivation (i.e. the standard
+ output and error of its builder) to the directory
+ /nix/var/log/nix/drvs. The build log can be
+ retrieved using the command nix-store -l
+ path.
+
+
+
+
+ keep-derivations
+
+ If true (default), the garbage
+ collector will keep the derivations from which non-garbage store
+ paths were built. If false, they will be
+ deleted unless explicitly registered as a root (or reachable from
+ other roots).
+
+ Keeping derivation around is useful for querying and
+ traceability (e.g., it allows you to ask with what dependencies or
+ options a store path was built), so by default this option is on.
+ Turn it off to save a bit of disk space (or a lot if
+ keep-outputs is also turned on).
+
+
+ keep-env-derivations
+
+ If false (default), derivations
+ are not stored in Nix user environments. That is, the derivations of
+ any build-time-only dependencies may be garbage-collected.
+
+ If true, when you add a Nix derivation to
+ a user environment, the path of the derivation is stored in the
+ user environment. Thus, the derivation will not be
+ garbage-collected until the user environment generation is deleted
+ (nix-env --delete-generations). To prevent
+ build-time-only dependencies from being collected, you should also
+ turn on keep-outputs.
+
+ The difference between this option and
+ keep-derivations is that this one is
+ “sticky”: it applies to any user environment created while this
+ option was enabled, while keep-derivations
+ only applies at the moment the garbage collector is
+ run.
+
+
+
+ keep-outputs
+
+ If true, the garbage collector
+ will keep the outputs of non-garbage derivations. If
+ false (default), outputs will be deleted unless
+ they are GC roots themselves (or reachable from other roots).
+
+ In general, outputs must be registered as roots separately.
+ However, even if the output of a derivation is registered as a
+ root, the collector will still delete store paths that are used
+ only at build time (e.g., the C compiler, or source tarballs
+ downloaded from the network). To prevent it from doing so, set
+ this option to true.
+
+
+ max-build-log-size
+
+
+
+ This option defines the maximum number of bytes that a
+ builder can write to its stdout/stderr. If the builder exceeds
+ this limit, it’s killed. A value of 0 (the
+ default) means that there is no limit.
+
+
+
+
+
+ max-free
+
+ When a garbage collection is triggered by the
+ min-free option, it stops as soon as
+ max-free bytes are available. The default is
+ infinity (i.e. delete all garbage).
+
+
+
+ max-jobs
+
+ This option defines the maximum number of jobs
+ that Nix will try to build in parallel. The default is
+ 1. The special value auto
+ causes Nix to use the number of CPUs in your system. 0
+ is useful when using remote builders to prevent any local builds (except for
+ preferLocalBuild derivation attribute which executes locally
+ regardless). It can be
+ overridden using the ()
+ command line switch.
+
+ See also .
+
+
+
+ max-silent-time
+
+
+
+ This option defines the maximum number of seconds that a
+ builder can go without producing any data on standard output or
+ standard error. This is useful (for instance in an automated
+ build system) to catch builds that are stuck in an infinite
+ loop, or to catch remote builds that are hanging due to network
+ problems. It can be overridden using the command
+ line switch.
+
+ The value 0 means that there is no
+ timeout. This is also the default.
+
+
+
+
+
+ min-free
+
+
+ When free disk space in /nix/store
+ drops below min-free during a build, Nix
+ performs a garbage-collection until max-free
+ bytes are available or there is no more garbage. A value of
+ 0 (the default) disables this feature.
+
+
+
+
+ narinfo-cache-negative-ttl
+
+
+
+ The TTL in seconds for negative lookups. If a store path is
+ queried from a substituter but was not found, there will be a
+ negative lookup cached in the local disk cache database for the
+ specified duration.
+
+
+
+
+
+ narinfo-cache-positive-ttl
+
+
+
+ The TTL in seconds for positive lookups. If a store path is
+ queried from a substituter, the result of the query will be cached
+ in the local disk cache database including some of the NAR
+ metadata. The default TTL is a month, setting a shorter TTL for
+ positive lookups can be useful for binary caches that have
+ frequent garbage collection, in which case having a more frequent
+ cache invalidation would prevent trying to pull the path again and
+ failing with a hash mismatch if the build isn't reproducible.
+
+
+
+
+
+
+ netrc-file
+
+ If set to an absolute path to a netrc
+ file, Nix will use the HTTP authentication credentials in this file when
+ trying to download from a remote host through HTTP or HTTPS. Defaults to
+ $NIX_CONF_DIR/netrc.
+
+ The netrc file consists of a list of
+ accounts in the following format:
+
+
+machine my-machine
+login my-username
+password my-password
+
+
+ For the exact syntax, see the
+ curl documentation.
+
+ This must be an absolute path, and ~
+ is not resolved. For example, ~/.netrc won't
+ resolve to your home directory's .netrc.
+
+
+
+
+
+
+ plugin-files
+
+
+ A list of plugin files to be loaded by Nix. Each of these
+ files will be dlopened by Nix, allowing them to affect
+ execution through static initialization. In particular, these
+ plugins may construct static instances of RegisterPrimOp to
+ add new primops or constants to the expression language,
+ RegisterStoreImplementation to add new store implementations,
+ RegisterCommand to add new subcommands to the
+ nix command, and RegisterSetting to add new
+ nix config settings. See the constructors for those types for
+ more details.
+
+
+ Since these files are loaded into the same address space as
+ Nix itself, they must be DSOs compatible with the instance of
+ Nix running at the time (i.e. compiled against the same
+ headers, not linked to any incompatible libraries). They
+ should not be linked to any Nix libs directly, as those will
+ be available already at load time.
+
+
+ If an entry in the list is a directory, all files in the
+ directory are loaded as plugins (non-recursively).
+
+
+
+
+
+ pre-build-hook
+
+
+
+
+ If set, the path to a program that can set extra
+ derivation-specific settings for this system. This is used for settings
+ that can't be captured by the derivation model itself and are too variable
+ between different versions of the same system to be hard-coded into nix.
+
+
+ The hook is passed the derivation path and, if sandboxes are enabled,
+ the sandbox directory. It can then modify the sandbox and send a series of
+ commands to modify various settings to stdout. The currently recognized
+ commands are:
+
+
+
+ extra-sandbox-paths
+
+
+
+ Pass a list of files and directories to be included in the
+ sandbox for this build. One entry per line, terminated by an empty
+ line. Entries have the same format as
+ sandbox-paths.
+
+
+
+
+
+
+
+
+
+
+ post-build-hook
+
+ Optional. The path to a program to execute after each build.
+
+ This option is only settable in the global
+ nix.conf, or on the command line by trusted
+ users.
+
+ When using the nix-daemon, the daemon executes the hook as
+ root. If the nix-daemon is not involved, the
+ hook runs as the user executing the nix-build.
+
+
+ The hook executes after an evaluation-time build.
+ The hook does not execute on substituted paths.
+ The hook's output always goes to the user's terminal.
+ If the hook fails, the build succeeds but no further builds execute.
+ The hook executes synchronously, and blocks other builds from progressing while it runs.
+
+
+ The program executes with no arguments. The program's environment
+ contains the following environment variables:
+
+
+
+ DRV_PATH
+
+ The derivation for the built paths.
+ Example:
+ /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv
+
+
+
+
+
+ OUT_PATHS
+
+ Output paths of the built derivation, separated by a space character.
+ Example:
+ /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev
+ /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc
+ /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info
+ /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man
+ /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23.
+
+
+
+
+
+ See for an example
+ implementation.
+
+
+
+
+ repeat
+
+ How many times to repeat builds to check whether
+ they are deterministic. The default value is 0. If the value is
+ non-zero, every build is repeated the specified number of
+ times. If the contents of any of the runs differs from the
+ previous ones and is
+ true, the build is rejected and the resulting store paths are not
+ registered as “valid” in Nix’s database.
+
+
+ require-sigs
+
+ If set to true (the default),
+ any non-content-addressed path added or copied to the Nix store
+ (e.g. when substituting from a binary cache) must have a valid
+ signature, that is, be signed using one of the keys listed in
+ or
+ . Set to false
+ to disable signature checking.
+
+
+
+
+ restrict-eval
+
+
+
+ If set to true, the Nix evaluator will
+ not allow access to any files outside of the Nix search path (as
+ set via the NIX_PATH environment variable or the
+ option), or to URIs outside of
+ . The default is
+ false.
+
+
+
+
+
+ run-diff-hook
+
+
+ If true, enable the execution of .
+
+
+
+ When using the Nix daemon, run-diff-hook must
+ be set in the nix.conf configuration file,
+ and cannot be passed at the command line.
+
+
+
+
+ sandbox
+
+ If set to true, builds will be
+ performed in a sandboxed environment, i.e.,
+ they’re isolated from the normal file system hierarchy and will
+ only see their dependencies in the Nix store, the temporary build
+ directory, private versions of /proc,
+ /dev, /dev/shm and
+ /dev/pts (on Linux), and the paths configured with the
+ sandbox-paths
+ option. This is useful to prevent undeclared dependencies
+ on files in directories such as /usr/bin. In
+ addition, on Linux, builds run in private PID, mount, network, IPC
+ and UTS namespaces to isolate them from other processes in the
+ system (except that fixed-output derivations do not run in private
+ network namespace to ensure they can access the network).
+
+ Currently, sandboxing only work on Linux and macOS. The use
+ of a sandbox requires that Nix is run as root (so you should use
+ the “build users”
+ feature to perform the actual builds under different users
+ than root).
+
+ If this option is set to relaxed, then
+ fixed-output derivations and derivations that have the
+ __noChroot attribute set to
+ true do not run in sandboxes.
+
+ The default is true on Linux and
+ false on all other platforms.
+
+
+
+
+
+ sandbox-dev-shm-size
+
+ This option determines the maximum size of the
+ tmpfs filesystem mounted on
+ /dev/shm in Linux sandboxes. For the format,
+ see the description of the option of
+ tmpfs in
+ mount8. The
+ default is 50%.
+
+
+
+
+
+ sandbox-paths
+
+ A list of paths bind-mounted into Nix sandbox
+ environments. You can use the syntax
+ target=source
+ to mount a path in a different location in the sandbox; for
+ instance, /bin=/nix-bin will mount the path
+ /nix-bin as /bin inside the
+ sandbox. If source is followed by
+ ?, then it is not an error if
+ source does not exist; for example,
+ /dev/nvidiactl? specifies that
+ /dev/nvidiactl will only be mounted in the
+ sandbox if it exists in the host filesystem.
+
+ Depending on how Nix was built, the default value for this option
+ may be empty or provide /bin/sh as a
+ bind-mount of bash.
+
+
+
+
+ secret-key-files
+
+ A whitespace-separated list of files containing
+ secret (private) keys. These are used to sign locally-built
+ paths. They can be generated using nix-store
+ --generate-binary-cache-key. The corresponding public
+ key can be distributed to other users, who can add it to
+ in their
+ nix.conf.
+
+
+
+
+ show-trace
+
+ Causes Nix to print out a stack trace in case of Nix
+ expression evaluation errors.
+
+
+
+
+ substitute
+
+ If set to true (default), Nix
+ will use binary substitutes if available. This option can be
+ disabled to force building from source.
+
+
+
+ stalled-download-timeout
+
+ The timeout (in seconds) for receiving data from servers
+ during download. Nix cancels idle downloads after this timeout's
+ duration.
+
+
+
+ substituters
+
+ A list of URLs of substituters, separated by
+ whitespace. The default is
+ https://cache.nixos.org.
+
+
+
+ system
+
+ This option specifies the canonical Nix system
+ name of the current installation, such as
+ i686-linux or
+ x86_64-darwin. Nix can only build derivations
+ whose system attribute equals the value
+ specified here. In general, it never makes sense to modify this
+ value from its default, since you can use it to ‘lie’ about the
+ platform you are building on (e.g., perform a Mac OS build on a
+ Linux machine; the result would obviously be wrong). It only
+ makes sense if the Nix binaries can run on multiple platforms,
+ e.g., ‘universal binaries’ that run on x86_64-linux and
+ i686-linux.
+
+ It defaults to the canonical Nix system name detected by
+ configure at build time.
+
+
+
+
+ system-features
+
+ A set of system “features” supported by this
+ machine, e.g. kvm. Derivations can express a
+ dependency on such features through the derivation attribute
+ requiredSystemFeatures. For example, the
+ attribute
+
+
+requiredSystemFeatures = [ "kvm" ];
+
+
+ ensures that the derivation can only be built on a machine with
+ the kvm feature.
+
+ This setting by default includes kvm if
+ /dev/kvm is accessible, and the
+ pseudo-features nixos-test,
+ benchmark and big-parallel
+ that are used in Nixpkgs to route builds to specific
+ machines.
+
+
+
+
+
+ tarball-ttl
+
+
+ Default: 3600 seconds.
+
+ The number of seconds a downloaded tarball is considered
+ fresh. If the cached tarball is stale, Nix will check whether
+ it is still up to date using the ETag header. Nix will download
+ a new version if the ETag header is unsupported, or the
+ cached ETag doesn't match.
+
+
+ Setting the TTL to 0 forces Nix to always
+ check if the tarball is up to date.
+
+ Nix caches tarballs in
+ $XDG_CACHE_HOME/nix/tarballs.
+
+ Files fetched via NIX_PATH,
+ fetchGit, fetchMercurial,
+ fetchTarball, and fetchurl
+ respect this TTL.
+
+
+
+
+ timeout
+
+
+
+ This option defines the maximum number of seconds that a
+ builder can run. This is useful (for instance in an automated
+ build system) to catch builds that are stuck in an infinite loop
+ but keep writing to their standard output or standard error. It
+ can be overridden using the command line
+ switch.
+
+ The value 0 means that there is no
+ timeout. This is also the default.
+
+
+
+
+
+ trace-function-calls
+
+
+
+ Default: false.
+
+ If set to true, the Nix evaluator will
+ trace every function call. Nix will print a log message at the
+ "vomit" level for every function entrance and function exit.
+
+
+function-trace entered undefined position at 1565795816999559622
+function-trace exited undefined position at 1565795816999581277
+function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+
+
+ The undefined position means the function
+ call is a builtin.
+
+ Use the contrib/stack-collapse.py script
+ distributed with the Nix source code to convert the trace logs
+ in to a format suitable for flamegraph.pl.
+
+
+
+
+
+ trusted-public-keys
+
+ A whitespace-separated list of public keys. When
+ paths are copied from another Nix store (such as a binary cache),
+ they must be signed with one of these keys. For example:
+ cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+ hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=.
+
+
+
+ trusted-substituters
+
+ A list of URLs of substituters, separated by
+ whitespace. These are not used by default, but can be enabled by
+ users of the Nix daemon by specifying --option
+ substituters urls on the
+ command line. Unprivileged users are only allowed to pass a
+ subset of the URLs listed in substituters and
+ trusted-substituters.
+
+
+
+ trusted-users
+
+
+
+ A list of names of users (separated by whitespace) that
+ have additional rights when connecting to the Nix daemon, such
+ as the ability to specify additional binary caches, or to import
+ unsigned NARs. You can also specify groups by prefixing them
+ with @; for instance,
+ @wheel means all users in the
+ wheel group. The default is
+ root.
+
+ Adding a user to
+ is essentially equivalent to giving that user root access to the
+ system. For example, the user can set
+ and thereby obtain read access to
+ directories that are otherwise inacessible to
+ them.
+
+
+
+
+
+
+
+
+
+ Deprecated Settings
+
+
+
+
+
+
+ binary-caches
+
+ Deprecated:
+ binary-caches is now an alias to
+ .
+
+
+
+ binary-cache-public-keys
+
+ Deprecated:
+ binary-cache-public-keys is now an alias to
+ .
+
+
+
+ build-compress-log
+
+ Deprecated:
+ build-compress-log is now an alias to
+ .
+
+
+
+ build-cores
+
+ Deprecated:
+ build-cores is now an alias to
+ .
+
+
+
+ build-extra-chroot-dirs
+
+ Deprecated:
+ build-extra-chroot-dirs is now an alias to
+ .
+
+
+
+ build-extra-sandbox-paths
+
+ Deprecated:
+ build-extra-sandbox-paths is now an alias to
+ .
+
+
+
+ build-fallback
+
+ Deprecated:
+ build-fallback is now an alias to
+ .
+
+
+
+ build-max-jobs
+
+ Deprecated:
+ build-max-jobs is now an alias to
+ .
+
+
+
+ build-max-log-size
+
+ Deprecated:
+ build-max-log-size is now an alias to
+ .
+
+
+
+ build-max-silent-time
+
+ Deprecated:
+ build-max-silent-time is now an alias to
+ .
+
+
+
+ build-repeat
+
+ Deprecated:
+ build-repeat is now an alias to
+ .
+
+
+
+ build-timeout
+
+ Deprecated:
+ build-timeout is now an alias to
+ .
+
+
+
+ build-use-chroot
+
+ Deprecated:
+ build-use-chroot is now an alias to
+ .
+
+
+
+ build-use-sandbox
+
+ Deprecated:
+ build-use-sandbox is now an alias to
+ .
+
+
+
+ build-use-substitutes
+
+ Deprecated:
+ build-use-substitutes is now an alias to
+ .
+
+
+
+ gc-keep-derivations
+
+ Deprecated:
+ gc-keep-derivations is now an alias to
+ .
+
+
+
+ gc-keep-outputs
+
+ Deprecated:
+ gc-keep-outputs is now an alias to
+ .
+
+
+
+ env-keep-derivations
+
+ Deprecated:
+ env-keep-derivations is now an alias to
+ .
+
+
+
+ extra-binary-caches
+
+ Deprecated:
+ extra-binary-caches is now an alias to
+ .
+
+
+
+ trusted-binary-caches
+
+ Deprecated:
+ trusted-binary-caches is now an alias to
+ .
+
+
+
+
+
+
+
+
diff --git a/doc/manual/src/command-ref/conf-file.md b/doc/manual/src/command-ref/conf-file.md
index ee8803e19..fdc8f5265 100644
--- a/doc/manual/src/command-ref/conf-file.md
+++ b/doc/manual/src/command-ref/conf-file.md
@@ -206,6 +206,26 @@ The following settings are currently available:
robustness in case of system crashes, but reduces performance. The
default is `true`.
+ - `hashed-mirrors`
+ A list of web servers used by `builtins.fetchurl` to obtain files by
+ hash. The default is `http://tarballs.nixos.org/`. Given a hash type
+ *ht* and a base-16 hash *h*, Nix will try to download the file from
+ *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if
+ they have disappeared from their original URI. For example, given
+ the default mirror `http://tarballs.nixos.org/`, when building the
+ derivation
+
+ ```nix
+ builtins.fetchurl {
+ url = "https://example.org/foo-1.2.3.tar.xz";
+ sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
+ }
+ ```
+
+ Nix will attempt to download this file from
+ `http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae`
+ first. If it is not available there, if will try the original URI.
+
- `http-connections`
The maximum number of parallel TCP connections used to fetch files
from binary caches and by other downloads. It defaults to 25. 0
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index c412e4640..4e038866e 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
- auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
+ auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef);
else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
- auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
+ auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
@@ -224,7 +224,7 @@ SV * hashString(char * algo, int base32, char * s)
SV * convertHash(char * algo, char * s, int toBase32)
PPCODE:
try {
- Hash h(s, parseHashType(algo));
+ auto h = Hash::parseAny(s, parseHashType(algo));
string s = h.to_string(toBase32 ? Base32 : Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
@@ -285,7 +285,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
PPCODE:
try {
- Hash h(hash, parseHashType(algo));
+ auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(method, h, name);
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
@@ -303,10 +303,10 @@ SV * derivationFromPath(char * drvPath)
hash = newHV();
HV * outputs = newHV();
- for (auto & i : drv.outputs)
+ for (auto & i : drv.outputsAndPaths(*store()))
hv_store(
outputs, i.first.c_str(), i.first.size(),
- newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
+ newSVpv(store()->printStorePath(i.second.second).c_str(), 0),
0);
hv_stores(hash, "outputs", newRV((SV *) outputs));
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 00c9d540b..e5cc4d7ed 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_CACERT="@cacert@"
+#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
+#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
readonly ROOT_HOME=$(echo ~root)
@@ -69,9 +71,11 @@ uninstall_directions() {
subheader "Uninstalling nix:"
local step=0
- if poly_service_installed_check; then
+ if [ -e /run/systemd/system ] && poly_service_installed_check; then
step=$((step + 1))
poly_service_uninstall_directions "$step"
+ else
+ step=$((step + 1))
fi
for profile_target in "${PROFILE_TARGETS[@]}"; do
@@ -250,7 +254,9 @@ function finish_success {
echo "But fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
fi
- cat <&2
-elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+elif [ "$(uname -s)" = "Linux" ]; then
echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
fi
@@ -122,7 +122,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
fi
if [ "$INSTALL_MODE" = "daemon" ]; then
- printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+ printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n'
exec "$self/install-multi-user"
exit 0
fi
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 3579d8fff..ce5127113 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -38,9 +38,9 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
-static bool allSupportedLocally(const std::set& requiredFeatures) {
+static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) {
for (auto & feature : requiredFeatures)
- if (!settings.systemFeatures.get().count(feature)) return false;
+ if (!store.systemFeatures.get().count(feature)) return false;
return true;
}
@@ -103,10 +103,10 @@ static int _main(int argc, char * * argv)
drvPath = store->parseStorePath(readString(source));
auto requiredFeatures = readStrings>(source);
- auto canBuildLocally = amWilling
+ auto canBuildLocally = amWilling
&& ( neededSystem == settings.thisSystem
|| settings.extraPlatforms.get().count(neededSystem) > 0)
- && allSupportedLocally(requiredFeatures);
+ && allSupportedLocally(*store, requiredFeatures);
/* Error ignored here, will be caught later */
mkdir(currentLoad.c_str(), 0777);
@@ -170,7 +170,45 @@ static int _main(int argc, char * * argv)
if (rightType && !canBuildLocally)
std::cerr << "# postpone\n";
else
+ {
+ // build the hint template.
+ string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)";
+ hintstring += "\n%s available machines:";
+ hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+
+ for (unsigned int i = 0; i < machines.size(); ++i) {
+ hintstring += "\n(%s, %s, %s, %s)";
+ }
+
+ // add the template values.
+ string drvstr;
+ if (drvPath.has_value())
+ drvstr = drvPath->to_string();
+ else
+ drvstr = "";
+
+ auto hint = hintformat(hintstring);
+ hint
+ % drvstr
+ % neededSystem
+ % concatStringsSep(", ", requiredFeatures)
+ % machines.size();
+
+ for (auto & m : machines) {
+ hint % concatStringsSep>(", ", m.systemTypes)
+ % m.maxJobs
+ % concatStringsSep(", ", m.supportedFeatures)
+ % concatStringsSep(", ", m.mandatoryFeatures);
+ }
+
+ logErrorInfo(lvlInfo, {
+ .name = "Remote build",
+ .description = "Failed to find a machine for remote build!",
+ .hint = hint
+ });
+
std::cerr << "# decline\n";
+ }
break;
}
@@ -186,15 +224,7 @@ static int _main(int argc, char * * argv)
Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri));
- Store::Params storeParams;
- if (hasPrefix(bestMachine->storeUri, "ssh://")) {
- storeParams["max-connections"] = "1";
- storeParams["log-fd"] = "4";
- if (bestMachine->sshKey != "")
- storeParams["ssh-key"] = bestMachine->sshKey;
- }
-
- sshStore = openStore(bestMachine->storeUri, storeParams);
+ sshStore = bestMachine->openStore();
sshStore->connect();
storeUri = bestMachine->storeUri;
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index deb32484f..46177a0a4 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -405,7 +405,7 @@ Value & AttrCursor::forceValue()
return v;
}
-std::shared_ptr AttrCursor::maybeGetAttr(Symbol name)
+std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors)
{
if (root->db) {
if (!cachedValue)
@@ -422,9 +422,12 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name)
if (attr) {
if (std::get_if(&attr->second))
return nullptr;
- else if (std::get_if(&attr->second))
- throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
- else
+ else if (std::get_if(&attr->second)) {
+ if (forceErrors)
+ debug("reevaluating failed cached attribute '%s'");
+ else
+ throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+ } else
return std::make_shared(root,
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
}
@@ -469,9 +472,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name)
return maybeGetAttr(root->state.symbols.create(name));
}
-std::shared_ptr AttrCursor::getAttr(Symbol name)
+std::shared_ptr AttrCursor::getAttr(Symbol name, bool forceErrors)
{
- auto p = maybeGetAttr(name);
+ auto p = maybeGetAttr(name, forceErrors);
if (!p)
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
return p;
@@ -600,7 +603,7 @@ bool AttrCursor::isDerivation()
StorePath AttrCursor::forceDerivation()
{
- auto aDrvPath = getAttr(root->state.sDrvPath);
+ auto aDrvPath = getAttr(root->state.sDrvPath, true);
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
/* The eval cache contains 'drvPath', but the actual path has
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index afee85fa9..8ffffc0ed 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -9,6 +9,8 @@
namespace nix::eval_cache {
+MakeError(CachedEvalError, EvalError);
+
class AttrDb;
class AttrCursor;
@@ -92,11 +94,11 @@ public:
std::string getAttrPathStr(Symbol name) const;
- std::shared_ptr maybeGetAttr(Symbol name);
+ std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false);
std::shared_ptr maybeGetAttr(std::string_view name);
- std::shared_ptr getAttr(Symbol name);
+ std::shared_ptr getAttr(Symbol name, bool forceErrors = false);
std::shared_ptr getAttr(std::string_view name);
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 7a2f55504..0123070d1 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -345,6 +345,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store)
, sStructuredAttrs(symbols.create("__structuredAttrs"))
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
+ , sContentAddressed(symbols.create("__contentAddressed"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
@@ -1256,10 +1257,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
try {
lambda.body->eval(*this, env2, v);
} catch (Error & e) {
- addErrorTrace(e, lambda.pos, "while evaluating %s",
- (lambda.name.set()
- ? "'" + (string) lambda.name + "'"
- : "anonymous lambdaction"));
+ addErrorTrace(e, lambda.pos, "while evaluating %s",
+ (lambda.name.set()
+ ? "'" + (string) lambda.name + "'"
+ : "anonymous lambda"));
addErrorTrace(e, pos, "from call site%s", "");
throw;
}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 8986952e3..5855b4ef2 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -74,6 +74,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+ sContentAddressed,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
sDescription, sSelf, sEpsilon;
@@ -374,6 +375,9 @@ struct EvalSettings : Config
Setting traceFunctionCalls{this, false, "trace-function-calls",
"Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."};
+
+ Setting useEvalCache{this, true, "eval-cache",
+ "Whether to use the flake evaluation cache."};
};
extern EvalSettings evalSettings;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 05d499d1f..4e35dedb0 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -113,9 +113,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
state.mkList(*outputsVal, drv.outputs.size());
unsigned int outputs_index = 0;
- for (const auto & o : drv.outputs) {
+ for (const auto & o : drv.outputsAndPaths(*state.store)) {
v2 = state.allocAttr(w, state.symbols.create(o.first));
- mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
+ mkString(*v2, state.store->printStorePath(o.second.second), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
}
@@ -583,6 +583,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context;
+ bool contentAddressed = false;
std::optional outputHash;
std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat;
@@ -639,9 +640,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (i->value->type == tNull) continue;
}
+ if (i->name == state.sContentAddressed) {
+ settings.requireExperimentalFeature("ca-derivations");
+ contentAddressed = state.forceBool(*i->value, pos);
+ }
+
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
- if (i->name == state.sArgs) {
+ else if (i->name == state.sArgs) {
state.forceList(*i->value, pos);
for (unsigned int n = 0; n < i->value->listSize(); ++n) {
string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
@@ -694,7 +700,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
} catch (Error & e) {
- e.addTrace(posDrvName,
+ e.addTrace(posDrvName,
"while evaluating the attribute '%1%' of the derivation '%2%'",
key, drvName);
throw;
@@ -761,7 +767,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
if (outputHash) {
- /* Handle fixed-output derivations. */
+ /* Handle fixed-output derivations.
+
+ Ignore `__contentAddressed` because fixed output derivations are
+ already content addressed. */
if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error({
.hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
@@ -774,7 +783,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
- .output = DerivationOutputFixed {
+ .output = DerivationOutputCAFixed {
.hash = FixedOutputHash {
.method = ingestionMethod,
.hash = std::move(h),
@@ -783,6 +792,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
+ else if (contentAddressed) {
+ HashType ht = parseHashType(outputHashAlgo);
+ for (auto & i : outputs) {
+ if (!jsonObject) drv.env[i] = hashPlaceholder(i);
+ drv.outputs.insert_or_assign(i, DerivationOutput {
+ .output = DerivationOutputCAFloating {
+ .method = ingestionMethod,
+ .hashType = std::move(ht),
+ },
+ });
+ }
+ }
+
else {
/* Compute a hash over the "masked" store derivation, which is
the final one except that in the list of outputs, the
@@ -800,7 +822,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
- Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
+ // Regular, non-CA derivation should always return a single hash and not
+ // hash per output.
+ Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
@@ -815,7 +839,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
/* Write the resulting term into the Nix store directory. */
- auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
+ auto drvPath = writeDerivation(state.store, drv, state.repair);
auto drvPathS = state.store->printStorePath(drvPath);
printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);
@@ -828,9 +852,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
- for (auto & i : drv.outputs) {
+ for (auto & i : drv.outputsAndPaths(*state.store)) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
- state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
+ state.store->printStorePath(i.second.second), {"!" + i.first + "!" + drvPathS});
}
v.attrs->sort();
}
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index fc2a6a1c2..cef85cfef 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
// be both a revision or a branch/tag name.
auto value = state.forceStringNoCtx(*attr.value, *attr.pos);
if (std::regex_match(value, revRegex))
- rev = Hash(value, htSHA1);
+ rev = Hash::parseAny(value, htSHA1);
else
ref = value;
}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index cddcf0e59..0dbf4ae1d 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -212,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
- *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
+ *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
if (state.allowedPaths)
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 28db8aa9c..eaa635595 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -130,12 +130,12 @@ std::pair Input::fetch(ref store) const
tree.actualPath = store->toRealPath(tree.storePath);
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
- input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
+ input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
}
if (auto prevLastModified = getLastModified()) {
@@ -200,9 +200,12 @@ std::string Input::getType() const
std::optional Input::getNarHash() const
{
- if (auto s = maybeGetStrAttr(attrs, "narHash"))
- // FIXME: require SRI hash.
- return newHashAllowEmpty(*s, htSHA256);
+ if (auto s = maybeGetStrAttr(attrs, "narHash")) {
+ auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s);
+ if (hash.type != htSHA256)
+ throw UsageError("narHash must use SHA-256");
+ return hash;
+ }
return {};
}
@@ -216,7 +219,7 @@ std::optional Input::getRef() const
std::optional Input::getRev() const
{
if (auto s = maybeGetStrAttr(attrs, "rev"))
- return Hash(*s, htSHA1);
+ return Hash::parseAny(*s, htSHA1);
return {};
}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 5d38e0c2b..5ca0f8521 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -121,7 +121,7 @@ struct GitInputScheme : InputScheme
args.push_back(*ref);
}
- if (input.getRev()) throw Error("cloning a specific revision is not implemented");
+ if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented");
args.push_back(destDir);
@@ -269,7 +269,7 @@ struct GitInputScheme : InputScheme
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
- haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
+ haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {
Tree(store->printStorePath(storePath), std::move(storePath)),
@@ -293,14 +293,14 @@ struct GitInputScheme : InputScheme
if (!input.getRev())
input.attrs.insert_or_assign("rev",
- Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
+ Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
} else {
if (auto res = getCache()->lookup(store, mutableAttrs)) {
- auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+ auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
@@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme
}
if (!input.getRev())
- input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
+ input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -421,7 +421,7 @@ struct GitInputScheme : InputScheme
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
- auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
+ auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 8bb7c2c1d..9f84ffb68 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -29,7 +29,7 @@ struct GitArchiveInputScheme : InputScheme
if (path.size() == 2) {
} else if (path.size() == 3) {
if (std::regex_match(path[2], revRegex))
- rev = Hash(path[2], htSHA1);
+ rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
@@ -41,7 +41,7 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") {
if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
- rev = Hash(value, htSHA1);
+ rev = Hash::parseAny(value, htSHA1);
}
else if (name == "ref") {
if (!std::regex_match(value, refRegex))
@@ -191,7 +191,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
- auto rev = Hash(std::string { json["sha"] }, htSHA1);
+ auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
@@ -235,7 +235,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
readFile(
store->toRealPath(
downloadFile(store, url, "source", false).storePath)));
- auto rev = Hash(std::string(json[0]["id"]), htSHA1);
+ auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index 91dc83740..b981d4d8e 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -18,7 +18,7 @@ struct IndirectInputScheme : InputScheme
if (path.size() == 1) {
} else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex))
- rev = Hash(path[1], htSHA1);
+ rev = Hash::parseAny(path[1], htSHA1);
else if (std::regex_match(path[1], refRegex))
ref = path[1];
else
@@ -29,7 +29,7 @@ struct IndirectInputScheme : InputScheme
ref = path[1];
if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
- rev = Hash(path[2], htSHA1);
+ rev = Hash::parseAny(path[2], htSHA1);
} else
throw BadURL("GitHub URL '%s' is invalid", url.url);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index c48cb6fd1..3e76ffc4d 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -209,7 +209,7 @@ struct MercurialInputScheme : InputScheme
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
- auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
+ auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
if (!input.getRev() || input.getRev() == rev2) {
input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
@@ -252,7 +252,7 @@ struct MercurialInputScheme : InputScheme
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
- input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
+ input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
input.attrs.insert_or_assign("ref", tokens[2]);
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 55158cece..a2d16365e 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -67,8 +67,10 @@ DownloadFileResult downloadFile(
StringSink sink;
dumpString(*res.data, sink);
auto hash = hashString(htSHA256, *res.data);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
- info.narHash = hashString(htSHA256, *sink.s);
+ ValidPathInfo info {
+ store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name),
+ hashString(htSHA256, *sink.s),
+ };
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 3f7d99a1d..be3c06a38 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -362,7 +362,7 @@ public:
auto width = getWindowSize().second;
if (width <= 0) width = std::numeric_limits::max();
- writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K");
+ writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K");
}
std::string getStatus(State & state)
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 30150eeba..5433fe50d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -143,7 +143,7 @@ struct FileSource : FdSource
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair, CheckSigsFlag checkSigs)
{
- assert(info.narHash && info.narSize);
+ assert(info.narSize);
if (!repair && isValidPath(info.path)) {
// FIXME: copyNAR -> null sink
@@ -153,6 +153,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
auto [fdTemp, fnTemp] = createTempFile();
+ AutoDelete autoDelete(fnTemp);
+
auto now1 = std::chrono::steady_clock::now();
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
@@ -167,6 +169,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
TeeSource teeSource(narSource, *compressionSink);
narAccessor = makeNarAccessor(teeSource);
compressionSink->finish();
+ fileSink.flush();
}
auto now2 = std::chrono::steady_clock::now();
@@ -216,7 +219,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
}
}
- upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
+ upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json");
}
/* Optionally maintain an index of DWARF debug info files
@@ -280,7 +283,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
upsertFile(narInfo->url,
- std::make_shared(fnTemp, std::ios_base::in),
+ std::make_shared(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar");
} else
stats.narWriteAverted++;
@@ -309,14 +312,10 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
{
auto info = queryPathInfo(storePath).cast();
- uint64_t narSize = 0;
+ LengthSink narSize;
+ TeeSink tee { sink, narSize };
- LambdaSink wrapperSink([&](const unsigned char * data, size_t len) {
- sink(data, len);
- narSize += len;
- });
-
- auto decompressor = makeDecompressionSink(info->compression, wrapperSink);
+ auto decompressor = makeDecompressionSink(info->compression, tee);
try {
getFile(info->url, *decompressor);
@@ -328,7 +327,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
stats.narRead++;
//stats.narReadCompressedBytes += nar->size(); // FIXME
- stats.narReadBytes += narSize;
+ stats.narReadBytes += narSize.length;
}
void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
@@ -382,7 +381,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}
- ValidPathInfo info(makeFixedOutputPath(method, *h, name));
+ ValidPathInfo info {
+ makeFixedOutputPath(method, *h, name),
+ Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+ };
auto source = StringSource { *sink.s };
addToStore(info, source, repair, CheckSigs);
@@ -393,7 +395,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
- ValidPathInfo info(computeStorePathForText(name, s, references));
+ ValidPathInfo info {
+ computeStorePathForText(name, s, references),
+ Hash::dummy, // Will be fixed in addToStore, which recomputes nar hash
+ };
info.references = references;
if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 4156cd678..afb2bb096 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -806,8 +806,8 @@ private:
/* RAII object to delete the chroot directory. */
std::shared_ptr autoDelChroot;
- /* Whether this is a fixed-output derivation. */
- bool fixedOutput;
+ /* The sort of derivation we are building. */
+ DerivationType derivationType;
/* Whether to run the build in a private network namespace. */
bool privateNetwork = false;
@@ -1181,8 +1181,8 @@ void DerivationGoal::haveDerivation()
retrySubstitution = false;
- for (auto & i : drv->outputs)
- worker.store.addTempRoot(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ worker.store.addTempRoot(i.second.second);
/* Check what outputs paths are not already valid. */
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@@ -1195,9 +1195,9 @@ void DerivationGoal::haveDerivation()
parsedDrv = std::make_unique(drvPath, *drv);
- if (parsedDrv->contentAddressed()) {
+ if (drv->type() == DerivationType::CAFloating) {
settings.requireExperimentalFeature("ca-derivations");
- throw Error("ca-derivations isn't implemented yet");
+ throw UnimplementedError("ca-derivations isn't implemented yet");
}
@@ -1288,14 +1288,14 @@ void DerivationGoal::repairClosure()
/* Get the output closure. */
StorePathSet outputClosure;
- for (auto & i : drv->outputs) {
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
if (!wantOutput(i.first, wantedOutputs)) continue;
- worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
+ worker.store.computeFSClosure(i.second.second, outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
- for (auto & i : drv->outputs)
- outputClosure.erase(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ outputClosure.erase(i.second.second);
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
@@ -1306,8 +1306,8 @@ void DerivationGoal::repairClosure()
for (auto & i : inputClosure)
if (i.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(i);
- for (auto & j : drv.outputs)
- outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
+ for (auto & j : drv.outputsAndPaths(worker.store))
+ outputsToDrv.insert_or_assign(j.second.second, i);
}
/* Check each path (slow!). */
@@ -1392,12 +1392,12 @@ void DerivationGoal::inputsRealised()
debug("added input paths %s", worker.store.showPaths(inputPaths));
- /* Is this a fixed-output derivation? */
- fixedOutput = drv->isFixedOutput();
+ /* What type of derivation are we building? */
+ derivationType = drv->type();
/* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/
- nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1;
+ nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a
@@ -1466,16 +1466,16 @@ void DerivationGoal::tryToBuild()
/* If any of the outputs already exist but are not valid, delete
them. */
- for (auto & i : drv->outputs) {
- if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
- debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
- deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ if (worker.store.isValidPath(i.second.second)) continue;
+ debug("removing invalid path '%s'", worker.store.printStorePath(i.second.second));
+ deletePath(worker.store.Store::toRealPath(i.second.second));
}
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
- bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
+ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
/* Is the build hook willing to accept this job? */
if (!buildLocally) {
@@ -1783,7 +1783,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
- fixedOutput || diskFull ? BuildResult::TransientFailure :
+ derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@@ -1919,8 +1919,8 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
for (auto & j : paths2) {
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
- for (auto & k : drv.outputs)
- worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
+ for (auto & k : drv.outputsAndPaths(worker.store))
+ worker.store.computeFSClosure(k.second.second, paths);
}
}
@@ -1964,13 +1964,13 @@ void linkOrCopy(const Path & from, const Path & to)
void DerivationGoal::startBuilder()
{
/* Right platform? */
- if (!parsedDrv->canBuildLocally())
+ if (!parsedDrv->canBuildLocally(worker.store))
throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
drv->platform,
concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
worker.store.printStorePath(drvPath),
settings.thisSystem,
- concatStringsSep(", ", settings.systemFeatures));
+ concatStringsSep(", ", worker.store.systemFeatures));
if (drv->isBuiltin())
preloadNSS();
@@ -1996,7 +1996,7 @@ void DerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
- useChroot = !fixedOutput && !noChroot;
+ useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
}
if (worker.store.storeDir != worker.store.realStoreDir) {
@@ -2014,8 +2014,8 @@ void DerivationGoal::startBuilder()
chownToBuilder(tmpDir);
/* Substitute output placeholders with the actual output paths. */
- for (auto & output : drv->outputs)
- inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
+ for (auto & output : drv->outputsAndPaths(worker.store))
+ inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.second);
/* Construct the environment passed to the builder. */
initEnv();
@@ -2165,7 +2165,7 @@ void DerivationGoal::startBuilder()
"nogroup:x:65534:\n") % sandboxGid).str());
/* Create /etc/hosts with localhost entry. */
- if (!fixedOutput)
+ if (!(derivationIsImpure(derivationType)))
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@@ -2199,8 +2199,8 @@ void DerivationGoal::startBuilder()
rebuilding a path that is in settings.dirsInChroot
(typically the dependencies of /bin/sh). Throw them
out. */
- for (auto & i : drv->outputs)
- dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ dirsInChroot.erase(worker.store.printStorePath(i.second.second));
#elif __APPLE__
/* We don't really have any parent prep work to do (yet?)
@@ -2373,7 +2373,7 @@ void DerivationGoal::startBuilder()
us.
*/
- if (!fixedOutput)
+ if (!(derivationIsImpure(derivationType)))
privateNetwork = true;
userNamespaceSync.create();
@@ -2574,7 +2574,7 @@ void DerivationGoal::initEnv()
derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment
variable won't be set, so `fetchurl' will do the check. */
- if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
+ if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
/* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the
@@ -2585,7 +2585,7 @@ void DerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
- if (fixedOutput) {
+ if (derivationIsImpure(derivationType)) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@@ -2612,8 +2612,8 @@ void DerivationGoal::writeStructuredAttrs()
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
- for (auto & i : drv->outputs)
- outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.second), inputRewrites);
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
@@ -2815,9 +2815,9 @@ struct RestrictedStore : public LocalFSStore
if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
auto drv = derivationFromPath(path.path);
- for (auto & output : drv.outputs)
+ for (auto & output : drv.outputsAndPaths(*this))
if (wantOutput(output.first, path.outputs))
- newPaths.insert(output.second.path(*this, drv.name));
+ newPaths.insert(output.second.second);
} else if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
}
@@ -2920,7 +2920,8 @@ void DerivationGoal::startDaemon()
FdSink to(remote.get());
try {
daemon::processConnection(store, from, to,
- daemon::NotTrusted, daemon::Recursive, "nobody", 65535);
+ daemon::NotTrusted, daemon::Recursive,
+ [&](Store & store) { store.createUser("nobody", 65535); });
debug("terminated daemon connection");
} catch (SysError &) {
ignoreException();
@@ -3179,7 +3180,7 @@ void DerivationGoal::runChild()
createDirs(chrootRootDir + "/dev/shm");
createDirs(chrootRootDir + "/dev/pts");
ss.push_back("/dev/full");
- if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
+ if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
ss.push_back("/dev/kvm");
ss.push_back("/dev/null");
ss.push_back("/dev/random");
@@ -3195,7 +3196,7 @@ void DerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
- if (fixedOutput) {
+ if (derivationIsImpure(derivationType)) {
ss.push_back("/etc/resolv.conf");
// Only use nss functions to resolve hosts and
@@ -3436,7 +3437,7 @@ void DerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
- if (fixedOutput)
+ if (derivationIsImpure(derivationType))
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Our rwx outputs */
@@ -3616,8 +3617,8 @@ void DerivationGoal::registerOutputs()
to do anything here. */
if (hook) {
bool allValid = true;
- for (auto & i : drv->outputs)
- if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
+ for (auto & i : drv->outputsAndPaths(worker.store))
+ if (!worker.store.isValidPath(i.second.second)) allValid = false;
if (allValid) return;
}
@@ -3638,23 +3639,23 @@ void DerivationGoal::registerOutputs()
Nix calls. */
StorePathSet referenceablePaths;
for (auto & p : inputPaths) referenceablePaths.insert(p);
- for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store)) referenceablePaths.insert(i.second.second);
for (auto & p : addedPaths) referenceablePaths.insert(p);
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
- for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
- if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ auto path = worker.store.printStorePath(i.second.second);
+ if (!missingPaths.count(i.second.second)) continue;
Path actualPath = path;
if (needsHashRewrite()) {
- auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
+ auto r = redirectedOutputs.find(i.second.second);
if (r != redirectedOutputs.end()) {
auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair
- && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
+ && redirectedBadOutputs.count(i.second.second)
&& pathExists(redirected))
replaceValidPath(path, redirected);
if (buildMode == bmCheck)
@@ -3721,9 +3722,22 @@ void DerivationGoal::registerOutputs()
hash). */
std::optional ca;
- if (fixedOutput) {
-
- FixedOutputHash outputHash = std::get(i.second.output).hash;
+ if (! std::holds_alternative(i.second.first.output)) {
+ DerivationOutputCAFloating outputHash;
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed doi) {
+ assert(false); // Enclosing `if` handles this case in other branch
+ },
+ [&](DerivationOutputCAFixed dof) {
+ outputHash = DerivationOutputCAFloating {
+ .method = dof.hash.method,
+ .hashType = dof.hash.hash.type,
+ };
+ },
+ [&](DerivationOutputCAFloating dof) {
+ outputHash = dof;
+ },
+ }, i.second.first.output);
if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
@@ -3737,12 +3751,17 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
Hash h2 = outputHash.method == FileIngestionMethod::Recursive
- ? hashPath(outputHash.hash.type, actualPath).first
- : hashFile(outputHash.hash.type, actualPath);
+ ? hashPath(outputHash.hashType, actualPath).first
+ : hashFile(outputHash.hashType, actualPath);
- auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
+ auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.second.name());
- if (outputHash.hash != h2) {
+ // true if either floating CA, or incorrect fixed hash.
+ bool needsMove = true;
+
+ if (auto p = std::get_if(& i.second.first.output)) {
+ Hash & h = p->hash.hash;
+ if (h != h2) {
/* Throw an error after registering the path as
valid. */
@@ -3750,9 +3769,15 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest),
- outputHash.hash.to_string(SRI, true),
+ h.to_string(SRI, true),
h2.to_string(SRI, true)));
+ } else {
+ // matched the fixed hash, so no move needed.
+ needsMove = false;
+ }
+ }
+ if (needsMove) {
Path actualDest = worker.store.Store::toRealPath(dest);
if (worker.store.isValidPath(dest))
@@ -3840,8 +3865,10 @@ void DerivationGoal::registerOutputs()
worker.markContentsGood(worker.store.parseStorePath(path));
}
- ValidPathInfo info(worker.store.parseStorePath(path));
- info.narHash = hash.first;
+ ValidPathInfo info {
+ worker.store.parseStorePath(path),
+ hash.first,
+ };
info.narSize = hash.second;
info.references = std::move(references);
info.deriver = drvPath;
@@ -3897,8 +3924,8 @@ void DerivationGoal::registerOutputs()
/* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
- for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ auto path = worker.store.printStorePath(i.second.second);
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
@@ -3915,8 +3942,8 @@ void DerivationGoal::registerOutputs()
/* Remove the .check directories if we're done. FIXME: keep them
if the result was not determistic? */
if (curRound == nrRounds) {
- for (auto & i : drv->outputs) {
- Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
+ Path prev = worker.store.printStorePath(i.second.second) + checkSuffix;
deletePath(prev);
}
}
@@ -4214,12 +4241,12 @@ void DerivationGoal::flushLine()
StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
{
StorePathSet result;
- for (auto & i : drv->outputs) {
+ for (auto & i : drv->outputsAndPaths(worker.store)) {
if (!wantOutput(i.first, wantedOutputs)) continue;
bool good =
- worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
- (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
- if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
+ worker.store.isValidPath(i.second.second) &&
+ (!checkHash || worker.pathContentsGood(i.second.second));
+ if (good == returnValid) result.insert(i.second.second);
}
return result;
}
@@ -4852,8 +4879,17 @@ void Worker::run(const Goals & _topGoals)
waitForInput();
else {
if (awake.empty() && 0 == settings.maxBuildJobs)
- throw Error("unable to start any build; either increase '--max-jobs' "
- "or enable remote builds");
+ {
+ if (getMachines().empty())
+ throw Error("unable to start any build; either increase '--max-jobs' "
+ "or enable remote builds."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+ else
+ throw Error("unable to start any build; remote machines may not have "
+ "all required system features."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
+
+ }
assert(!awake.empty());
}
}
@@ -5037,7 +5073,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
- HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
+ HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 6585a480d..4fb5d8a06 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,6 +58,20 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
+ /* Try the hashed mirrors first. */
+ if (getAttr("outputHashMode") == "flat")
+ for (auto hashedMirror : settings.hashedMirrors.get())
+ try {
+ if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
+ std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
+ Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
+ fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+ return;
+ } catch (Error & e) {
+ debug(e.what());
+ }
+
+ /* Otherwise try the specified URL. */
fetch(mainUrl);
}
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index f83b98a98..0885c3d0e 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -1,4 +1,6 @@
+#include "args.hh"
#include "content-address.hh"
+#include "split.hh"
namespace nix {
@@ -24,10 +26,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+ hash.to_string(Base32, true);
}
-// FIXME Put this somewhere?
-template struct overloaded : Ts... { using Ts::operator()...; };
-template overloaded(Ts...) -> overloaded;
-
std::string renderContentAddress(ContentAddress ca) {
return std::visit(overloaded {
[](TextHash th) {
@@ -40,38 +38,46 @@ std::string renderContentAddress(ContentAddress ca) {
}
ContentAddress parseContentAddress(std::string_view rawCa) {
- auto prefixSeparator = rawCa.find(':');
- if (prefixSeparator != string::npos) {
- auto prefix = string(rawCa, 0, prefixSeparator);
- if (prefix == "text") {
- auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
- Hash hash = Hash(string(hashTypeAndHash));
- if (hash.type != htSHA256) {
- throw Error("parseContentAddress: the text hash should have type SHA256");
- }
- return TextHash { hash };
- } else if (prefix == "fixed") {
- // This has to be an inverse of makeFixedOutputCA
- auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
- if (methodAndHash.substr(0,2) == "r:") {
- std::string_view hashRaw = methodAndHash.substr(2,string::npos);
- return FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = Hash(string(hashRaw)),
- };
- } else {
- std::string_view hashRaw = methodAndHash;
- return FixedOutputHash {
- .method = FileIngestionMethod::Flat,
- .hash = Hash(string(hashRaw)),
- };
- }
- } else {
- throw Error("parseContentAddress: format not recognized; has to be text or fixed");
- }
- } else {
- throw Error("Not a content address because it lacks an appropriate prefix");
+ auto rest = rawCa;
+
+ std::string_view prefix;
+ {
+ auto optPrefix = splitPrefixTo(rest, ':');
+ if (!optPrefix)
+ throw UsageError("not a content address because it is not in the form ':': %s", rawCa);
+ prefix = *optPrefix;
}
+
+ auto parseHashType_ = [&](){
+ auto hashTypeRaw = splitPrefixTo(rest, ':');
+ if (!hashTypeRaw)
+ throw UsageError("content address hash must be in form ':', but found: %s", rawCa);
+ HashType hashType = parseHashType(*hashTypeRaw);
+ return std::move(hashType);
+ };
+
+ // Switch on prefix
+ if (prefix == "text") {
+ // No parsing of the method, "text" only support flat.
+ HashType hashType = parseHashType_();
+ if (hashType != htSHA256)
+ throw Error("text content address hash should use %s, but instead uses %s",
+ printHashType(htSHA256), printHashType(hashType));
+ return TextHash {
+ .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+ };
+ } else if (prefix == "fixed") {
+ // Parse method
+ auto method = FileIngestionMethod::Flat;
+ if (splitPrefix(rest, "r:"))
+ method = FileIngestionMethod::Recursive;
+ HashType hashType = parseHashType_();
+ return FixedOutputHash {
+ .method = method,
+ .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
+ };
+ } else
+ throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
};
std::optional parseContentAddressOpt(std::string_view rawCaOpt) {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index b9a750425..ad3fe1847 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -289,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
- to << hash->to_string(Base16, false);
+ to << hash.to_string(Base16, false);
break;
}
@@ -454,8 +454,46 @@ static void performOp(TunnelLogger * logger, ref store,
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
- if (!trusted)
- throw Error("you are not privileged to build derivations");
+
+ /* Content-addressed derivations are trustless because their output paths
+ are verified by their content alone, so any derivation is free to
+ try to produce such a path.
+
+ Input-addressed derivation output paths, however, are calculated
+ from the derivation closure that produced them---even knowing the
+ root derivation is not enough. That the output data actually came
+ from those derivations is fundamentally unverifiable, but the daemon
+ trusts itself on that matter. The question instead is whether the
+ submitted plan has rights to the output paths it wants to fill, and
+ at least the derivation closure proves that.
+
+ It would have been nice if input-address algorithm merely depended
+ on the build time closure, rather than depending on the derivation
+ closure. That would mean input-addressed paths used at build time
+ would just be trusted and not need their own evidence. This is in
+ fact fine as the same guarantees would hold *inductively*: either
+ the remote builder has those paths and already trusts them, or it
+ needs to build them too and thus their evidence must be provided in
+ turn. The advantage of this variant algorithm is that the evidence
+ for input-addressed paths which the remote builder already has
+ doesn't need to be sent again.
+
+ That said, now that we have floating CA derivations, it is better
+ that people just migrate to those which also solve this problem, and
+ others. It's the same migration difficulty with strictly more
+ benefit.
+
+ Lastly, do note that when we parse fixed-output content-addressed
+ derivations, we throw out the precomputed output paths and just
+ store the hashes, so there aren't two competing sources of truth an
+ attacker could exploit. */
+ if (drv.type() == DerivationType::InputAddressed && !trusted)
+ throw Error("you are not privileged to build input-addressed derivations");
+
+ /* Make sure that the non-input-addressed derivations that got this far
+ are in fact content-addressed if we don't trust them. */
+ assert(derivationIsCA(drv.type()) || trusted);
+
auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork();
to << res.status << res.errorMsg;
@@ -638,7 +676,7 @@ static void performOp(TunnelLogger * logger, ref store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
- << info->narHash->to_string(Base16, false);
+ << info->narHash.to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
@@ -688,17 +726,18 @@ static void performOp(TunnelLogger * logger, ref store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
logger->stopWork();
- dumpPath(store->printStorePath(path), to);
+ dumpPath(store->toRealPath(path), to);
break;
}
case wopAddToStoreNar: {
bool repair, dontCheckSigs;
- ValidPathInfo info(store->parseStorePath(readString(from)));
+ auto path = store->parseStorePath(readString(from));
auto deriver = readString(from);
+ auto narHash = Hash::parseAny(readString(from), htSHA256);
+ ValidPathInfo info { path, narHash };
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
- info.narHash = Hash(readString(from), htSHA256);
info.references = readStorePaths(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings(from);
@@ -817,8 +856,7 @@ void processConnection(
FdSink & to,
TrustedFlag trusted,
RecursiveFlag recursive,
- const std::string & userName,
- uid_t userId)
+ std::function authHook)
{
auto monitor = !recursive ? std::make_unique(from.fd) : nullptr;
@@ -859,15 +897,7 @@ void processConnection(
/* If we can't accept clientVersion, then throw an error
*here* (not above). */
-
-#if 0
- /* Prevent users from doing something very dangerous. */
- if (geteuid() == 0 &&
- querySetting("build-users-group", "") == "")
- throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!");
-#endif
-
- store->createUser(userName, userId);
+ authHook(*store);
tunnelLogger->stopWork();
to.flush();
diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh
index 266932013..841ace316 100644
--- a/src/libstore/daemon.hh
+++ b/src/libstore/daemon.hh
@@ -12,7 +12,10 @@ void processConnection(
FdSink & to,
TrustedFlag trusted,
RecursiveFlag recursive,
- const std::string & userName,
- uid_t userId);
+ /* Arbitrary hook to check authorization / initialize user data / whatever
+ after the protocol has been negotiated. The idea is that this function
+ and everything it calls doesn't know about this stuff, and the
+ `nix-daemon` handles that instead. */
+ std::function authHook);
}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 7d0a5abeb..a9fed2564 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -7,23 +7,54 @@
namespace nix {
-// FIXME Put this somewhere?
-template struct overloaded : Ts... { using Ts::operator()...; };
-template overloaded(Ts...) -> overloaded;
-
-StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
+std::optional DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
{
return std::visit(overloaded {
- [](DerivationOutputInputAddressed doi) {
- return doi.path;
+ [](DerivationOutputInputAddressed doi) -> std::optional {
+ return { doi.path };
+ },
+ [&](DerivationOutputCAFixed dof) -> std::optional {
+ return {
+ store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
+ };
+ },
+ [](DerivationOutputCAFloating dof) -> std::optional {
+ return std::nullopt;
},
- [&](DerivationOutputFixed dof) {
- return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
- }
}, output);
}
+bool derivationIsCA(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return false;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return true;
+ };
+ // Since enums can have non-variant values, but making a `default:` would
+ // disable exhaustiveness warnings.
+ assert(false);
+}
+
+bool derivationIsFixed(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return false;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return false;
+ };
+ assert(false);
+}
+
+bool derivationIsImpure(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return false;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return false;
+ };
+ assert(false);
+}
+
+
bool BasicDerivation::isBuiltin() const
{
return string(builder, 0, 8) == "builtin:";
@@ -31,7 +62,7 @@ bool BasicDerivation::isBuiltin() const
StorePath writeDerivation(ref store,
- const Derivation & drv, std::string_view name, RepairFlag repair)
+ const Derivation & drv, RepairFlag repair)
{
auto references = drv.inputSrcs;
for (auto & i : drv.inputDrvs)
@@ -39,7 +70,7 @@ StorePath writeDerivation(ref store,
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
- auto suffix = std::string(name) + drvExtension;
+ auto suffix = std::string(drv.name) + drvExtension;
auto contents = drv.unparse(*store, false);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
@@ -108,29 +139,33 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
-static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+static DerivationOutput parseDerivationOutput(const Store & store,
+ StorePath path, std::string_view hashAlgo, std::string_view hash)
{
- expect(str, ","); auto path = store.parseStorePath(parsePath(str));
- expect(str, ","); auto hashAlgo = parseString(str);
- expect(str, ","); const auto hash = parseString(str);
- expect(str, ")");
-
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
method = FileIngestionMethod::Recursive;
- hashAlgo = string(hashAlgo, 2);
+ hashAlgo = hashAlgo.substr(2);
}
const HashType hashType = parseHashType(hashAlgo);
- return DerivationOutput {
- .output = DerivationOutputFixed {
- .hash = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
- },
- }
- };
+ return hash != ""
+ ? DerivationOutput {
+ .output = DerivationOutputCAFixed {
+ .hash = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
+ },
+ }
+ }
+ : (settings.requireExperimentalFeature("ca-derivations"),
+ DerivationOutput {
+ .output = DerivationOutputCAFloating {
+ .method = std::move(method),
+ .hashType = std::move(hashType),
+ },
+ });
} else
return DerivationOutput {
.output = DerivationOutputInputAddressed {
@@ -139,6 +174,16 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
};
}
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
+{
+ expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+ expect(str, ","); const auto hashAlgo = parseString(str);
+ expect(str, ","); const auto hash = parseString(str);
+ expect(str, ")");
+
+ return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
+}
+
static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
@@ -278,13 +323,20 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
- if (auto hash = std::get_if(&i.second.output)) {
- s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
- s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
- } else {
- s += ','; printUnquotedString(s, "");
- s += ','; printUnquotedString(s, "");
- }
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed doi) {
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, "");
+ },
+ [&](DerivationOutputCAFixed dof) {
+ s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
+ s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
+ },
+ [&](DerivationOutputCAFloating dof) {
+ s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+ s += ','; printUnquotedString(s, "");
+ },
+ }, i.second.output);
s += ')';
}
@@ -336,60 +388,134 @@ bool isDerivation(const string & fileName)
}
-bool BasicDerivation::isFixedOutput() const
+DerivationType BasicDerivation::type() const
{
- return outputs.size() == 1 &&
- outputs.begin()->first == "out" &&
- std::holds_alternative(outputs.begin()->second.output);
+ std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
+ std::optional floatingHashType;
+ for (auto & i : outputs) {
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed _) {
+ inputAddressedOutputs.insert(i.first);
+ },
+ [&](DerivationOutputCAFixed _) {
+ fixedCAOutputs.insert(i.first);
+ },
+ [&](DerivationOutputCAFloating dof) {
+ floatingCAOutputs.insert(i.first);
+ if (!floatingHashType) {
+ floatingHashType = dof.hashType;
+ } else {
+ if (*floatingHashType != dof.hashType)
+ throw Error("All floating outputs must use the same hash type");
+ }
+ },
+ }, i.second.output);
+ }
+
+ if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ throw Error("Must have at least one output");
+ } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ return DerivationType::InputAddressed;
+ } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ if (fixedCAOutputs.size() > 1)
+ // FIXME: Experimental feature?
+ throw Error("Only one fixed output is allowed for now");
+ if (*fixedCAOutputs.begin() != "out")
+ throw Error("Single fixed output must be named \"out\"");
+ return DerivationType::CAFixed;
+ } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
+ return DerivationType::CAFloating;
+ } else {
+ throw Error("Can't mix derivation output types");
+ }
}
DrvHashes drvHashes;
+/* pathDerivationModulo and hashDerivationModulo are mutually recursive
+ */
-/* Returns the hash of a derivation modulo fixed-output
- subderivations. A fixed-output derivation is a derivation with one
- output (`out') for which an expected hash and hash algorithm are
- specified (using the `outputHash' and `outputHashAlgo'
- attributes). We don't want changes to such derivations to
- propagate upwards through the dependency graph, changing output
- paths everywhere.
+/* Look up the derivation by value and memoize the
+ `hashDerivationModulo` call.
+ */
+static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
+{
+ auto h = drvHashes.find(drvPath);
+ if (h == drvHashes.end()) {
+ assert(store.isValidPath(drvPath));
+ // Cache it
+ h = drvHashes.insert_or_assign(
+ drvPath,
+ hashDerivationModulo(
+ store,
+ store.readDerivation(drvPath),
+ false)).first;
+ }
+ return h->second;
+}
- For instance, if we change the url in a call to the `fetchurl'
- function, we do not want to rebuild everything depending on it
- (after all, (the hash of) the file being downloaded is unchanged).
- So the *output paths* should not change. On the other hand, the
- *derivation paths* should change to reflect the new dependency
- graph.
+/* See the header for interface details. These are the implementation details.
- That's what this function does: it returns a hash which is just the
- hash of the derivation ATerm, except that any input derivation
- paths have been replaced by the result of a recursive call to this
- function, and that for fixed-output derivations we return a hash of
- its output path. */
-Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
+ For fixed-output derivations, each hash in the map is not the
+ corresponding output's content hash, but a hash of that hash along
+ with other constant data. The key point is that the value is a pure
+ function of the output's contents, and there are no preimage attacks
+ either spoofing an output's contents for a derivation, or
+ spoofing a derivation for an output's contents.
+
+ For regular derivations, it looks up each subderivation from its hash
+ and recurs. If the subderivation is also regular, it simply
+ substitutes the derivation path with its hash. If the subderivation
+ is fixed-output, however, it takes each output hash and pretends it
+ is a derivation hash producing a single "out" output. This is so we
+ don't leak the provenance of fixed outputs, reducing pointless cache
+ misses as the build itself won't know this.
+ */
+DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
/* Return a fixed hash for fixed-output derivations. */
- if (drv.isFixedOutput()) {
- DerivationOutputs::const_iterator i = drv.outputs.begin();
- auto hash = std::get(i->second.output);
- return hashString(htSHA256, "fixed:out:"
- + hash.hash.printMethodAlgo() + ":"
- + hash.hash.hash.to_string(Base16, false) + ":"
- + store.printStorePath(i->second.path(store, drv.name)));
+ switch (drv.type()) {
+ case DerivationType::CAFloating:
+ throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
+ case DerivationType::CAFixed: {
+ std::map outputHashes;
+ for (const auto & i : drv.outputsAndPaths(store)) {
+ auto & dof = std::get(i.second.first.output);
+ auto hash = hashString(htSHA256, "fixed:out:"
+ + dof.hash.printMethodAlgo() + ":"
+ + dof.hash.hash.to_string(Base16, false) + ":"
+ + store.printStorePath(i.second.second));
+ outputHashes.insert_or_assign(i.first, std::move(hash));
+ }
+ return outputHashes;
+ }
+ case DerivationType::InputAddressed:
+ break;
}
/* For other derivations, replace the inputs paths with recursive
- calls to this function.*/
+ calls to this function. */
std::map inputs2;
for (auto & i : drv.inputDrvs) {
- auto h = drvHashes.find(i.first);
- if (h == drvHashes.end()) {
- assert(store.isValidPath(i.first));
- h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
- store.readDerivation(i.first), false)).first;
- }
- inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
+ const auto & res = pathDerivationModulo(store, i.first);
+ std::visit(overloaded {
+ // Regular non-CA derivation, replace derivation
+ [&](Hash drvHash) {
+ inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
+ },
+ // CA derivation's output hashes
+ [&](CaOutputHashes outputHashes) {
+ std::set justOut = { "out" };
+ for (auto & output : i.second) {
+ /* Put each one in with a single "out" output.. */
+ const auto h = outputHashes.at(output);
+ inputs2.insert_or_assign(
+ h.to_string(Base16, false),
+ justOut);
+ }
+ },
+ }, res);
}
return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@@ -413,38 +539,18 @@ bool wantOutput(const string & output, const std::set & wanted)
StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
- for (auto & i : outputs)
- paths.insert(i.second.path(store, name));
+ for (auto & i : outputsAndPaths(store))
+ paths.insert(i.second.second);
return paths;
}
static DerivationOutput readDerivationOutput(Source & in, const Store & store)
{
auto path = store.parseStorePath(readString(in));
- auto hashAlgo = readString(in);
- auto hash = readString(in);
+ const auto hashAlgo = readString(in);
+ const auto hash = readString(in);
- if (hashAlgo != "") {
- auto method = FileIngestionMethod::Flat;
- if (string(hashAlgo, 0, 2) == "r:") {
- method = FileIngestionMethod::Recursive;
- hashAlgo = string(hashAlgo, 2);
- }
- auto hashType = parseHashType(hashAlgo);
- return DerivationOutput {
- .output = DerivationOutputFixed {
- .hash = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
- },
- }
- };
- } else
- return DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = std::move(path),
- }
- };
+ return parseDerivationOutput(store, std::move(path), hashAlgo, hash);
}
StringSet BasicDerivation::outputNames() const
@@ -455,6 +561,27 @@ StringSet BasicDerivation::outputNames() const
return names;
}
+DerivationOutputsAndPaths BasicDerivation::outputsAndPaths(const Store & store) const {
+ DerivationOutputsAndPaths outsAndPaths;
+ for (auto output : outputs)
+ outsAndPaths.insert(std::make_pair(
+ output.first,
+ std::make_pair(output.second, output.second.path(store, name))
+ )
+ );
+ return outsAndPaths;
+}
+
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+ DerivationOutputsAndOptPaths outsAndOptPaths;
+ for (auto output : outputs)
+ outsAndOptPaths.insert(std::make_pair(
+ output.first,
+ std::make_pair(output.second, output.second.pathOpt(store, output.first))
+ )
+ );
+ return outsAndOptPaths;
+}
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
auto nameWithSuffix = drvPath.name();
@@ -495,15 +622,22 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv)
{
out << drv.outputs.size();
- for (auto & i : drv.outputs) {
+ for (auto & i : drv.outputsAndPaths(store)) {
out << i.first
- << store.printStorePath(i.second.path(store, drv.name));
- if (auto hash = std::get_if(&i.second.output)) {
- out << hash->hash.printMethodAlgo()
- << hash->hash.hash.to_string(Base16, false);
- } else {
- out << "" << "";
- }
+ << store.printStorePath(i.second.second);
+ std::visit(overloaded {
+ [&](DerivationOutputInputAddressed doi) {
+ out << "" << "";
+ },
+ [&](DerivationOutputCAFixed dof) {
+ out << dof.hash.printMethodAlgo()
+ << dof.hash.hash.to_string(Base16, false);
+ },
+ [&](DerivationOutputCAFloating dof) {
+ out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+ << "";
+ },
+ }, i.second.first.output);
}
writeStorePaths(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 133ffe50e..3aae30ab2 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -6,6 +6,7 @@
#include "content-address.hh"
#include