Merge remote-tracking branch 'upstream/master' into best-effort-supplementary-groups

This commit is contained in:
John Ericson 2023-05-15 16:50:11 -04:00
commit 746c6aae3f
75 changed files with 1544 additions and 519 deletions

3
.gitignore vendored
View file

@ -51,6 +51,8 @@ perl/Makefile.config
/src/nix/nix /src/nix/nix
/src/nix/doc
# /src/nix-env/ # /src/nix-env/
/src/nix-env/nix-env /src/nix-env/nix-env
@ -85,6 +87,7 @@ perl/Makefile.config
/tests/shell.drv /tests/shell.drv
/tests/config.nix /tests/config.nix
/tests/ca/config.nix /tests/ca/config.nix
/tests/dyn-drv/config.nix
/tests/repl-result-out /tests/repl-result-out
# /tests/lang/ # /tests/lang/

View file

@ -11,6 +11,7 @@ man-pages := $(foreach n, \
nix-prefetch-url.1 nix-channel.1 \ nix-prefetch-url.1 nix-channel.1 \
nix-hash.1 nix-copy-closure.1 \ nix-hash.1 nix-copy-closure.1 \
nix.conf.5 nix-daemon.8 \ nix.conf.5 nix-daemon.8 \
nix-profiles.5 \
, $(d)/$(n)) , $(d)/$(n))
# man pages for subcommands # man pages for subcommands
@ -85,6 +86,12 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md
$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
@rm $^.tmp @rm $^.tmp
$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md
@printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp
@cat $^ >> $^.tmp
$(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@
@rm $^.tmp
$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md
@cp $< $@ @cp $< $@
@$(call process-includes,$@,$@) @$(call process-includes,$@,$@)

View file

@ -92,6 +92,11 @@
{{#include ./command-ref/new-cli/SUMMARY.md}} {{#include ./command-ref/new-cli/SUMMARY.md}}
- [Files](command-ref/files.md) - [Files](command-ref/files.md)
- [nix.conf](command-ref/conf-file.md) - [nix.conf](command-ref/conf-file.md)
- [Profiles](command-ref/files/profiles.md)
- [manifest.nix](command-ref/files/manifest.nix.md)
- [manifest.json](command-ref/files/manifest.json.md)
- [Channels](command-ref/files/channels.md)
- [Default Nix expression](command-ref/files/default-nix-expression.md)
- [Architecture](architecture/architecture.md) - [Architecture](architecture/architecture.md)
- [Glossary](glossary.md) - [Glossary](glossary.md)
- [Contributing](contributing/contributing.md) - [Contributing](contributing/contributing.md)

View file

@ -0,0 +1,26 @@
## Channels
A directory containing symlinks to Nix channels, managed by [`nix-channel`]:
- `$XDG_STATE_HOME/nix/profiles/channels` for regular users
- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root`
[`nix-channel`] uses a [profile](@docroot@/command-ref/files/profiles.md) to store channels.
This profile contains symlinks to the contents of those channels.
## Subscribed channels
The list of subscribed channels is stored in
- `~/.nix-channels`
- `$XDG_STATE_HOME/nix/channels` if [`use-xdg-base-directories`] is set to `true`
in the following format:
```
<url> <name>
...
```
[`nix-channel`]: @docroot@/command-ref/nix-channel.md
[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories

View file

@ -0,0 +1,52 @@
## Default Nix expression
The source for the default [Nix expressions](@docroot@/language/index.md) used by [`nix-env`]:
- `~/.nix-defexpr`
- `$XDG_STATE_HOME/nix/defexpr` if [`use-xdg-base-directories`] is set to `true`.
It is loaded as follows:
- If the default expression is a file, it is loaded as a Nix expression.
- If the default expression is a directory containing a `default.nix` file, that `default.nix` file is loaded as a Nix expression.
- If the default expression is a directory without a `default.nix` file, then its contents (both files and subdirectories) are loaded as Nix expressions.
The expressions are combined into a single attribute set, each expression under an attribute with the same name as the original file or subdirectory.
Subdirectories without a `default.nix` file are traversed recursively in search of more Nix expressions, but the names of these intermediate directories are not added to the attribute paths of the default Nix expression.
Then, the resulting expression is interpreted like this:
- If the expression is an attribute set, it is used as the default Nix expression.
- If the expression is a function, an empty set is passed as argument and the return value is used as the default Nix expression.
For example, if the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to
```nix
{
foo = import ~/.nix-defexpr/foo.nix;
bar = import ~/.nix-defexpr/bar.nix;
}
```
The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored.
The command [`nix-channel`] places a symlink to the user's current [channels profile](@docroot@/command-ref/files/channels.md) in this directory.
This makes all subscribed channels available as attributes in the default expression.
## User channel link
A symlink that ensures that [`nix-env`] can find your channels:
- `~/.nix-defexpr/channels`
- `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`.
This symlink points to:
- `$XDG_STATE_HOME/profiles/channels` for regular users
- `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root`
In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.[`nix-env`]: ../nix-env.md
[`nix-env`]: @docroot@/command-ref/nix-env.md
[`nix-channel`]: @docroot@/command-ref/nix-channel.md
[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories

View file

@ -0,0 +1,45 @@
## `manifest.json`
The manifest file records the provenance of the packages that are installed in a [profile](./profiles.md) managed by [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental).
Here is an example of what the file might look like after installing `zoom-us` from Nixpkgs:
```json
{
"version": 1,
"elements": [
{
"active": true,
"attrPath": "legacyPackages.x86_64-linux.zoom-us",
"originalUrl": "flake:nixpkgs",
"storePaths": [
"/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
],
"uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a"
},
]
}
```
Each object in the array `elements` denotes an installed package and
has the following fields:
* `originalUrl`: The [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md) specified by
the user at the time of installation (e.g. `nixpkgs`). This is also
the flake reference that will be used by `nix profile upgrade`.
* `uri`: The locked flake reference to which `originalUrl` resolved.
* `attrPath`: The flake output attribute that provided this
package. Note that this is not necessarily the attribute that the
user specified, but the one resulting from applying the default
attribute paths and prefixes; for instance, `hello` might resolve to
`packages.x86_64-linux.hello` and the empty string to
`packages.x86_64-linux.default`.
* `storePath`: The paths in the Nix store containing the package.
* `active`: Whether the profile contains symlinks to the files of this
package. If set to false, the package is kept in the Nix store, but
is not "visible" in the profile's symlink tree.

View file

@ -0,0 +1,128 @@
## `manifest.nix`
The manifest file records the provenance of the packages that are installed in a [profile](./profiles.md) managed by [`nix-env`](@docroot@/command-ref/nix-env.md).
Here is an example of how this file might look like after installing `hello` from Nixpkgs:
```nix
[{
meta = {
available = true;
broken = false;
changelog =
"https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v2.12.1";
description = "A program that produces a familiar, friendly greeting";
homepage = "https://www.gnu.org/software/hello/manual/";
insecure = false;
license = {
deprecated = false;
free = true;
fullName = "GNU General Public License v3.0 or later";
redistributable = true;
shortName = "gpl3Plus";
spdxId = "GPL-3.0-or-later";
url = "https://spdx.org/licenses/GPL-3.0-or-later.html";
};
longDescription = ''
GNU Hello is a program that prints "Hello, world!" when you run it.
It is fully customizable.
'';
maintainers = [{
email = "edolstra+nixpkgs@gmail.com";
github = "edolstra";
githubId = 1148549;
name = "Eelco Dolstra";
}];
name = "hello-2.12.1";
outputsToInstall = [ "out" ];
platforms = [
"i686-cygwin"
"x86_64-cygwin"
"x86_64-darwin"
"i686-darwin"
"aarch64-darwin"
"armv7a-darwin"
"i686-freebsd13"
"x86_64-freebsd13"
"aarch64-genode"
"i686-genode"
"x86_64-genode"
"x86_64-solaris"
"js-ghcjs"
"aarch64-linux"
"armv5tel-linux"
"armv6l-linux"
"armv7a-linux"
"armv7l-linux"
"i686-linux"
"m68k-linux"
"microblaze-linux"
"microblazeel-linux"
"mipsel-linux"
"mips64el-linux"
"powerpc64-linux"
"powerpc64le-linux"
"riscv32-linux"
"riscv64-linux"
"s390-linux"
"s390x-linux"
"x86_64-linux"
"mmix-mmixware"
"aarch64-netbsd"
"armv6l-netbsd"
"armv7a-netbsd"
"armv7l-netbsd"
"i686-netbsd"
"m68k-netbsd"
"mipsel-netbsd"
"powerpc-netbsd"
"riscv32-netbsd"
"riscv64-netbsd"
"x86_64-netbsd"
"aarch64_be-none"
"aarch64-none"
"arm-none"
"armv6l-none"
"avr-none"
"i686-none"
"microblaze-none"
"microblazeel-none"
"msp430-none"
"or1k-none"
"m68k-none"
"powerpc-none"
"powerpcle-none"
"riscv32-none"
"riscv64-none"
"rx-none"
"s390-none"
"s390x-none"
"vc4-none"
"x86_64-none"
"i686-openbsd"
"x86_64-openbsd"
"x86_64-redox"
"wasm64-wasi"
"wasm32-wasi"
"x86_64-windows"
"i686-windows"
];
position =
"/nix/store/7niq32w715567hbph0q13m5lqna64c1s-nixos-unstable.tar.gz/nixos-unstable.tar.gz/pkgs/applications/misc/hello/default.nix:34";
unfree = false;
unsupported = false;
};
name = "hello-2.12.1";
out = {
outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1";
};
outPath = "/nix/store/260q5867crm1xjs4khgqpl6vr9kywql1-hello-2.12.1";
outputs = [ "out" ];
system = "x86_64-linux";
type = "derivation";
}]
```
Each element in this list corresponds to an installed package.
It incorporates some attributes of the original derivation, including `meta`, `name`, `out`, `outPath`, `outputs`, `system`.
This information is used by Nix for querying and updating the package.

View file

@ -0,0 +1,74 @@
## Profiles
A directory that contains links to profiles managed by [`nix-env`] and [`nix profile`]:
- `$XDG_STATE_HOME/nix/profiles` for regular users
- `$NIX_STATE_DIR/profiles/per-user/root` if the user is `root`
A profile is a directory of symlinks to files in the Nix store.
### Filesystem layout
Profiles are versioned as follows. When using a profile named *path*, *path* is a symlink to *path*`-`*N*`-link`, where *N* is the version of the profile.
In turn, *path*`-`*N*`-link` is a symlink to a path in the Nix store.
For example:
```console
$ ls -l ~alice/.local/state/nix/profiles/profile*
lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link
lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile
lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile
lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile
```
Each of these symlinks is a root for the Nix garbage collector.
The contents of the store path corresponding to each version of the
profile is a tree of symlinks to the files of the installed packages,
e.g.
```console
$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/
/home/eelco/.local/state/nix/profiles/profile-7-link/:
total 20
dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin
-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.nix
dr-xr-xr-x 4 root root 4096 Jan 1 1970 share
/home/eelco/.local/state/nix/profiles/profile-7-link/bin:
total 20
lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications:
total 12
lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
```
Each profile version contains a manifest file:
- [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) used by [`nix-env`](@docroot@/command-ref/nix-env.md).
- [`manifest.json`](@docroot@/command-ref/files/manifest.json.md) used by [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental).
## User profile link
A symbolic link to the user's current profile:
- `~/.nix-profile`
- `$XDG_STATE_HOME/nix/profile` if [`use-xdg-base-directories`] is set to `true`.
By default, this symlink points to:
- `$XDG_STATE_HOME/nix/profiles/profile` for regular users
- `$NIX_STATE_DIR/profiles/per-user/root/profile` for `root`
The `PATH` environment variable should include `/bin` subdirectory of the profile link (e.g. `~/.nix-profile/bin`) for the user environment to be visible to the user.
The [installer](@docroot@/installation/installing-binary.md) sets this up by default, unless you enable [`use-xdg-base-directories`].
[`nix-env`]: @docroot@/command-ref/nix-env.md
[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md
[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories

View file

@ -22,6 +22,9 @@ This command has the following operations:
channels. If *name* is omitted, it defaults to the last component of channels. If *name* is omitted, it defaults to the last component of
*url*, with the suffixes `-stable` or `-unstable` removed. *url*, with the suffixes `-stable` or `-unstable` removed.
A channel URL must point to a directory containing a file `nixexprs.tar.gz`.
At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channels entry point.
- `--remove` *name*\ - `--remove` *name*\
Removes the channel named *name* from the list of subscribed Removes the channel named *name* from the list of subscribed
channels. channels.
@ -71,30 +74,3 @@ switching from generation 483 to 482
$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' $ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
"14.04.526.dbadfad" "14.04.526.dbadfad"
``` ```
# Files
- `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/channels`\
`nix-channel` uses a `nix-env` profile to keep track of previous
versions of the subscribed channels. Every time you run `nix-channel
--update`, a new channel generation (that is, a symlink to the
channel Nix expressions in the Nix store) is created. This enables
`nix-channel --rollback` to revert to previous versions.
- `~/.nix-defexpr/channels`\
This is a symlink to
`${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/channels`. It ensures that
`nix-env` can find your channels. In a multi-user installation, you
may also have `~/.nix-defexpr/channels_root`, which links to the
channels of the root user.
# Channel format
A channel URL should point to a directory containing the following
files:
- `nixexprs.tar.xz`\
A tarball containing Nix expressions and files referenced by them
(such as build scripts and patches). At the top level, the tarball
should contain a single directory. That directory must contain a
file `default.nix` that serves as the channels “entry point”.

View file

@ -83,46 +83,6 @@ match. Here are some examples:
# Files # Files
- `~/.nix-defexpr`\ {{#include ./files/default-nix-expression.md}}
The source for the default Nix expressions used by the
`--install`, `--upgrade`, and `--query --available` operations to
obtain derivations. The `--file` option may be used to override
this default.
If `~/.nix-defexpr` is a file, it is loaded as a Nix expression. If {{#include ./files/profiles.md}}
the expression is a set, it is used as the default Nix expression.
If the expression is a function, an empty set is passed as argument
and the return value is used as the default Nix expression.
If `~/.nix-defexpr` is a directory containing a `default.nix` file,
that file is loaded as in the above paragraph.
If `~/.nix-defexpr` is a directory without a `default.nix` file,
then its contents (both files and subdirectories) are loaded as Nix
expressions. The expressions are combined into a single set, each
expression under an attribute with the same name as the original
file or subdirectory.
For example, if `~/.nix-defexpr` contains two files, `foo.nix` and
`bar.nix`, then the default Nix expression will essentially be
```nix
{
foo = import ~/.nix-defexpr/foo.nix;
bar = import ~/.nix-defexpr/bar.nix;
}
```
The file `manifest.nix` is always ignored. Subdirectories without a
`default.nix` file are traversed recursively in search of more Nix
expressions, but the names of these intermediate directories are not
added to the attribute paths of the default Nix expression.
The command `nix-channel` places symlinks to the downloaded Nix
expressions from each subscribed channel in this directory.
- `~/.nix-profile`\
A symbolic link to the user's current profile. By default, this
symlink points to `prefix/var/nix/profiles/default`. The `PATH`
environment variable should include `~/.nix-profile/bin` for the
user environment to be visible to the user.

View file

@ -24,23 +24,10 @@ If you are on Linux with systemd:
sudo systemctl daemon-reload sudo systemctl daemon-reload
``` ```
1. Remove systemd service files:
```console
sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket
```
1. The installer script uses systemd-tmpfiles to create the socket directory.
You may also want to remove the configuration for that:
```console
sudo rm /etc/tmpfiles.d/nix-daemon.conf
```
Remove files created by Nix: Remove files created by Nix:
```console ```console
sudo rm -rf /nix /etc/nix /etc/profile/nix.sh ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile
``` ```
Remove build users and their group: Remove build users and their group:
@ -54,8 +41,10 @@ sudo groupdel nixbld
There may also be references to Nix in There may also be references to Nix in
- `/etc/profile` - `/etc/bash.bashrc`
- `/etc/bashrc` - `/etc/bashrc`
- `/etc/profile`
- `/etc/zsh/zshrc`
- `/etc/zshrc` - `/etc/zshrc`
which you may remove. which you may remove.

View file

@ -1,20 +1,19 @@
# Built-in Constants # Built-in Constants
Here are the constants built into the Nix expression evaluator: These constants are built into the Nix language evaluator:
- `builtins`\ - [`builtins`]{#builtins-builtins} (attribute set)
The set `builtins` contains all the built-in functions and values.
You can use `builtins` to test for the availability of features in
the Nix installation, e.g.,
```nix
if builtins ? getEnv then builtins.getEnv "PATH" else ""
```
This allows a Nix expression to fall back gracefully on older Nix
installations that dont have the desired built-in function.
- [`builtins.currentSystem`]{#builtins-currentSystem}\ Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope.
The built-in value `currentSystem` evaluates to the Nix platform
identifier for the Nix installation on which the expression is being Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations:
evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.
```nix
if builtins ? getEnv then builtins.getEnv "PATH" else ""
```
- [`builtins.currentSystem`]{#builtins-currentSystem} (string)
The built-in value `currentSystem` evaluates to the Nix platform
identifier for the Nix installation on which the expression is being
evaluated, such as `"i686-linux"` or `"x86_64-darwin"`.

View file

@ -1,16 +1,16 @@
# Built-in Functions # Built-in Functions
This section lists the functions built into the Nix expression This section lists the functions built into the Nix language evaluator.
evaluator. (The built-in function `derivation` is discussed above.) All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant.
Some built-ins, such as `derivation`, are always in scope of every Nix
expression; you can just access them right away. But to prevent For convenience, some built-ins are can be accessed directly:
polluting the namespace too much, most built-ins are not in
scope. Instead, you can access them through the `builtins` built-in - [`derivation`](#builtins-derivation)
value, which is a set that contains all built-in functions and values. - [`import`](#builtins-import)
For instance, `derivation` is also available as `builtins.derivation`. - [`abort`](#builtins-abort)
- [`throw`](#builtins-throw)
<dl> <dl>
<dt><code>derivation <var>attrs</var></code>; <dt id="builtins-derivation"><a href="#builtins-derivation"><code>derivation <var>attrs</var></code></a></dt>
<code>builtins.derivation <var>attrs</var></code></dt>
<dd><p><var>derivation</var> is described in <dd><p><var>derivation</var> is described in
<a href="derivations.md">its own section</a>.</p></dd> <a href="derivations.md">its own section</a>.</p></dd>

View file

@ -36,7 +36,7 @@
## Attribute selection ## Attribute selection
Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*.
If the attribute doesnt exist, return *value* if provided, otherwise abort evaluation. If the attribute doesnt exist, return the *expr* after `or` if provided, otherwise abort evaluation.
<!-- FIXME: the following should to into its own language syntax section, but that needs more work to fit in well --> <!-- FIXME: the following should to into its own language syntax section, but that needs more work to fit in well -->

View file

@ -190,13 +190,17 @@ instance,
``` ```
evaluates to `"Foo"`. It is possible to provide a default value in an evaluates to `"Foo"`. It is possible to provide a default value in an
attribute selection using the `or` keyword. For example, attribute selection using the `or` keyword:
```nix ```nix
{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" { a = "Foo"; b = "Bar"; }.c or "Xyzzy"
``` ```
will evaluate to `"Xyzzy"` because there is no `c` attribute in the set. ```nix
{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy"
```
will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set.
You can use arbitrary double-quoted strings as attribute names: You can use arbitrary double-quoted strings as attribute names:

View file

@ -1,2 +1,6 @@
# Release X.Y (202?-??-??) # Release X.Y (202?-??-??)
- Speed-up of downloads from binary caches.
The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs).
The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs).
The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs).

5
mk/cxx-big-literal.mk Normal file
View file

@ -0,0 +1,5 @@
%.gen.hh: %
@echo 'R"foo(' >> $@.tmp
$(trace-gen) cat $< >> $@.tmp
@echo ')foo"' >> $@.tmp
@mv $@.tmp $@

View file

@ -101,6 +101,7 @@ include mk/libraries.mk
include mk/programs.mk include mk/programs.mk
include mk/patterns.mk include mk/patterns.mk
include mk/templates.mk include mk/templates.mk
include mk/cxx-big-literal.mk
include mk/tests.mk include mk/tests.mk

View file

@ -46,7 +46,15 @@ std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
{ {
auto v = toValue(*state).first; auto [v, pos] = toValue(*state);
if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
*v,
pos,
fmt("while evaluating the attribute '%s'", attrPath)))
{
return { *derivedPathWithInfo };
}
Bindings & autoArgs = *cmd.getAutoArgs(*state); Bindings & autoArgs = *cmd.getAutoArgs(*state);

View file

@ -95,31 +95,13 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
// FIXME: use eval cache? // FIXME: use eval cache?
auto v = attr->forceValue(); auto v = attr->forceValue();
if (v.type() == nPath) { if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
auto storePath = v.path().fetchToStore(state->store); v,
return {{ noPos,
.path = DerivedPath::Opaque { fmt("while evaluating the flake output attribute '%s'", attrPath)))
.path = std::move(storePath), {
}, return { *derivedPathWithInfo };
.info = make_ref<ExtraPathInfo>(),
}};
} }
else if (v.type() == nString) {
NixStringContext context;
auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
auto storePath = state->store->maybeParseStorePath(s);
if (storePath && context.count(NixStringContextElem::Opaque { .path = *storePath })) {
return {{
.path = DerivedPath::Opaque {
.path = std::move(*storePath),
},
.info = make_ref<ExtraPathInfo>(),
}};
} else
throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
}
else else
throw Error("flake output attribute '%s' is not a derivation or path", attrPath); throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
} }
@ -234,7 +216,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
} }
} }
return InstallableValue::nixpkgsFlakeRef(); return defaultNixpkgsFlakeRef();
} }
} }

View file

@ -67,9 +67,22 @@ struct InstallableFlake : InstallableValue
std::shared_ptr<flake::LockedFlake> getLockedFlake() const; std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
FlakeRef nixpkgsFlakeRef() const override; FlakeRef nixpkgsFlakeRef() const;
}; };
/**
* Default flake ref for referring to Nixpkgs. For flakes that don't
* have their own Nixpkgs input, or other installables.
*
* It is a layer violation for Nix to know about Nixpkgs; currently just
* `nix develop` does. Be wary of using this /
* `InstallableFlake::nixpkgsFlakeRef` more places.
*/
static inline FlakeRef defaultNixpkgsFlakeRef()
{
return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
}
ref<eval_cache::EvalCache> openEvalCache( ref<eval_cache::EvalCache> openEvalCache(
EvalState & state, EvalState & state,
std::shared_ptr<flake::LockedFlake> lockedFlake); std::shared_ptr<flake::LockedFlake> lockedFlake);

View file

@ -41,4 +41,26 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
return ref { castedInstallable }; return ref { castedInstallable };
} }
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{
if (v.type() == nPath) {
auto storePath = v.path().fetchToStore(state->store);
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),
},
.info = make_ref<ExtraPathInfo>(),
}};
}
else if (v.type() == nString) {
return {{
.path = state->coerceToDerivedPath(pos, v, errorCtx),
.info = make_ref<ExtraPathInfo>(),
}};
}
else return std::nullopt;
}
} }

View file

@ -96,13 +96,26 @@ struct InstallableValue : Installable
UnresolvedApp toApp(EvalState & state); UnresolvedApp toApp(EvalState & state);
virtual FlakeRef nixpkgsFlakeRef() const
{
return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
}
static InstallableValue & require(Installable & installable); static InstallableValue & require(Installable & installable);
static ref<InstallableValue> require(ref<Installable> installable); static ref<InstallableValue> require(ref<Installable> installable);
protected:
/**
* Handles either a plain path, or a string with a single string
* context elem in the right format. The latter case is handled by
* `EvalState::coerceToDerivedPath()`; see it for details.
*
* @param v Value that is hopefully a string or path per the above.
*
* @param pos Position of value to aid with diagnostics.
*
* @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`.
*
* @result A derived path (with empty info, for now) if the value
* matched the above criteria.
*/
std::optional<DerivedPathWithInfo> trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx);
}; };
} }

View file

@ -94,7 +94,6 @@ RootValue allocRootValue(Value * v)
#endif #endif
} }
void Value::print(const SymbolTable & symbols, std::ostream & str, void Value::print(const SymbolTable & symbols, std::ostream & str,
std::set<const void *> * seen) const std::set<const void *> * seen) const
{ {
@ -1048,6 +1047,27 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
} }
void EvalState::mkOutputString(
Value & value,
const StorePath & drvPath,
const std::string outputName,
std::optional<StorePath> optOutputPath)
{
value.mkString(
optOutputPath
? store->printStorePath(*std::move(optOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: downstreamPlaceholder(*store, drvPath, outputName),
NixStringContext {
NixStringContextElem::Built {
.drvPath = drvPath,
.output = outputName,
}
});
}
/* Create a thunk for the delayed computation of the given expression /* Create a thunk for the delayed computation of the given expression
in the given environment. But if the expression is a variable, in the given environment. But if the expression is a variable,
then look it up right away. This significantly reduces the number then look it up right away. This significantly reduces the number
@ -2298,6 +2318,80 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
} }
std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
{
NixStringContext context;
auto s = forceString(v, context, pos, errorCtx);
auto csize = context.size();
if (csize != 1)
error(
"string '%s' has %d entries in its context. It should only have exactly one entry",
s, csize)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
auto derivedPath = std::visit(overloaded {
[&](NixStringContextElem::Opaque && o) -> DerivedPath {
return DerivedPath::Opaque {
.path = std::move(o.path),
};
},
[&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
error(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s).withTrace(pos, errorCtx).debugThrow<EvalError>();
},
[&](NixStringContextElem::Built && b) -> DerivedPath {
return DerivedPath::Built {
.drvPath = std::move(b.drvPath),
.outputs = OutputsSpec::Names { std::move(b.output) },
};
},
}, ((NixStringContextElem &&) *context.begin()).raw());
return {
std::move(derivedPath),
std::move(s),
};
}
DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
{
auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_;
std::visit(overloaded {
[&](const DerivedPath::Opaque & o) {
auto sExpected = store->printStorePath(o.path);
if (s != sExpected)
error(
"path string '%s' has context with the different path '%s'",
s, sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
},
[&](const DerivedPath::Built & b) {
// TODO need derived path with single output to make this
// total. Will add as part of RFC 92 work and then this is
// cleaned up.
auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
auto drv = store->readDerivation(b.drvPath);
auto i = drv.outputs.find(output);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
auto optOutputPath = i->second.path(*store, drv.name, output);
// This is testing for the case of CA derivations
auto sExpected = optOutputPath
? store->printStorePath(*optOutputPath)
: downstreamPlaceholder(*store, b.drvPath, output);
if (s != sExpected)
error(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s, output, store->printStorePath(b.drvPath), sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
}
}, derivedPath.raw());
return derivedPath;
}
bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{ {
forceValue(v1, noPos); forceValue(v1, noPos);

View file

@ -21,6 +21,7 @@ namespace nix {
class Store; class Store;
class EvalState; class EvalState;
class StorePath; class StorePath;
struct DerivedPath;
enum RepairFlag : bool; enum RepairFlag : bool;
@ -473,6 +474,28 @@ public:
*/ */
StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
/**
* Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
*/
std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
/**
* Coerce to `DerivedPath`.
*
* Must be a string which is either a literal store path or a
* "placeholder (see `downstreamPlaceholder()`).
*
* Even more importantly, the string context must be exactly one
* element, which is either a `NixStringContextElem::Opaque` or
* `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep`
* is not permitted).
*
* The string is parsed based on the context --- the context is the
* source of truth, and ultimately tells us what we want, and then
* we ensure the string corresponds to it.
*/
DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
public: public:
/** /**
@ -576,12 +599,37 @@ public:
void mkThunk_(Value & v, Expr * expr); void mkThunk_(Value & v, Expr * expr);
void mkPos(Value & v, PosIdx pos); void mkPos(Value & v, PosIdx pos);
/* Create a string representing a store path. /**
* Create a string representing a store path.
The string is the printed store path with a context containing a single *
`Opaque` element of that store path. */ * The string is the printed store path with a context containing a single
* `NixStringContextElem::Opaque` element of that store path.
*/
void mkStorePathString(const StorePath & storePath, Value & v); void mkStorePathString(const StorePath & storePath, Value & v);
/**
* Create a string representing a `DerivedPath::Built`.
*
* The string is the printed store path with a context containing a single
* `NixStringContextElem::Built` element of the drv path and output name.
*
* @param value Value we are settings
*
* @param drvPath Path the drv whose output we are making a string for
*
* @param outputName Name of the output
*
* @param optOutputPath Optional output path for that string. Must
* be passed if and only if output store object is input-addressed.
* Will be printed to form string if passed, otherwise a placeholder
* will be used (see `downstreamPlaceholder()`).
*/
void mkOutputString(
Value & value,
const StorePath & drvPath,
const std::string outputName,
std::optional<StorePath> optOutputPath);
void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
/** /**

View file

@ -129,40 +129,31 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
} }
} }
/* Add and attribute to the given attribute map from the output name to /**
the output path, or a placeholder. * Add and attribute to the given attribute map from the output name to
* the output path, or a placeholder.
Where possible the path is used, but for floating CA derivations we *
may not know it. For sake of determinism we always assume we don't * Where possible the path is used, but for floating CA derivations we
and instead put in a place holder. In either case, however, the * may not know it. For sake of determinism we always assume we don't
string context will contain the drv path and output name, so * and instead put in a place holder. In either case, however, the
downstream derivations will have the proper dependency, and in * string context will contain the drv path and output name, so
addition, before building, the placeholder will be rewritten to be * downstream derivations will have the proper dependency, and in
the actual path. * addition, before building, the placeholder will be rewritten to be
* the actual path.
The 'drv' and 'drvPath' outputs must correspond. */ *
* The 'drv' and 'drvPath' outputs must correspond.
*/
static void mkOutputString( static void mkOutputString(
EvalState & state, EvalState & state,
BindingsBuilder & attrs, BindingsBuilder & attrs,
const StorePath & drvPath, const StorePath & drvPath,
const BasicDerivation & drv,
const std::pair<std::string, DerivationOutput> & o) const std::pair<std::string, DerivationOutput> & o)
{ {
auto optOutputPath = o.second.path(*state.store, drv.name, o.first); state.mkOutputString(
attrs.alloc(o.first).mkString( attrs.alloc(o.first),
optOutputPath drvPath,
? state.store->printStorePath(*optOutputPath) o.first,
/* Downstream we would substitute this for an actual path once o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
we build the floating CA derivation */
/* FIXME: we need to depend on the basic derivation, not
derivation */
: downstreamPlaceholder(*state.store, drvPath, o.first),
NixStringContext {
NixStringContextElem::Built {
.drvPath = drvPath,
.output = o.first,
}
});
} }
/* Load and evaluate an expression from path specified by the /* Load and evaluate an expression from path specified by the
@ -193,7 +184,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.mkList(outputsVal, drv.outputs.size()); state.mkList(outputsVal, drv.outputs.size());
for (const auto & [i, o] : enumerate(drv.outputs)) { for (const auto & [i, o] : enumerate(drv.outputs)) {
mkOutputString(state, attrs, *storePath, drv, o); mkOutputString(state, attrs, *storePath, o);
(outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first);
} }
@ -1100,7 +1091,7 @@ drvName, Bindings * attrs, Value & v)
bool isImpure = false; bool isImpure = false;
std::optional<std::string> outputHash; std::optional<std::string> outputHash;
std::string outputHashAlgo; std::string outputHashAlgo;
std::optional<FileIngestionMethod> ingestionMethod; std::optional<ContentAddressMethod> ingestionMethod;
StringSet outputs; StringSet outputs;
outputs.insert("out"); outputs.insert("out");
@ -1113,7 +1104,10 @@ drvName, Bindings * attrs, Value & v)
auto handleHashMode = [&](const std::string_view s) { auto handleHashMode = [&](const std::string_view s) {
if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else else if (s == "text") {
experimentalFeatureSettings.require(Xp::DynamicDerivations);
ingestionMethod = TextIngestionMethod {};
} else
state.debugThrowLastTrace(EvalError({ state.debugThrowLastTrace(EvalError({
.msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
.errPos = state.positions[noPos] .errPos = state.positions[noPos]
@ -1280,11 +1274,16 @@ drvName, Bindings * attrs, Value & v)
})); }));
/* Check whether the derivation name is valid. */ /* Check whether the derivation name is valid. */
if (isDerivation(drvName)) if (isDerivation(drvName) &&
!(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } &&
outputs.size() == 1 &&
*(outputs.begin()) == "out"))
{
state.debugThrowLastTrace(EvalError({ state.debugThrowLastTrace(EvalError({
.msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension),
.errPos = state.positions[noPos] .errPos = state.positions[noPos]
})); }));
}
if (outputHash) { if (outputHash) {
/* Handle fixed-output derivations. /* Handle fixed-output derivations.
@ -1300,21 +1299,15 @@ drvName, Bindings * attrs, Value & v)
auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo));
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo {
.hash = { DerivationOutput::CAFixed dof {
.method = method, .ca = ContentAddress::fromParts(
.hash = h, std::move(method),
}, std::move(h)),
.references = {}, };
});
drv.env["out"] = state.store->printStorePath(outPath); drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
drv.outputs.insert_or_assign("out", drv.outputs.insert_or_assign("out", std::move(dof));
DerivationOutput::CAFixed {
.hash = FixedOutputHash {
.method = method,
.hash = std::move(h),
},
});
} }
else if (contentAddressed || isImpure) { else if (contentAddressed || isImpure) {
@ -1332,13 +1325,13 @@ drvName, Bindings * attrs, Value & v)
if (isImpure) if (isImpure)
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput::Impure { DerivationOutput::Impure {
.method = method, .method = method.raw,
.hashType = ht, .hashType = ht,
}); });
else else
drv.outputs.insert_or_assign(i, drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating { DerivationOutput::CAFloating {
.method = method, .method = method.raw,
.hashType = ht, .hashType = ht,
}); });
} }
@ -1403,7 +1396,7 @@ drvName, Bindings * attrs, Value & v)
NixStringContextElem::DrvDeep { .drvPath = drvPath }, NixStringContextElem::DrvDeep { .drvPath = drvPath },
}); });
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
mkOutputString(state, result, drvPath, drv, i); mkOutputString(state, result, drvPath, i);
v.mkAttrs(result); v.mkAttrs(result);
} }

View file

@ -1,4 +1,5 @@
#include "print.hh" #include "print.hh"
#include <unordered_set>
namespace nix { namespace nix {
@ -25,11 +26,26 @@ printLiteralBool(std::ostream & str, bool boolean)
return str; return str;
} }
// Returns `true' is a string is a reserved keyword which requires quotation
// when printing attribute set field names.
//
// This list should generally be kept in sync with `./lexer.l'.
// You can test if a keyword needs to be added by running:
// $ nix eval --expr '{ <KEYWORD> = 1; }'
// For example `or' doesn't need to be quoted.
bool isReservedKeyword(const std::string_view str)
{
static const std::unordered_set<std::string_view> reservedKeywords = {
"if", "then", "else", "assert", "with", "let", "in", "rec", "inherit"
};
return reservedKeywords.contains(str);
}
std::ostream & std::ostream &
printIdentifier(std::ostream & str, std::string_view s) { printIdentifier(std::ostream & str, std::string_view s) {
if (s.empty()) if (s.empty())
str << "\"\""; str << "\"\"";
else if (s == "if") // FIXME: handle other keywords else if (isReservedKeyword(s))
str << '"' << s << '"'; str << '"' << s << '"';
else { else {
char c = s[0]; char c = s[0];
@ -50,10 +66,10 @@ printIdentifier(std::ostream & str, std::string_view s) {
return str; return str;
} }
// FIXME: keywords
static bool isVarName(std::string_view s) static bool isVarName(std::string_view s)
{ {
if (s.size() == 0) return false; if (s.size() == 0) return false;
if (isReservedKeyword(s)) return false;
char c = s[0]; char c = s[0];
if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false;
for (auto & i : s) for (auto & i : s)

View file

@ -35,6 +35,12 @@ namespace nix {
*/ */
std::ostream & printAttributeName(std::ostream & o, std::string_view s); std::ostream & printAttributeName(std::ostream & o, std::string_view s);
/**
* Returns `true' is a string is a reserved keyword which requires quotation
* when printing attribute set field names.
*/
bool isReservedKeyword(const std::string_view str);
/** /**
* Print a string as an identifier in the Nix expression language syntax. * Print a string as an identifier in the Nix expression language syntax.
* *

View file

@ -0,0 +1,65 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "tests/derived-path.hh"
#include "tests/libexpr.hh"
namespace nix {
// Testing of trivial expressions
class DerivedPathExpressionTest : public LibExprTest {};
// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
// no a real fixture.
//
// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
TEST_F(DerivedPathExpressionTest, force_init)
{
}
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_opaque_path_round_trip,
(const DerivedPath::Opaque & o))
{
auto * v = state.allocValue();
state.mkStorePathString(o.path, *v);
auto d = state.coerceToDerivedPath(noPos, *v, "");
RC_ASSERT(DerivedPath { o } == d);
}
// TODO use DerivedPath::Built for parameter once it supports a single output
// path only.
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_built_path_placeholder_round_trip,
(const StorePath & drvPath, const StorePathName & outputName))
{
auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, std::nullopt);
auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
DerivedPath::Built b {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName.name },
};
RC_ASSERT(DerivedPath { b } == d);
}
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_built_path_out_path_round_trip,
(const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
{
auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, outPath);
auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
DerivedPath::Built b {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName.name },
};
RC_ASSERT(DerivedPath { b } == d);
}
} /* namespace nix */

View file

@ -95,13 +95,15 @@ Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitra
Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
{ {
switch (*gen::inRange<uint8_t>(0, 2)) { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {
case 0: case 0:
return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>()); return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>());
case 1: case 1:
return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>()); return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>());
default: case 2:
return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>()); return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>());
default:
assert(false);
} }
} }

View file

@ -274,11 +274,13 @@ void DerivationGoal::haveDerivation()
) )
) )
); );
else else {
auto * cap = getDerivationCA(*drv);
addWaitee(upcast_goal(worker.makePathSubstitutionGoal( addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
status.known->path, status.known->path,
buildMode == bmRepair ? Repair : NoRepair, buildMode == bmRepair ? Repair : NoRepair,
getDerivationCA(*drv)))); cap ? std::optional { *cap } : std::nullopt)));
}
} }
if (waitees.empty()) /* to prevent hang (no wake-up event) */ if (waitees.empty()) /* to prevent hang (no wake-up event) */
@ -1020,43 +1022,33 @@ void DerivationGoal::resolvedFinished()
StorePathSet outputPaths; StorePathSet outputPaths;
// `wantedOutputs` might merely indicate “all the outputs” for (auto & outputName : resolvedDrv.outputNames()) {
auto realWantedOutputs = std::visit(overloaded { auto initialOutput = get(initialOutputs, outputName);
[&](const OutputsSpec::All &) { auto resolvedHash = get(resolvedHashes, outputName);
return resolvedDrv.outputNames();
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, wantedOutputs.raw());
for (auto & wantedOutput : realWantedOutputs) {
auto initialOutput = get(initialOutputs, wantedOutput);
auto resolvedHash = get(resolvedHashes, wantedOutput);
if ((!initialOutput) || (!resolvedHash)) if ((!initialOutput) || (!resolvedHash))
throw Error( throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
worker.store.printStorePath(drvPath), wantedOutput); worker.store.printStorePath(drvPath), outputName);
auto realisation = [&]{ auto realisation = [&]{
auto take1 = get(resolvedResult.builtOutputs, wantedOutput); auto take1 = get(resolvedResult.builtOutputs, outputName);
if (take1) return *take1; if (take1) return *take1;
/* The above `get` should work. But sateful tracking of /* The above `get` should work. But sateful tracking of
outputs in resolvedResult, this can get out of sync with the outputs in resolvedResult, this can get out of sync with the
store, which is our actual source of truth. For now we just store, which is our actual source of truth. For now we just
check the store directly if it fails. */ check the store directly if it fails. */
auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }); auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, outputName });
if (take2) return *take2; if (take2) return *take2;
throw Error( throw Error(
"derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); worker.store.printStorePath(resolvedDrvGoal->drvPath), outputName);
}(); }();
if (drv->type().isPure()) { if (drv->type().isPure()) {
auto newRealisation = realisation; auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
newRealisation.signatures.clear(); newRealisation.signatures.clear();
if (!drv->type().isFixed()) if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
@ -1064,7 +1056,7 @@ void DerivationGoal::resolvedFinished()
worker.store.registerDrvOutput(newRealisation); worker.store.registerDrvOutput(newRealisation);
} }
outputPaths.insert(realisation.outPath); outputPaths.insert(realisation.outPath);
builtOutputs.emplace(wantedOutput, realisation); builtOutputs.emplace(outputName, realisation);
} }
runPostBuildHook( runPostBuildHook(
@ -1406,7 +1398,7 @@ std::pair<bool, SingleDrvOutputs> DerivationGoal::checkPathValidity()
); );
} }
} }
if (info.wanted && info.known && info.known->isValid()) if (info.known && info.known->isValid())
validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path });
} }
@ -1457,8 +1449,9 @@ void DerivationGoal::done(
mcRunningBuilds.reset(); mcRunningBuilds.reset();
if (buildResult.success()) { if (buildResult.success()) {
assert(!builtOutputs.empty()); auto wantedBuiltOutputs = filterDrvOutputs(wantedOutputs, std::move(builtOutputs));
buildResult.builtOutputs = std::move(builtOutputs); assert(!wantedBuiltOutputs.empty());
buildResult.builtOutputs = std::move(wantedBuiltOutputs);
if (status == BuildResult::Built) if (status == BuildResult::Built)
worker.doneBuilds++; worker.doneBuilds++;
} else { } else {

View file

@ -306,15 +306,13 @@ struct DerivationGoal : public Goal
* Update 'initialOutputs' to determine the current status of the * Update 'initialOutputs' to determine the current status of the
* outputs of the derivation. Also returns a Boolean denoting * outputs of the derivation. Also returns a Boolean denoting
* whether all outputs are valid and non-corrupt, and a * whether all outputs are valid and non-corrupt, and a
* 'SingleDrvOutputs' structure containing the valid and wanted * 'SingleDrvOutputs' structure containing the valid outputs.
* outputs.
*/ */
std::pair<bool, SingleDrvOutputs> checkPathValidity(); std::pair<bool, SingleDrvOutputs> checkPathValidity();
/** /**
* Aborts if any output is not valid or corrupt, and otherwise * Aborts if any output is not valid or corrupt, and otherwise
* returns a 'SingleDrvOutputs' structure containing the wanted * returns a 'SingleDrvOutputs' structure containing all outputs.
* outputs.
*/ */
SingleDrvOutputs assertPathValidity(); SingleDrvOutputs assertPathValidity();
@ -335,6 +333,8 @@ struct DerivationGoal : public Goal
void waiteeDone(GoalPtr waitee, ExitCode result) override; void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths); StorePathSet exportReferences(const StorePathSet & storePaths);
JobCategory jobCategory() override { return JobCategory::Build; };
}; };
MakeError(NotDeterministic, BuildError); MakeError(NotDeterministic, BuildError);

View file

@ -21,7 +21,7 @@ class Worker;
class DrvOutputSubstitutionGoal : public Goal { class DrvOutputSubstitutionGoal : public Goal {
/** /**
* The drv output we're trying to substitue * The drv output we're trying to substitute
*/ */
DrvOutput id; DrvOutput id;
@ -72,6 +72,8 @@ public:
void work() override; void work() override;
void handleEOF(int fd) override; void handleEOF(int fd) override;
JobCategory jobCategory() override { return JobCategory::Substitution; };
}; };
} }

View file

@ -34,6 +34,17 @@ typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals;
*/ */
typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
/**
* Used as a hint to the worker on how to schedule a particular goal. For example,
* builds are typically CPU- and memory-bound, while substitutions are I/O bound.
* Using this information, the worker might decide to schedule more or fewer goals
* of each category in parallel.
*/
enum struct JobCategory {
Build,
Substitution,
};
struct Goal : public std::enable_shared_from_this<Goal> struct Goal : public std::enable_shared_from_this<Goal>
{ {
typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
@ -150,6 +161,8 @@ public:
void amDone(ExitCode result, std::optional<Error> ex = {}); void amDone(ExitCode result, std::optional<Error> ex = {});
virtual void cleanup() { } virtual void cleanup() { }
virtual JobCategory jobCategory() = 0;
}; };
void addToWeakGoals(WeakGoals & goals, GoalPtr p); void addToWeakGoals(WeakGoals & goals, GoalPtr p);

View file

@ -2421,37 +2421,51 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
throw BuildError( throw BuildError(
"output path %1% without valid stats info", "output path %1% without valid stats info",
actualPath); actualPath);
if (outputHash.method == FileIngestionMethod::Flat) { if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextIngestionMethod {} })
{
/* The output path should be a regular file without execute permission. */ /* The output path should be a regular file without execute permission. */
if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0)
throw BuildError( throw BuildError(
"output path '%1%' should be a non-executable regular file " "output path '%1%' should be a non-executable regular file "
"since recursive hashing is not enabled (outputHashMode=flat)", "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)",
actualPath); actualPath);
} }
rewriteOutput(); rewriteOutput();
/* FIXME optimize and deduplicate with addToStore */ /* FIXME optimize and deduplicate with addToStore */
std::string oldHashPart { scratchPath->hashPart() }; std::string oldHashPart { scratchPath->hashPart() };
HashModuloSink caSink { outputHash.hashType, oldHashPart }; HashModuloSink caSink { outputHash.hashType, oldHashPart };
switch (outputHash.method) { std::visit(overloaded {
case FileIngestionMethod::Recursive: [&](const TextIngestionMethod &) {
dumpPath(actualPath, caSink); readFile(actualPath, caSink);
break; },
case FileIngestionMethod::Flat: [&](const FileIngestionMethod & m2) {
readFile(actualPath, caSink); switch (m2) {
break; case FileIngestionMethod::Recursive:
} dumpPath(actualPath, caSink);
break;
case FileIngestionMethod::Flat:
readFile(actualPath, caSink);
break;
}
},
}, outputHash.method.raw);
auto got = caSink.finish().first; auto got = caSink.finish().first;
auto optCA = ContentAddressWithReferences::fromPartsOpt(
outputHash.method,
std::move(got),
rewriteRefs());
if (!optCA) {
// TODO track distinct failure modes separately (at the time of
// writing there is just one but `nullopt` is unclear) so this
// message can't get out of sync.
throw BuildError("output path '%s' has illegal content address, probably a spurious self-reference with text hashing");
}
ValidPathInfo newInfo0 { ValidPathInfo newInfo0 {
worker.store, worker.store,
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
FixedOutputInfo { *std::move(optCA),
.hash = {
.method = outputHash.method,
.hash = got,
},
.references = rewriteRefs(),
},
Hash::dummy, Hash::dummy,
}; };
if (*scratchPath != newInfo0.path) { if (*scratchPath != newInfo0.path) {
@ -2498,13 +2512,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
auto wanted = dof.ca.getHash();
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.hash.method, .method = dof.ca.getMethod(),
.hashType = dof.hash.hash.type, .hashType = wanted.type,
}); });
/* Check wanted hash */ /* Check wanted hash */
const Hash & wanted = dof.hash.hash;
assert(newInfo0.ca); assert(newInfo0.ca);
auto got = newInfo0.ca->getHash(); auto got = newInfo0.ca->getHash();
if (wanted != got) { if (wanted != got) {
@ -2517,6 +2532,11 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
wanted.to_string(SRI, true), wanted.to_string(SRI, true),
got.to_string(SRI, true))); got.to_string(SRI, true)));
} }
if (!newInfo0.references.empty())
delayedException = std::make_exception_ptr(
BuildError("illegal path references in fixed-output derivation '%s'",
worker.store.printStorePath(drvPath)));
return newInfo0; return newInfo0;
}, },
@ -2696,8 +2716,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
signRealisation(thisRealisation); signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation); worker.store.registerDrvOutput(thisRealisation);
} }
if (wantedOutputs.contains(outputName)) builtOutputs.emplace(outputName, thisRealisation);
builtOutputs.emplace(outputName, thisRealisation);
} }
return builtOutputs; return builtOutputs;

View file

@ -200,11 +200,10 @@ void PathSubstitutionGoal::tryToRun()
{ {
trace("trying to run"); trace("trying to run");
/* Make sure that we are allowed to start a build. Note that even /* Make sure that we are allowed to start a substitution. Note that even
if maxBuildJobs == 0 (no local builds allowed), we still allow if maxSubstitutionJobs == 0, we still allow a substituter to run. This
a substituter to run. This is because substitutions cannot be prevents infinite waiting. */
distributed to another machine via the build hook. */ if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) {
worker.waitForBuildSlot(shared_from_this()); worker.waitForBuildSlot(shared_from_this());
return; return;
} }

View file

@ -115,6 +115,8 @@ public:
void handleEOF(int fd) override; void handleEOF(int fd) override;
void cleanup() override; void cleanup() override;
JobCategory jobCategory() override { return JobCategory::Substitution; };
}; };
} }

View file

@ -18,6 +18,7 @@ Worker::Worker(Store & store, Store & evalStore)
{ {
/* Debugging: prevent recursive workers. */ /* Debugging: prevent recursive workers. */
nrLocalBuilds = 0; nrLocalBuilds = 0;
nrSubstitutions = 0;
lastWokenUp = steady_time_point::min(); lastWokenUp = steady_time_point::min();
permanentFailure = false; permanentFailure = false;
timedOut = false; timedOut = false;
@ -176,6 +177,12 @@ unsigned Worker::getNrLocalBuilds()
} }
unsigned Worker::getNrSubstitutions()
{
return nrSubstitutions;
}
void Worker::childStarted(GoalPtr goal, const std::set<int> & fds, void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
bool inBuildSlot, bool respectTimeouts) bool inBuildSlot, bool respectTimeouts)
{ {
@ -187,7 +194,10 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
child.inBuildSlot = inBuildSlot; child.inBuildSlot = inBuildSlot;
child.respectTimeouts = respectTimeouts; child.respectTimeouts = respectTimeouts;
children.emplace_back(child); children.emplace_back(child);
if (inBuildSlot) nrLocalBuilds++; if (inBuildSlot) {
if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
else nrLocalBuilds++;
}
} }
@ -198,8 +208,13 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
if (i == children.end()) return; if (i == children.end()) return;
if (i->inBuildSlot) { if (i->inBuildSlot) {
assert(nrLocalBuilds > 0); if (goal->jobCategory() == JobCategory::Substitution) {
nrLocalBuilds--; assert(nrSubstitutions > 0);
nrSubstitutions--;
} else {
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
}
} }
children.erase(i); children.erase(i);
@ -220,7 +235,9 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
void Worker::waitForBuildSlot(GoalPtr goal) void Worker::waitForBuildSlot(GoalPtr goal)
{ {
debug("wait for build slot"); debug("wait for build slot");
if (getNrLocalBuilds() < settings.maxBuildJobs) bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution;
if ((!isSubstitutionGoal && getNrLocalBuilds() < settings.maxBuildJobs) ||
(isSubstitutionGoal && getNrSubstitutions() < settings.maxSubstitutionJobs))
wakeUp(goal); /* we can do it right away */ wakeUp(goal); /* we can do it right away */
else else
addToWeakGoals(wantingToBuild, goal); addToWeakGoals(wantingToBuild, goal);

View file

@ -88,11 +88,16 @@ private:
std::list<Child> children; std::list<Child> children;
/** /**
* Number of build slots occupied. This includes local builds and * Number of build slots occupied. This includes local builds but does not
* substitutions but not remote builds via the build hook. * include substitutions or remote builds via the build hook.
*/ */
unsigned int nrLocalBuilds; unsigned int nrLocalBuilds;
/**
* Number of substitution slots occupied.
*/
unsigned int nrSubstitutions;
/** /**
* Maps used to prevent multiple instantiations of a goal for the * Maps used to prevent multiple instantiations of a goal for the
* same derivation / path. * same derivation / path.
@ -220,12 +225,16 @@ public:
void wakeUp(GoalPtr goal); void wakeUp(GoalPtr goal);
/** /**
* Return the number of local build and substitution processes * Return the number of local build processes currently running (but not
* currently running (but not remote builds via the build * remote builds via the build hook).
* hook).
*/ */
unsigned int getNrLocalBuilds(); unsigned int getNrLocalBuilds();
/**
* Return the number of substitution processes currently running.
*/
unsigned int getNrSubstitutions();
/** /**
* Registers a running child process. `inBuildSlot` means that * Registers a running child process. `inBuildSlot` means that
* the process counts towards the jobs limit. * the process counts towards the jobs limit.

View file

@ -21,6 +21,27 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m)
} }
} }
std::string ContentAddressMethod::renderPrefix() const
{
return std::visit(overloaded {
[](TextIngestionMethod) -> std::string { return "text:"; },
[](FileIngestionMethod m2) {
/* Not prefixed for back compat with things that couldn't produce text before. */
return makeFileIngestionPrefix(m2);
},
}, raw);
}
ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
{
ContentAddressMethod method = FileIngestionMethod::Flat;
if (splitPrefix(m, "r:"))
method = FileIngestionMethod::Recursive;
else if (splitPrefix(m, "text:"))
method = TextIngestionMethod {};
return method;
}
std::string ContentAddress::render() const std::string ContentAddress::render() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
@ -36,14 +57,14 @@ std::string ContentAddress::render() const
}, raw); }, raw);
} }
std::string ContentAddressMethod::render() const std::string ContentAddressMethod::render(HashType ht) const
{ {
return std::visit(overloaded { return std::visit(overloaded {
[](const TextHashMethod & th) { [&](const TextIngestionMethod & th) {
return std::string{"text:"} + printHashType(htSHA256); return std::string{"text:"} + printHashType(ht);
}, },
[](const FixedOutputHashMethod & fshm) { [&](const FileIngestionMethod & fim) {
return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht);
} }
}, raw); }, raw);
} }
@ -51,7 +72,7 @@ std::string ContentAddressMethod::render() const
/** /**
* Parses content address strings up to the hash. * Parses content address strings up to the hash.
*/ */
static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix(std::string_view & rest)
{ {
std::string_view wholeInput { rest }; std::string_view wholeInput { rest };
@ -75,46 +96,47 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r
if (prefix == "text") { if (prefix == "text") {
// No parsing of the ingestion method, "text" only support flat. // No parsing of the ingestion method, "text" only support flat.
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
if (hashType != htSHA256) return {
throw Error("text content address hash should use %s, but instead uses %s", TextIngestionMethod {},
printHashType(htSHA256), printHashType(hashType)); std::move(hashType),
return TextHashMethod {}; };
} else if (prefix == "fixed") { } else if (prefix == "fixed") {
// Parse method // Parse method
auto method = FileIngestionMethod::Flat; auto method = FileIngestionMethod::Flat;
if (splitPrefix(rest, "r:")) if (splitPrefix(rest, "r:"))
method = FileIngestionMethod::Recursive; method = FileIngestionMethod::Recursive;
HashType hashType = parseHashType_(); HashType hashType = parseHashType_();
return FixedOutputHashMethod { return {
.fileIngestionMethod = method, std::move(method),
.hashType = std::move(hashType), std::move(hashType),
}; };
} else } else
throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
} }
ContentAddress ContentAddress::parse(std::string_view rawCa) { ContentAddress ContentAddress::parse(std::string_view rawCa)
{
auto rest = rawCa; auto rest = rawCa;
ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
auto hashType = hashType_; // work around clang bug
return std::visit( return std::visit(overloaded {
overloaded { [&](TextIngestionMethod &) {
[&](TextHashMethod & thm) { return ContentAddress(TextHash {
return ContentAddress(TextHash { .hash = Hash::parseNonSRIUnprefixed(rest, hashType)
.hash = Hash::parseNonSRIUnprefixed(rest, htSHA256) });
}); },
}, [&](FileIngestionMethod & fim) {
[&](FixedOutputHashMethod & fohMethod) { return ContentAddress(FixedOutputHash {
return ContentAddress(FixedOutputHash { .method = fim,
.method = fohMethod.fileIngestionMethod, .hash = Hash::parseNonSRIUnprefixed(rest, hashType),
.hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), });
}); },
}, }, caMethod.raw);
}, caMethod.raw);
} }
ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
{ {
std::string asPrefix = std::string{caMethod} + ":"; std::string asPrefix = std::string{caMethod} + ":";
// parseContentAddressMethodPrefix takes its argument by reference // parseContentAddressMethodPrefix takes its argument by reference
@ -134,6 +156,36 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
return ca ? ca->render() : ""; return ca ? ca->render() : "";
} }
ContentAddress ContentAddress::fromParts(
ContentAddressMethod method, Hash hash) noexcept
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> ContentAddress {
return TextHash {
.hash = std::move(hash),
};
},
[&](FileIngestionMethod m2) -> ContentAddress {
return FixedOutputHash {
.method = std::move(m2),
.hash = std::move(hash),
};
},
}, method.raw);
}
ContentAddressMethod ContentAddress::getMethod() const
{
return std::visit(overloaded {
[](const TextHash & th) -> ContentAddressMethod {
return TextIngestionMethod {};
},
[](const FixedOutputHash & fsh) -> ContentAddressMethod {
return fsh.method;
},
}, raw);
}
const Hash & ContentAddress::getHash() const const Hash & ContentAddress::getHash() const
{ {
return std::visit(overloaded { return std::visit(overloaded {
@ -146,6 +198,12 @@ const Hash & ContentAddress::getHash() const
}, raw); }, raw);
} }
std::string ContentAddress::printMethodAlgo() const
{
return getMethod().renderPrefix()
+ printHashType(getHash().type);
}
bool StoreReferences::empty() const bool StoreReferences::empty() const
{ {
return !self && others.empty(); return !self && others.empty();
@ -156,7 +214,8 @@ size_t StoreReferences::size() const
return (self ? 1 : 0) + others.size(); return (self ? 1 : 0) + others.size();
} }
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
{
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHash & h) -> ContentAddressWithReferences { [&](const TextHash & h) -> ContentAddressWithReferences {
return TextInfo { return TextInfo {
@ -173,4 +232,56 @@ ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const Con
}, ca.raw); }, ca.raw);
} }
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> std::optional<ContentAddressWithReferences> {
if (refs.self)
return std::nullopt;
return ContentAddressWithReferences {
TextInfo {
.hash = { .hash = std::move(hash) },
.references = std::move(refs.others),
}
};
},
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
return ContentAddressWithReferences {
FixedOutputInfo {
.hash = {
.method = m2,
.hash = std::move(hash),
},
.references = std::move(refs),
}
};
},
}, method.raw);
}
ContentAddressMethod ContentAddressWithReferences::getMethod() const
{
return std::visit(overloaded {
[](const TextInfo & th) -> ContentAddressMethod {
return TextIngestionMethod {};
},
[](const FixedOutputInfo & fsh) -> ContentAddressMethod {
return fsh.hash.method;
},
}, raw);
}
Hash ContentAddressWithReferences::getHash() const
{
return std::visit(overloaded {
[](const TextInfo & th) {
return th.hash.hash;
},
[](const FixedOutputInfo & fsh) {
return fsh.hash.hash;
},
}, raw);
}
} }

View file

@ -21,8 +21,14 @@ namespace nix {
* *
* Somewhat obscure, used by \ref Derivation derivations and * Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently. * `builtins.toFile` currently.
*
* TextIngestionMethod is identical to FileIngestionMethod::Fixed except that
* the former may not have self-references and is tagged `text:${algo}:${hash}`
* rather than `fixed:${algo}:${hash}`. The contents of the store path are
* ingested and hashed identically, aside from the slightly different tag and
* restriction on self-references.
*/ */
struct TextHashMethod : std::monostate { }; struct TextIngestionMethod : std::monostate { };
/** /**
* An enumeration of the main ways we can serialize file system * An enumeration of the main ways we can serialize file system
@ -46,13 +52,6 @@ enum struct FileIngestionMethod : uint8_t {
*/ */
std::string makeFileIngestionPrefix(FileIngestionMethod m); std::string makeFileIngestionPrefix(FileIngestionMethod m);
struct FixedOutputHashMethod {
FileIngestionMethod fileIngestionMethod;
HashType hashType;
GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType);
};
/** /**
* An enumeration of all the ways we can serialize file system objects. * An enumeration of all the ways we can serialize file system objects.
* *
@ -64,8 +63,8 @@ struct FixedOutputHashMethod {
struct ContentAddressMethod struct ContentAddressMethod
{ {
typedef std::variant< typedef std::variant<
TextHashMethod, TextIngestionMethod,
FixedOutputHashMethod FileIngestionMethod
> Raw; > Raw;
Raw raw; Raw raw;
@ -77,9 +76,36 @@ struct ContentAddressMethod
: raw(std::forward<decltype(arg)>(arg)...) : raw(std::forward<decltype(arg)>(arg)...)
{ } { }
static ContentAddressMethod parse(std::string_view rawCaMethod);
std::string render() const; /**
* Parse the prefix tag which indicates how the files
* were ingested, with the fixed output case not prefixed for back
* compat.
*
* @param [in] m A string that should begin with the prefix.
* @param [out] m The remainder of the string after the prefix.
*/
static ContentAddressMethod parsePrefix(std::string_view & m);
/**
* Render the prefix tag which indicates how the files wre ingested.
*
* The rough inverse of `parsePrefix()`.
*/
std::string renderPrefix() const;
/**
* Parse a content addressing method and hash type.
*/
static std::pair<ContentAddressMethod, HashType> parse(std::string_view rawCaMethod);
/**
* Render a content addressing method and hash type in a
* nicer way, prefixing both cases.
*
* The rough inverse of `parse()`.
*/
std::string render(HashType ht) const;
}; };
@ -147,8 +173,9 @@ struct ContentAddress
{ } { }
/** /**
* Compute the content-addressability assertion (ValidPathInfo::ca) for * Compute the content-addressability assertion
* paths created by Store::makeFixedOutputPath() / Store::addToStore(). * (`ValidPathInfo::ca`) for paths created by
* `Store::makeFixedOutputPath()` / `Store::addToStore()`.
*/ */
std::string render() const; std::string render() const;
@ -156,9 +183,27 @@ struct ContentAddress
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt); static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
/**
* Create a `ContentAddress` from 2 parts:
*
* @param method Way ingesting the file system data.
*
* @param hash Hash of ingested file system data.
*/
static ContentAddress fromParts(
ContentAddressMethod method, Hash hash) noexcept;
ContentAddressMethod getMethod() const;
const Hash & getHash() const; const Hash & getHash() const;
std::string printMethodAlgo() const;
}; };
/**
* Render the `ContentAddress` if it exists to a string, return empty
* string otherwise.
*/
std::string renderContentAddress(std::optional<ContentAddress> ca); std::string renderContentAddress(std::optional<ContentAddress> ca);
@ -244,10 +289,29 @@ struct ContentAddressWithReferences
{ } { }
/** /**
* Create a ContentAddressWithReferences from a mere ContentAddress, by * Create a `ContentAddressWithReferences` from a mere
* assuming no references in all cases. * `ContentAddress`, by claiming no references.
*/ */
static ContentAddressWithReferences withoutRefs(const ContentAddress &); static ContentAddressWithReferences withoutRefs(const ContentAddress &) noexcept;
/**
* Create a `ContentAddressWithReferences` from 3 parts:
*
* @param method Way ingesting the file system data.
*
* @param hash Hash of ingested file system data.
*
* @param refs References to other store objects or oneself.
*
* Do note that not all combinations are supported; `nullopt` is
* returns for invalid combinations.
*/
static std::optional<ContentAddressWithReferences> fromPartsOpt(
ContentAddressMethod method, Hash hash, StoreReferences refs) noexcept;
ContentAddressMethod getMethod() const;
Hash getHash() const;
}; };
} }

View file

@ -401,18 +401,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto pathInfo = [&]() { auto pathInfo = [&]() {
// NB: FramedSource must be out of scope before logger->stopWork(); // NB: FramedSource must be out of scope before logger->stopWork();
ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr);
auto hashType = hashType_; // work around clang bug
FramedSource source(from); FramedSource source(from);
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
return std::visit(overloaded { return std::visit(overloaded {
[&](const TextHashMethod &) { [&](const TextIngestionMethod &) {
if (hashType != htSHA256)
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
name, printHashType(hashType));
// We could stream this by changing Store // We could stream this by changing Store
std::string contents = source.drain(); std::string contents = source.drain();
auto path = store->addTextToStore(name, contents, refs, repair); auto path = store->addTextToStore(name, contents, refs, repair);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
[&](const FixedOutputHashMethod & fohm) { [&](const FileIngestionMethod & fim) {
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs);
return store->queryPathInfo(path); return store->queryPathInfo(path);
}, },
}, contentAddressMethod.raw); }, contentAddressMethod.raw);

View file

@ -2,6 +2,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
#include "split.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include <boost/container/small_vector.hpp> #include <boost/container/small_vector.hpp>
@ -35,9 +36,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{ {
return store.makeFixedOutputPath( return store.makeFixedOutputPathFromCA(
outputPathName(drvName, outputName), outputPathName(drvName, outputName),
{ hash, {} }); ContentAddressWithReferences::withoutRefs(ca));
} }
@ -211,29 +212,27 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
static DerivationOutput parseDerivationOutput(const Store & store, static DerivationOutput parseDerivationOutput(const Store & store,
std::string_view pathS, std::string_view hashAlgo, std::string_view hash) std::string_view pathS, std::string_view hashAlgo, std::string_view hashS)
{ {
if (hashAlgo != "") { if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat; ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo);
if (hashAlgo.substr(0, 2) == "r:") { if (method == TextIngestionMethod {})
method = FileIngestionMethod::Recursive; experimentalFeatureSettings.require(Xp::DynamicDerivations);
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo); const auto hashType = parseHashType(hashAlgo);
if (hash == "impure") { if (hashS == "impure") {
experimentalFeatureSettings.require(Xp::ImpureDerivations); experimentalFeatureSettings.require(Xp::ImpureDerivations);
assert(pathS == ""); assert(pathS == "");
return DerivationOutput::Impure { return DerivationOutput::Impure {
.method = std::move(method), .method = std::move(method),
.hashType = std::move(hashType), .hashType = std::move(hashType),
}; };
} else if (hash != "") { } else if (hashS != "") {
validatePath(pathS); validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
return DerivationOutput::CAFixed { return DerivationOutput::CAFixed {
.hash = FixedOutputHash { .ca = ContentAddress::fromParts(
.method = std::move(method), std::move(method),
.hash = Hash::parseNonSRIUnprefixed(hash, hashType), std::move(hash)),
},
}; };
} else { } else {
experimentalFeatureSettings.require(Xp::CaDerivations); experimentalFeatureSettings.require(Xp::CaDerivations);
@ -393,12 +392,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType));
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
@ -409,7 +408,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutputImpure & doi) { [&](const DerivationOutputImpure & doi) {
// FIXME // FIXME
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType));
s += ','; printUnquotedString(s, "impure"); s += ','; printUnquotedString(s, "impure");
} }
}, i.second.raw()); }, i.second.raw());
@ -626,8 +625,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
for (const auto & i : drv.outputs) { for (const auto & i : drv.outputs) {
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw()); auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
auto hash = hashString(htSHA256, "fixed:out:" auto hash = hashString(htSHA256, "fixed:out:"
+ dof.hash.printMethodAlgo() + ":" + dof.ca.printMethodAlgo() + ":"
+ dof.hash.hash.to_string(Base16, false) + ":" + dof.ca.getHash().to_string(Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
@ -777,12 +776,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first)) out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.hash.printMethodAlgo() << dof.ca.printMethodAlgo()
<< dof.hash.hash.to_string(Base16, false); << dof.ca.getHash().to_string(Base16, false);
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
out << "" out << ""
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << (dof.method.renderPrefix() + printHashType(dof.hashType))
<< ""; << "";
}, },
[&](const DerivationOutput::Deferred &) { [&](const DerivationOutput::Deferred &) {
@ -792,7 +791,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}, },
[&](const DerivationOutput::Impure & doi) { [&](const DerivationOutput::Impure & doi) {
out << "" out << ""
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) << (doi.method.renderPrefix() + printHashType(doi.hashType))
<< "impure"; << "impure";
}, },
}, i.second.raw()); }, i.second.raw());
@ -942,7 +941,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
envHasRightPath(doia.path, i.first); envHasRightPath(doia.path, i.first);
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} }); auto path = dof.path(store, drvName, i.first);
envHasRightPath(path, i.first); envHasRightPath(path, i.first);
}, },
[&](const DerivationOutput::CAFloating &) { [&](const DerivationOutput::CAFloating &) {
@ -971,15 +970,16 @@ nlohmann::json DerivationOutput::toJSON(
}, },
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.hash.printMethodAlgo(); res["hashAlgo"] = dof.ca.printMethodAlgo();
res["hash"] = dof.hash.hash.to_string(Base16, false); res["hash"] = dof.ca.getHash().to_string(Base16, false);
// FIXME print refs?
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType);
}, },
[&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) { [&](const DerivationOutput::Impure & doi) {
res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType);
res["impure"] = true; res["impure"] = true;
}, },
}, raw()); }, raw());
@ -998,15 +998,15 @@ DerivationOutput DerivationOutput::fromJSON(
for (const auto & [key, _] : json) for (const auto & [key, _] : json)
keys.insert(key); keys.insert(key);
auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> { auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> {
std::string hashAlgo = json["hashAlgo"]; std::string hashAlgo = json["hashAlgo"];
auto method = FileIngestionMethod::Flat; // remaining to parse, will be mutated by parsers
if (hashAlgo.substr(0, 2) == "r:") { std::string_view s = hashAlgo;
method = FileIngestionMethod::Recursive; ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
hashAlgo = hashAlgo.substr(2); if (method == TextIngestionMethod {})
} xpSettings.require(Xp::DynamicDerivations);
auto hashType = parseHashType(hashAlgo); auto hashType = parseHashType(s);
return { method, hashType }; return { std::move(method), std::move(hashType) };
}; };
if (keys == (std::set<std::string_view> { "path" })) { if (keys == (std::set<std::string_view> { "path" })) {
@ -1018,10 +1018,9 @@ DerivationOutput DerivationOutput::fromJSON(
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) { else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
auto [method, hashType] = methodAlgo(); auto [method, hashType] = methodAlgo();
auto dof = DerivationOutput::CAFixed { auto dof = DerivationOutput::CAFixed {
.hash = { .ca = ContentAddress::fromParts(
.method = method, std::move(method),
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
},
}; };
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
throw Error("Path doesn't match derivation output"); throw Error("Path doesn't match derivation output");
@ -1032,8 +1031,8 @@ DerivationOutput DerivationOutput::fromJSON(
xpSettings.require(Xp::CaDerivations); xpSettings.require(Xp::CaDerivations);
auto [method, hashType] = methodAlgo(); auto [method, hashType] = methodAlgo();
return DerivationOutput::CAFloating { return DerivationOutput::CAFloating {
.method = method, .method = std::move(method),
.hashType = hashType, .hashType = std::move(hashType),
}; };
} }
@ -1045,7 +1044,7 @@ DerivationOutput DerivationOutput::fromJSON(
xpSettings.require(Xp::ImpureDerivations); xpSettings.require(Xp::ImpureDerivations);
auto [method, hashType] = methodAlgo(); auto [method, hashType] = methodAlgo();
return DerivationOutput::Impure { return DerivationOutput::Impure {
.method = method, .method = std::move(method),
.hashType = hashType, .hashType = hashType,
}; };
} }

View file

@ -36,9 +36,11 @@ struct DerivationOutputInputAddressed
struct DerivationOutputCAFixed struct DerivationOutputCAFixed
{ {
/** /**
* hash used for expected hash computation * Method and hash used for expected hash computation.
*
* References are not allowed by fiat.
*/ */
FixedOutputHash hash; ContentAddress ca;
/** /**
* Return the \ref StorePath "store path" corresponding to this output * Return the \ref StorePath "store path" corresponding to this output
@ -48,7 +50,7 @@ struct DerivationOutputCAFixed
*/ */
StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
GENERATE_CMP(DerivationOutputCAFixed, me->hash); GENERATE_CMP(DerivationOutputCAFixed, me->ca);
}; };
/** /**
@ -61,7 +63,7 @@ struct DerivationOutputCAFloating
/** /**
* How the file system objects will be serialized for hashing * How the file system objects will be serialized for hashing
*/ */
FileIngestionMethod method; ContentAddressMethod method;
/** /**
* How the serialization will be hashed * How the serialization will be hashed
@ -88,7 +90,7 @@ struct DerivationOutputImpure
/** /**
* How the file system objects will be serialized for hashing * How the file system objects will be serialized for hashing
*/ */
FileIngestionMethod method; ContentAddressMethod method;
/** /**
* How the serialization will be hashed * How the serialization will be hashed
@ -343,12 +345,14 @@ struct Derivation : BasicDerivation
Store & store, Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
/* Check that the derivation is valid and does not present any /**
illegal states. * Check that the derivation is valid and does not present any
* illegal states.
This is mainly a matter of checking the outputs, where our C++ *
representation supports all sorts of combinations we do not yet * This is mainly a matter of checking the outputs, where our C++
allow. */ * representation supports all sorts of combinations we do not yet
* allow.
*/
void checkInvariants(Store & store, const StorePath & drvPath) const; void checkInvariants(Store & store, const StorePath & drvPath) const;
Derivation() = default; Derivation() = default;

View file

@ -159,6 +159,15 @@ public:
)", )",
{"build-max-jobs"}}; {"build-max-jobs"}};
Setting<unsigned int> maxSubstitutionJobs{
this, 16, "max-substitution-jobs",
R"(
This option defines the maximum number of substitution jobs that Nix
will try to run in parallel. The default is `16`. The minimum value
one can choose is `1` and lower values will be interpreted as `1`.
)",
{"substitution-max-jobs"}};
Setting<unsigned int> buildCores{ Setting<unsigned int> buildCores{
this, this,
getDefaultCores(), getDefaultCores(),
@ -991,7 +1000,7 @@ public:
this, false, "use-xdg-base-directories", this, false, "use-xdg-base-directories",
R"( R"(
If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`. If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`.
The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md). The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/command-ref/env-common.md).
[XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html

View file

@ -57,12 +57,6 @@ $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(d)/build.cc: $(d)/build.cc:
%.gen.hh: %
@echo 'R"foo(' >> $@.tmp
$(trace-gen) cat $< >> $@.tmp
@echo ')foo"' >> $@.tmp
@mv $@.tmp $@
clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644))

View file

@ -83,14 +83,15 @@ void Store::computeFSClosure(const StorePath & startPath,
} }
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) const ContentAddress * getDerivationCA(const BasicDerivation & drv)
{ {
auto out = drv.outputs.find("out"); auto out = drv.outputs.find("out");
if (out != drv.outputs.end()) { if (out == drv.outputs.end())
if (const auto * v = std::get_if<DerivationOutput::CAFixed>(&out->second.raw())) return nullptr;
return v->hash; if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
return &dof->ca;
} }
return std::nullopt; return nullptr;
} }
void Store::queryMissing(const std::vector<DerivedPath> & targets, void Store::queryMissing(const std::vector<DerivedPath> & targets,
@ -140,7 +141,13 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
if (drvState_->lock()->done) return; if (drvState_->lock()->done) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos); auto * cap = getDerivationCA(*drv);
querySubstitutablePathInfos({
{
outPath,
cap ? std::optional { *cap } : std::nullopt,
},
}, infos);
if (infos.empty()) { if (infos.empty()) {
drvState_->lock()->done = true; drvState_->lock()->done = true;

View file

@ -136,6 +136,19 @@ size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
return good; return good;
} }
SingleDrvOutputs filterDrvOutputs(const OutputsSpec& wanted, SingleDrvOutputs&& outputs)
{
SingleDrvOutputs ret = std::move(outputs);
for (auto it = ret.begin(); it != ret.end(); ) {
if (!wanted.contains(it->first))
it = ret.erase(it);
else
++it;
}
return ret;
}
StorePath RealisedPath::path() const { StorePath RealisedPath::path() const {
return std::visit([](auto && arg) { return arg.getPath(); }, raw); return std::visit([](auto && arg) { return arg.getPath(); }, raw);
} }

View file

@ -12,6 +12,7 @@
namespace nix { namespace nix {
class Store; class Store;
struct OutputsSpec;
/** /**
* A general `Realisation` key. * A general `Realisation` key.
@ -93,6 +94,14 @@ typedef std::map<std::string, Realisation> SingleDrvOutputs;
*/ */
typedef std::map<DrvOutput, Realisation> DrvOutputs; typedef std::map<DrvOutput, Realisation> DrvOutputs;
/**
* Filter a SingleDrvOutputs to include only specific output names
*
* Moves the `outputs` input.
*/
SingleDrvOutputs filterDrvOutputs(const OutputsSpec&, SingleDrvOutputs&&);
struct OpaquePath { struct OpaquePath {
StorePath path; StorePath path;

View file

@ -597,6 +597,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
Source & dump, Source & dump,
std::string_view name, std::string_view name,
ContentAddressMethod caMethod, ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair) RepairFlag repair)
{ {
@ -608,7 +609,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< caMethod.render(); << caMethod.render(hashType);
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn->to << repair; conn->to << repair;
@ -628,26 +629,29 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
std::visit(overloaded { std::visit(overloaded {
[&](const TextHashMethod & thm) -> void { [&](const TextIngestionMethod & thm) -> void {
if (hashType != htSHA256)
throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given",
name, printHashType(hashType));
std::string s = dump.drain(); std::string s = dump.drain();
conn->to << wopAddTextToStore << name << s; conn->to << wopAddTextToStore << name << s;
worker_proto::write(*this, conn->to, references); worker_proto::write(*this, conn->to, references);
conn.processStderr(); conn.processStderr();
}, },
[&](const FixedOutputHashMethod & fohm) -> void { [&](const FileIngestionMethod & fim) -> void {
conn->to conn->to
<< wopAddToStore << wopAddToStore
<< name << name
<< ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */
<< (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) << (fim == FileIngestionMethod::Recursive ? 1 : 0)
<< printHashType(fohm.hashType); << printHashType(hashType);
try { try {
conn->to.written = 0; conn->to.written = 0;
connections->incCapacity(); connections->incCapacity();
{ {
Finally cleanup([&]() { connections->decCapacity(); }); Finally cleanup([&]() { connections->decCapacity(); });
if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { if (fim == FileIngestionMethod::Recursive) {
dump.drainInto(conn->to); dump.drainInto(conn->to);
} else { } else {
std::string contents = dump.drain(); std::string contents = dump.drain();
@ -678,7 +682,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name,
FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references)
{ {
return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; return addCAToStore(dump, name, method, hashType, references, repair)->path;
} }
@ -778,7 +782,7 @@ StorePath RemoteStore::addTextToStore(
RepairFlag repair) RepairFlag repair)
{ {
StringSource source(s); StringSource source(s);
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path;
} }
void RemoteStore::registerDrvOutput(const Realisation & info) void RemoteStore::registerDrvOutput(const Realisation & info)

View file

@ -78,6 +78,7 @@ public:
Source & dump, Source & dump,
std::string_view name, std::string_view name,
ContentAddressMethod caMethod, ContentAddressMethod caMethod,
HashType hashType,
const StorePathSet & references, const StorePathSet & references,
RepairFlag repair); RepairFlag repair);

View file

@ -1022,7 +1022,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
*/ */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath> drvOutputReferences( std::map<DrvOutput, StorePath> drvOutputReferences(
Store & store, Store & store,

View file

@ -26,6 +26,14 @@ class CaDerivationTest : public DerivationTest
} }
}; };
class DynDerivationTest : public DerivationTest
{
void SetUp() override
{
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
}
};
class ImpureDerivationTest : public DerivationTest class ImpureDerivationTest : public DerivationTest
{ {
void SetUp() override void SetUp() override
@ -66,20 +74,47 @@ TEST_JSON(DerivationTest, inputAddressed,
}), }),
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DerivationTest, caFixed, TEST_JSON(DerivationTest, caFixedFlat,
R"({
"hashAlgo": "sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = FixedOutputHash {
.method = FileIngestionMethod::Flat,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
}),
"drv-name", "output-name")
TEST_JSON(DerivationTest, caFixedNAR,
R"({ R"({
"hashAlgo": "r:sha256", "hashAlgo": "r:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})", })",
(DerivationOutput::CAFixed { (DerivationOutput::CAFixed {
.hash = { .ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive, .method = FileIngestionMethod::Recursive,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
}, },
}), }),
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DynDerivationTest, caFixedText,
R"({
"hashAlgo": "text:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = TextHash {
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
}),
"drv-name", "output-name")
TEST_JSON(CaDerivationTest, caFloating, TEST_JSON(CaDerivationTest, caFloating,
R"({ R"({
"hashAlgo": "r:sha256" "hashAlgo": "r:sha256"

View file

@ -27,11 +27,13 @@ Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary() Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
{ {
switch (*gen::inRange<uint8_t>(0, 1)) { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
case 0: case 0:
return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>()); return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>());
default: case 1:
return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>()); return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>());
default:
assert(false);
} }
} }

View file

@ -206,15 +206,17 @@ using namespace nix;
Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary() Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary()
{ {
switch (*gen::inRange<uint8_t>(0, 1)) { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<OutputsSpec::Raw>)) {
case 0: case 0:
return gen::just((OutputsSpec) OutputsSpec::All { }); return gen::just((OutputsSpec) OutputsSpec::All { });
default: case 1:
return gen::just((OutputsSpec) OutputsSpec::Names { return gen::just((OutputsSpec) OutputsSpec::Names {
*gen::nonEmpty(gen::container<StringSet>(gen::map( *gen::nonEmpty(gen::container<StringSet>(gen::map(
gen::arbitrary<StorePathName>(), gen::arbitrary<StorePathName>(),
[](StorePathName n) { return n.name; }))), [](StorePathName n) { return n.name; }))),
}); });
default:
assert(false);
} }
} }

View file

@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails
std::string_view description; std::string_view description;
}; };
constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{
{ {
.tag = Xp::CaDerivations, .tag = Xp::CaDerivations,
.name = "ca-derivations", .name = "ca-derivations",
@ -199,6 +199,16 @@ constexpr std::array<ExperimentalFeatureDetails, 12> xpFeatureDetails = {{
networking. networking.
)", )",
}, },
{
.tag = Xp::DynamicDerivations,
.name = "dynamic-derivations",
.description = R"(
Allow the use of a few things related to dynamic derivations:
- "text hashing" derivation outputs, so we can build .drv
files.
)",
},
}}; }};
static_assert( static_assert(

View file

@ -29,6 +29,7 @@ enum struct ExperimentalFeature
Cgroups, Cgroups,
DiscardReferences, DiscardReferences,
DaemonTrustOverride, DaemonTrustOverride,
DynamicDerivations,
}; };
/** /**

View file

@ -252,7 +252,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
throw Error("get-env.sh failed to produce an environment"); throw Error("get-env.sh failed to produce an environment");
} }
struct Common : InstallableValueCommand, MixProfile struct Common : InstallableCommand, MixProfile
{ {
std::set<std::string> ignoreVars{ std::set<std::string> ignoreVars{
"BASHOPTS", "BASHOPTS",
@ -374,7 +374,7 @@ struct Common : InstallableValueCommand, MixProfile
return res; return res;
} }
StorePath getShellOutPath(ref<Store> store, ref<InstallableValue> installable) StorePath getShellOutPath(ref<Store> store, ref<Installable> installable)
{ {
auto path = installable->getStorePath(); auto path = installable->getStorePath();
if (path && hasSuffix(path->to_string(), "-env")) if (path && hasSuffix(path->to_string(), "-env"))
@ -393,7 +393,7 @@ struct Common : InstallableValueCommand, MixProfile
} }
std::pair<BuildEnvironment, std::string> std::pair<BuildEnvironment, std::string>
getBuildEnvironment(ref<Store> store, ref<InstallableValue> installable) getBuildEnvironment(ref<Store> store, ref<Installable> installable)
{ {
auto shellOutPath = getShellOutPath(store, installable); auto shellOutPath = getShellOutPath(store, installable);
@ -481,7 +481,7 @@ struct CmdDevelop : Common, MixEnvironment
; ;
} }
void run(ref<Store> store, ref<InstallableValue> installable) override void run(ref<Store> store, ref<Installable> installable) override
{ {
auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable); auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable);
@ -538,10 +538,14 @@ struct CmdDevelop : Common, MixEnvironment
nixpkgsLockFlags.inputOverrides = {}; nixpkgsLockFlags.inputOverrides = {};
nixpkgsLockFlags.inputUpdates = {}; nixpkgsLockFlags.inputUpdates = {};
auto nixpkgs = defaultNixpkgsFlakeRef();
if (auto * i = dynamic_cast<const InstallableFlake *>(&*installable))
nixpkgs = i->nixpkgsFlakeRef();
auto bashInstallable = make_ref<InstallableFlake>( auto bashInstallable = make_ref<InstallableFlake>(
this, this,
state, state,
installable->nixpkgsFlakeRef(), std::move(nixpkgs),
"bashInteractive", "bashInteractive",
DefaultOutputs(), DefaultOutputs(),
Strings{}, Strings{},
@ -605,7 +609,7 @@ struct CmdPrintDevEnv : Common, MixJSON
Category category() override { return catUtility; } Category category() override { return catUtility; }
void run(ref<Store> store, ref<InstallableValue> installable) override void run(ref<Store> store, ref<Installable> installable) override
{ {
auto buildEnvironment = getBuildEnvironment(store, installable).first; auto buildEnvironment = getBuildEnvironment(store, installable).first;

View file

@ -32,3 +32,9 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh
src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh
src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh
src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md
@mkdir -p $$(dirname $@)
@cp $< $@
src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh

View file

@ -7,100 +7,39 @@ profile is a set of packages that can be installed and upgraded
independently from each other. Nix profiles are versioned, allowing independently from each other. Nix profiles are versioned, allowing
them to be rolled back easily. them to be rolled back easily.
# Default profile # Files
The default profile used by `nix profile` is `$HOME/.nix-profile`, )""
which, if it does not exist, is created as a symlink to
`/nix/var/nix/profiles/default` if Nix is invoked by the #include "doc/files/profiles.md.gen.hh"
`root` user, or `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile` otherwise.
R""(
You can specify another profile location using `--profile` *path*.
### Profile compatibility
# Filesystem layout
> **Warning**
Profiles are versioned as follows. When using profile *path*, *path* >
is a symlink to *path*`-`*N*, where *N* is the current *version* of > Once you have used [`nix profile`] you can no longer use [`nix-env`] without first deleting `$XDG_STATE_HOME/nix/profiles/profile`
the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix
store. For example: [`nix-env`]: @docroot@/command-ref/nix-env.md
[`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md
```console
$ ls -l ~alice/.local/state/nix/profiles/profile* Once you installed a package with [`nix profile`], you get the following error message when using [`nix-env`]:
lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link
lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile ```console
lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile $ nix-env -f '<nixpkgs>' -iA 'hello'
lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile error: nix-env
``` profile '/home/alice/.local/state/nix/profiles/profile' is incompatible with 'nix-env'; please use 'nix profile' instead
```
Each of these symlinks is a root for the Nix garbage collector.
To migrate back to `nix-env` you can delete your current profile:
The contents of the store path corresponding to each version of the
profile is a tree of symlinks to the files of the installed packages, > **Warning**
e.g. >
> This will delete packages that have been installed before, so you may want to back up this information before running the command.
```console
$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/ ```console
/home/eelco/.local/state/nix/profiles/profile-7-link/: $ rm -rf "${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile"
total 20 ```
dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin
-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json
dr-xr-xr-x 4 root root 4096 Jan 1 1970 share
/home/eelco/.local/state/nix/profiles/profile-7-link/bin:
total 20
lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications:
total 12
lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
```
The file `manifest.json` records the provenance of the packages that
are installed in this version of the profile. It looks like this:
```json
{
"version": 1,
"elements": [
{
"active": true,
"attrPath": "legacyPackages.x86_64-linux.zoom-us",
"originalUrl": "flake:nixpkgs",
"storePaths": [
"/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
],
"uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a"
},
]
}
```
Each object in the array `elements` denotes an installed package and
has the following fields:
* `originalUrl`: The [flake reference](./nix3-flake.md) specified by
the user at the time of installation (e.g. `nixpkgs`). This is also
the flake reference that will be used by `nix profile upgrade`.
* `uri`: The locked flake reference to which `originalUrl` resolved.
* `attrPath`: The flake output attribute that provided this
package. Note that this is not necessarily the attribute that the
user specified, but the one resulting from applying the default
attribute paths and prefixes; for instance, `hello` might resolve to
`packages.x86_64-linux.hello` and the empty string to
`packages.x86_64-linux.default`.
* `storePath`: The paths in the Nix store containing the package.
* `active`: Whether the profile contains symlinks to the files of this
package. If set to false, the package is kept in the Nix store, but
is not "visible" in the profile's symlink tree.
)"" )""

View file

@ -57,6 +57,30 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
(.outputs | keys == ["a_a", "b", "c"])) (.outputs | keys == ["a_a", "b", "c"]))
' '
# test buidling from non-drv attr path
nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a_a"]))
'
# Illegal type of string context
expectStderr 1 nix build -f multiple-outputs.nix 'e.a_a.drvPath' \
| grepQuiet "has a context which refers to a complete source and binary closure."
# No string context
expectStderr 1 nix build --expr '""' --no-link \
| grepQuiet "has 0 entries in its context. It should only have exactly one entry"
# Too much string context
expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e.a_a; "${drvPath}${outPath}"' --no-link \
| grepQuiet "has 2 entries in its context. It should only have exactly one entry"
nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status '
(.[0] | .path | match(".*multiple-outputs-e.drv"))
'
# Test building from raw store path to drv not expression. # Test building from raw store path to drv not expression.
drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath) drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath)

8
tests/dyn-drv/common.sh Normal file
View file

@ -0,0 +1,8 @@
source ../common.sh
# Need backend to support text-hashing too
requireDaemonNewerThan "2.16.0pre20230419"
enableFeatures "ca-derivations dynamic-derivations"
restartDaemon

1
tests/dyn-drv/config.nix.in Symbolic link
View file

@ -0,0 +1 @@
../config.nix.in

View file

@ -0,0 +1,33 @@
with import ./config.nix;
let innerName = "foo"; in
mkDerivation rec {
name = "${innerName}.drv";
SHELL = shell;
requiredSystemFeatures = [ "recursive-nix" ];
drv = builtins.unsafeDiscardOutputDependency (import ./text-hashed-output.nix).hello.drvPath;
buildCommand = ''
export NIX_CONFIG='experimental-features = nix-command ca-derivations'
PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH
# JSON of pre-existing drv
nix derivation show $drv | jq .[] > drv0.json
# Fix name
jq < drv0.json '.name = "${innerName}"' > drv1.json
# Extend `buildCommand`
jq < drv1.json '.env.buildCommand += "echo \"I am alive!\" >> $out/hello\n"' > drv0.json
# Used as our output
cp $(nix derivation add < drv0.json) $out
'';
__contentAddressed = true;
outputHashMode = "text";
outputHashAlgo = "sha256";
}

View file

@ -0,0 +1,25 @@
source common.sh
# FIXME
if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
enableFeatures 'recursive-nix'
restartDaemon
clearStore
rm -f $TEST_ROOT/result
EXTRA_PATH=$(dirname $(type -p nix)):$(dirname $(type -p jq))
export EXTRA_PATH
# Will produce a drv
metaDrv=$(nix-instantiate ./recursive-mod-json.nix)
# computed "dynamic" derivation
drv=$(nix-store -r $metaDrv)
# build that dyn drv
res=$(nix-store -r $drv)
grep 'I am alive!' $res/hello

View file

@ -0,0 +1,29 @@
with import ./config.nix;
# A simple content-addressed derivation.
# The derivation can be arbitrarily modified by passing a different `seed`,
# but the output will always be the same
rec {
hello = mkDerivation {
name = "hello";
buildCommand = ''
set -x
echo "Building a CA derivation"
mkdir -p $out
echo "Hello World" > $out/hello
'';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
producingDrv = mkDerivation {
name = "hello.drv";
buildCommand = ''
echo "Copying the derivation"
cp ${builtins.unsafeDiscardOutputDependency hello.drvPath} $out
'';
__contentAddressed = true;
outputHashMode = "text";
outputHashAlgo = "sha256";
};
}

View file

@ -0,0 +1,26 @@
#!/usr/bin/env bash
source common.sh
# In the corresponding nix file, we have two derivations: the first, named root,
# is a normal recursive derivation, while the second, named dependent, has the
# new outputHashMode "text". Note that in "dependent", we don't refer to the
# build output of root, but only to the path of the drv file. For this reason,
# we only need to:
#
# - instantiate the root derivation
# - build the dependent derivation
# - check that the path of the output coincides with that of the original derivation
drv=$(nix-instantiate ./text-hashed-output.nix -A hello)
nix show-derivation "$drv"
drvProducingDrv=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
nix show-derivation "$drvProducingDrv"
out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link)
nix path-info $drv --derivation --json | jq
nix path-info $out1 --derivation --json | jq
test $out1 == $drv

View file

@ -16,9 +16,10 @@ nix eval --expr 'assert 1 + 2 == 3; true'
[[ $(nix eval int -f "./eval.nix") == 123 ]] [[ $(nix eval int -f "./eval.nix") == 123 ]]
[[ $(nix eval str -f "./eval.nix") == '"foo"' ]] [[ $(nix eval str -f "./eval.nix") == '"foo"' ]]
[[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]] [[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]]
[[ $(nix eval attr -f "./eval.nix") == '{ foo = "bar"; }' ]] [[ "$(nix eval attr -f "./eval.nix")" == '{ foo = "bar"; }' ]]
[[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]]
[[ $(nix eval int -f - < "./eval.nix") == 123 ]] [[ $(nix eval int -f - < "./eval.nix") == 123 ]]
[[ "$(nix eval --expr '{"assert"=1;bar=2;}')" == '{ "assert" = 1; bar = 2; }' ]]
# Check if toFile can be utilized during restricted eval # Check if toFile can be utilized during restricted eval
[[ $(nix eval --restrict-eval --expr 'import (builtins.toFile "source" "42")') == 42 ]] [[ $(nix eval --restrict-eval --expr 'import (builtins.toFile "source" "42")') == 42 ]]
@ -26,9 +27,10 @@ nix eval --expr 'assert 1 + 2 == 3; true'
nix-instantiate --eval -E 'assert 1 + 2 == 3; true' nix-instantiate --eval -E 'assert 1 + 2 == 3; true'
[[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]]
[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]] [[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]]
[[ $(nix-instantiate -A attr --eval "./eval.nix") == '{ foo = "bar"; }' ]] [[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]]
[[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]]
[[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]]
[[ "$(nix-instantiate --eval -E '{"assert"=1;bar=2;}')" == '{ "assert" = 1; bar = 2; }' ]]
# Check that symlink cycles don't cause a hang. # Check that symlink cycles don't cause a hang.
ln -sfn cycle.nix $TEST_ROOT/cycle.nix ln -sfn cycle.nix $TEST_ROOT/cycle.nix

View file

@ -41,10 +41,27 @@ cat > $flake1Dir/flake.nix <<EOF
a8 = builtins.storePath $dep; a8 = builtins.storePath $dep;
a9 = "$dep"; a9 = "$dep";
drvCall = with import ./config.nix; mkDerivation {
name = "simple";
builder = ./simple.builder.sh;
PATH = "";
goodPath = path;
};
a10 = builtins.unsafeDiscardOutputDependency self.drvCall.drvPath;
a11 = self.drvCall.drvPath;
a12 = self.drvCall.outPath;
a13 = "\${self.drvCall.drvPath}\${self.drvCall.outPath}";
}; };
} }
EOF EOF
cp ../simple.nix ../simple.builder.sh ../config.nix $flake1Dir/
echo bar > $flake1Dir/foo echo bar > $flake1Dir/foo
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1
@ -63,4 +80,17 @@ nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6
nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8
diff common.sh $TEST_ROOT/result diff common.sh $TEST_ROOT/result
(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9) expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9 \
| grepQuiet "has 0 entries in its context. It should only have exactly one entry"
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a10
[[ $(readlink -e $TEST_ROOT/result) = *simple.drv ]]
expectStderr 1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a11 \
| grepQuiet "has a context which refers to a complete source and binary closure"
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12
[[ -e $TEST_ROOT/result/hello ]]
expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \
| grepQuiet "has 2 entries in its context. It should only have exactly one entry"

View file

@ -111,6 +111,8 @@ nix_tests = \
ca/derivation-json.sh \ ca/derivation-json.sh \
import-derivation.sh \ import-derivation.sh \
ca/import-derivation.sh \ ca/import-derivation.sh \
dyn-drv/text-hashed-output.sh \
dyn-drv/recursive-mod-json.sh \
nix_path.sh \ nix_path.sh \
case-hack.sh \ case-hack.sh \
placeholders.sh \ placeholders.sh \
@ -138,11 +140,19 @@ ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh nix_tests += compute-levels.sh
endif endif
install-tests += $(foreach x, $(nix_tests), tests/$(x)) install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
clean-files += $(d)/common/vars-and-functions.sh $(d)/config.nix $(d)/ca/config.nix clean-files += \
$(d)/common/vars-and-functions.sh \
$(d)/config.nix \
$(d)/ca/config.nix \
$(d)/dyn-drv/config.nix
test-deps += tests/common/vars-and-functions.sh tests/config.nix tests/ca/config.nix test-deps += \
tests/common/vars-and-functions.sh \
tests/config.nix \
tests/ca/config.nix \
tests/dyn-drv/config.nix
ifeq ($(BUILD_SHARED_LIBS), 1) ifeq ($(BUILD_SHARED_LIBS), 1)
test-deps += tests/plugins/libplugintest.$(SO_EXT) test-deps += tests/plugins/libplugintest.$(SO_EXT)

View file

@ -98,6 +98,18 @@ nix develop -f "$shellDotNix" shellDrv -c echo foo |& grepQuiet foo
nix print-dev-env -f "$shellDotNix" shellDrv > $TEST_ROOT/dev-env.sh nix print-dev-env -f "$shellDotNix" shellDrv > $TEST_ROOT/dev-env.sh
nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json
# Test with raw drv
shellDrv=$(nix-instantiate "$shellDotNix" -A shellDrv.out)
nix develop $shellDrv -c bash -c '[[ -n $stdenv ]]'
nix print-dev-env $shellDrv > $TEST_ROOT/dev-env2.sh
nix print-dev-env $shellDrv --json > $TEST_ROOT/dev-env2.json
diff $TEST_ROOT/dev-env{,2}.sh
diff $TEST_ROOT/dev-env{,2}.json
# Ensure `nix print-dev-env --json` contains variable assignments. # Ensure `nix print-dev-env --json` contains variable assignments.
[[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]] [[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]]

View file

@ -17,6 +17,10 @@ fi
# Build the dependencies and push them to the remote store. # Build the dependencies and push them to the remote store.
nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook "$pushToStore" nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook "$pushToStore"
# See if all outputs are passed to the post-build hook by only specifying one
# We're not able to test CA tests this way
export BUILD_HOOK_ONLY_OUT_PATHS=$([ ! $NIX_TESTS_CA_BY_DEFAULT ])
nix-build -o $TEST_ROOT/result-mult multiple-outputs.nix -A a.first --post-build-hook "$pushToStore"
clearStore clearStore
@ -24,3 +28,4 @@ clearStore
# closure of what we've just built. # closure of what we've just built.
nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix
nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv
nix copy --from "$REMOTE_STORE" --no-require-sigs -f multiple-outputs.nix a^second

View file

@ -7,4 +7,8 @@ set -e
[ -n "$DRV_PATH" ] [ -n "$DRV_PATH" ]
echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" echo Pushing "$OUT_PATHS" to "$REMOTE_STORE"
printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs if [ -n "$BUILD_HOOK_ONLY_OUT_PATHS" ]; then
printf "%s" "$OUT_PATHS" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs
else
printf "%s" "$DRV_PATH" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs
fi

View file

@ -7,4 +7,8 @@ set -e
[ -n "$DRV_PATH" ] [ -n "$DRV_PATH" ]
echo Pushing "$OUT_PATHS" to "$REMOTE_STORE" echo Pushing "$OUT_PATHS" to "$REMOTE_STORE"
printf "%s" "$DRV_PATH"^'*' | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs if [ -n "$BUILD_HOOK_ONLY_OUT_PATHS" ]; then
printf "%s" "$OUT_PATHS" | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs
else
printf "%s" "$DRV_PATH"^'*' | xargs nix copy --to "$REMOTE_STORE" --no-require-sigs
fi

View file

@ -1,11 +1,11 @@
source common.sh source common.sh
sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf
restartDaemon
# FIXME # FIXME
if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
enableFeatures 'recursive-nix'
restartDaemon
clearStore clearStore
rm -f $TEST_ROOT/result rm -f $TEST_ROOT/result