From 4a79cba5118f29b896f3d50164beacd4901ab01f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Apr 2022 15:17:01 +0200 Subject: [PATCH] Allow selecting derivation outputs using 'installable!outputs' E.g. 'nixpkgs#glibc^dev,static' or 'nixpkgs#glibc^*'. --- doc/manual/src/release-notes/rl-next.md | 10 ++++++ src/libcmd/installables.cc | 46 ++++++++++++++++++++----- src/libcmd/installables.hh | 2 ++ src/libexpr/flake/flakeref.cc | 11 ++++++ src/libexpr/flake/flakeref.hh | 8 +++++ src/libstore/path-with-outputs.cc | 16 +++++++++ src/libstore/path-with-outputs.hh | 12 +++++++ src/libstore/tests/path-with-outputs.cc | 46 +++++++++++++++++++++++++ src/nix/bundle.cc | 4 +-- src/nix/develop.cc | 1 + src/nix/flake.cc | 2 +- src/nix/nix.md | 45 ++++++++++++++++++++++++ src/nix/profile.cc | 1 + tests/build.sh | 41 ++++++++++++++++++++++ tests/multiple-outputs.nix | 7 ++++ tests/shell-hello.nix | 11 +++++- tests/shell.sh | 4 +++ 17 files changed, 255 insertions(+), 12 deletions(-) create mode 100644 src/libstore/tests/path-with-outputs.cc diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7b3ad4e58..efd893662 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -14,3 +14,13 @@ * `nix build` has a new `--print-out-paths` flag to print the resulting output paths. This matches the default behaviour of `nix-build`. + +* You can now specify which outputs of a derivation `nix` should + operate on using the syntax `installable^outputs`, + e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default, + `nix` will use the outputs specified by the derivation's + `meta.outputsToInstall` attribute if it exists, or all outputs + otherwise. + + Selecting derivation outputs using the attribute selection syntax + (e.g. `nixpkgs#glibc.dev`) no longer works. diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index e2ee47dea..55a5e91e9 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -464,9 +464,19 @@ struct InstallableAttrPath : InstallableValue SourceExprCommand & cmd; RootValue v; std::string attrPath; + OutputsSpec outputsSpec; - InstallableAttrPath(ref state, SourceExprCommand & cmd, Value * v, const std::string & attrPath) - : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) + InstallableAttrPath( + ref state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + OutputsSpec outputsSpec) + : InstallableValue(state) + , cmd(cmd) + , v(allocRootValue(v)) + , attrPath(attrPath) + , outputsSpec(std::move(outputsSpec)) { } std::string what() const override { return attrPath; } @@ -495,9 +505,15 @@ std::vector InstallableAttrPath::toDerivations auto drvPath = drvInfo.queryDrvPath(); if (!drvPath) throw Error("'%s' is not a derivation", what()); + std::set outputsToInstall; - for (auto & output : drvInfo.queryOutputs(false, true)) - outputsToInstall.insert(output.first); + + if (auto outputNames = std::get_if(&outputsSpec)) + outputsToInstall = *outputNames; + else + for (auto & output : drvInfo.queryOutputs(false, std::get_if(&outputsSpec))) + outputsToInstall.insert(output.first); + res.push_back(DerivationInfo { .drvPath = *drvPath, .outputsToInstall = std::move(outputsToInstall) @@ -578,6 +594,7 @@ InstallableFlake::InstallableFlake( ref state, FlakeRef && flakeRef, std::string_view fragment, + OutputsSpec outputsSpec, Strings attrPaths, Strings prefixes, const flake::LockFlags & lockFlags) @@ -585,6 +602,7 @@ InstallableFlake::InstallableFlake( flakeRef(flakeRef), attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), prefixes(fragment == "" ? Strings{} : prefixes), + outputsSpec(std::move(outputsSpec)), lockFlags(lockFlags) { if (cmd && cmd->getAutoArgs(*state)->size()) @@ -609,14 +627,19 @@ std::tuple InstallableF for (auto & s : aOutputsToInstall->getListOfStrings()) outputsToInstall.insert(s); - if (outputsToInstall.empty()) + if (outputsToInstall.empty() || std::get_if(&outputsSpec)) { + outputsToInstall.clear(); if (auto aOutputs = attr->maybeGetAttr(state->sOutputs)) for (auto & s : aOutputs->getListOfStrings()) outputsToInstall.insert(s); + } if (outputsToInstall.empty()) outputsToInstall.insert("out"); + if (auto outputNames = std::get_if(&outputsSpec)) + outputsToInstall = *outputNames; + auto drvInfo = DerivationInfo { .drvPath = std::move(drvPath), .outputsToInstall = std::move(outputsToInstall), @@ -742,8 +765,14 @@ std::vector> SourceExprCommand::parseInstallables( state->eval(e, *vFile); } - for (auto & s : ss) - result.push_back(std::make_shared(state, *this, vFile, s == "." ? "" : s)); + for (auto & s : ss) { + auto [prefix, outputsSpec] = parseOutputsSpec(s); + result.push_back( + std::make_shared( + state, *this, vFile, + prefix == "." ? "" : prefix, + outputsSpec)); + } } else { @@ -762,12 +791,13 @@ std::vector> SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); + auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath(".")); result.push_back(std::make_shared( this, getEvalState(), std::move(flakeRef), fragment, + outputsSpec, getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 3c2c33549..1a5a96153 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue FlakeRef flakeRef; Strings attrPaths; Strings prefixes; + OutputsSpec outputsSpec; const flake::LockFlags & lockFlags; mutable std::shared_ptr _lockedFlake; @@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue ref state, FlakeRef && flakeRef, std::string_view fragment, + OutputsSpec outputsSpec, Strings attrPaths, Strings prefixes, const flake::LockFlags & lockFlags); diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index c1eae413f..1dcc4555a 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -238,4 +238,15 @@ std::pair FlakeRef::fetchTree(ref store) const return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } +std::tuple parseFlakeRefWithFragmentAndOutputsSpec( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + auto [prefix, outputsSpec] = parseOutputsSpec(url); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake); + return {std::move(flakeRef), fragment, outputsSpec}; +} + } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 1fddfd9a0..a9182f4bf 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "hash.hh" #include "fetchers.hh" +#include "path-with-outputs.hh" #include @@ -79,4 +80,11 @@ std::pair parseFlakeRefWithFragment( std::optional> maybeParseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir = {}); +std::tuple parseFlakeRefWithFragmentAndOutputsSpec( + const std::string & url, + const std::optional & baseDir = {}, + bool allowMissing = false, + bool isFlake = true); + + } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 078c117bd..7d180a0f6 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,6 +1,8 @@ #include "path-with-outputs.hh" #include "store-api.hh" +#include + namespace nix { std::string StorePathWithOutputs::to_string(const Store & store) const @@ -68,4 +70,18 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } +std::pair parseOutputsSpec(const std::string & s) +{ + static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); + + std::smatch match; + if (!std::regex_match(s, match, regex)) + return {s, DefaultOutputs()}; + + if (match[3].matched) + return {match[1], AllOutputs()}; + + return {match[1], tokenizeString(match[4].str(), ",")}; +} + } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 4c4023dcb..e4235d197 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -32,4 +32,16 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +typedef std::set OutputNames; + +struct AllOutputs { }; + +struct DefaultOutputs { }; + +typedef std::variant OutputsSpec; + +/* Parse a string of the form 'prefix^output1,...outputN' or + 'prefix^*', returning the prefix and the outputs spec. */ +std::pair parseOutputsSpec(const std::string & s); + } diff --git a/src/libstore/tests/path-with-outputs.cc b/src/libstore/tests/path-with-outputs.cc new file mode 100644 index 000000000..350ea7ffd --- /dev/null +++ b/src/libstore/tests/path-with-outputs.cc @@ -0,0 +1,46 @@ +#include "path-with-outputs.hh" + +#include + +namespace nix { + +TEST(parseOutputsSpec, basic) +{ + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get_if(&outputsSpec)); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^*"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get_if(&outputsSpec)); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^out"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out"})); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out", "bin"})); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin"); + ASSERT_EQ(prefix, "foo^bar"); + ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out", "bin"})); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()"); + ASSERT_EQ(prefix, "foo^&*()"); + ASSERT_TRUE(std::get_if(&outputsSpec)); + } +} + +} diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2421adf4e..2e48e4c74 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, + evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec, {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 7fc74d34e..1190b8348 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment state, installable->nixpkgsFlakeRef(), "bashInteractive", + DefaultOutputs(), Strings{}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6a34ca67b..1938ce4e6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, - evalState, std::move(templateFlakeRef), templateName, + evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(), defaultTemplateAttrPaths, defaultTemplateAttrPathsPrefixes, lockFlags); diff --git a/src/nix/nix.md b/src/nix/nix.md index 0dacadee6..d48682a94 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`, i.e. Nix will operate on the default flake output attribute of the flake in the current directory. +## Derivation output selection + +Derivations can have multiple outputs, each corresponding to a +different store path. For instance, a package can have a `bin` output +that contains programs, and a `dev` output that provides development +artifacts like C/C++ header files. The outputs on which `nix` commands +operate are determined as follows: + +* You can explicitly specify the desired outputs using the syntax + *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can + obtain the `dev` and `static` outputs of the `glibc` package: + + ```console + # nix build 'nixpkgs#glibc^dev,static' + # ls ./result-dev/include/ ./result-static/lib/ + … + ``` + +* You can also specify that *all* outputs should be used using the + syntax *installable*`^*`. For example, the following shows the size + of all outputs of the `glibc` package in the binary cache: + + ```console + # nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*' + /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200 + /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896 + /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312 + /nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static 42488328 + /nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560 + ``` + +* If you didn't specify the desired outputs, but the derivation has an + attribute `meta.outputsToInstall`, Nix will use those outputs. For + example, since the package `nixpkgs#libxml2` has this attribute: + + ```console + # nix eval 'nixpkgs#libxml2.meta.outputsToInstall' + [ "bin" "man" ] + ``` + + a command like `nix shell nixpkgs#libxml2` will provide only those + two outputs by default. + +* Otherwise, Nix will use all outputs of the derivation. + # Nix stores Most `nix` subcommands operate on a *Nix store*. diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 52c918016..78c8af80c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -443,6 +443,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf getEvalState(), FlakeRef(element.source->originalRef), "", + DefaultOutputs(), // FIXME Strings{element.source->attrPath}, Strings{}, lockFlags); diff --git a/tests/build.sh b/tests/build.sh index 339155991..ff16d1603 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -2,6 +2,8 @@ source common.sh clearStore +set -o pipefail + # Make sure that 'nix build' returns all outputs by default. nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' (.[0] | @@ -15,6 +17,45 @@ nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' (.outputs.out | match(".*multiple-outputs-b"))) ' +# Test output selection using the '^' syntax. +nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first"])) +' + +nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first", "second"])) +' + +nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys == ["first", "second"])) +' + +# Test that 'outputsToInstall' is respected by default. +nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a", "b"])) +' + +# But not when it's overriden. +nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a"])) +' + +nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a", "b", "c"])) +' + testNormalization () { clearStore outPath=$(nix-build ./simple.nix --no-out-link) diff --git a/tests/multiple-outputs.nix b/tests/multiple-outputs.nix index b915493f7..624a5dade 100644 --- a/tests/multiple-outputs.nix +++ b/tests/multiple-outputs.nix @@ -80,4 +80,11 @@ rec { ''; }).a; + e = mkDerivation { + name = "multiple-outputs-e"; + outputs = [ "a" "b" "c" ]; + meta.outputsToInstall = [ "a" "b" ]; + buildCommand = "mkdir $a $b $c"; + }; + } diff --git a/tests/shell-hello.nix b/tests/shell-hello.nix index 77dcbd2a9..3fdd3501d 100644 --- a/tests/shell-hello.nix +++ b/tests/shell-hello.nix @@ -3,15 +3,24 @@ with import ./config.nix; { hello = mkDerivation { name = "hello"; + outputs = [ "out" "dev" ]; + meta.outputsToInstall = [ "out" ]; buildCommand = '' - mkdir -p $out/bin + mkdir -p $out/bin $dev/bin + cat > $out/bin/hello < $dev/bin/hello2 <