Allow selecting derivation outputs using 'installable!outputs'

E.g. 'nixpkgs#glibc^dev,static' or 'nixpkgs#glibc^*'.

doc/manual/src/release-notes/rl-next.md

@@ -14,3 +14,13 @@
 
 * `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
   This matches the default behaviour of `nix-build`.
+
+* You can now specify which outputs of a derivation `nix` should
+  operate on using the syntax `installable^outputs`,
+  e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
+  `nix` will use the outputs specified by the derivation's
+  `meta.outputsToInstall` attribute if it exists, or all outputs
+  otherwise.
+
+  Selecting derivation outputs using the attribute selection syntax
+  (e.g. `nixpkgs#glibc.dev`) no longer works.

src/libcmd/installables.cc

@@ -464,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
     SourceExprCommand & cmd;
     RootValue v;
     std::string attrPath;
+    OutputsSpec outputsSpec;
 
-    InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
+    InstallableAttrPath(
-        : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
+        ref<EvalState> state,
+        SourceExprCommand & cmd,
+        Value * v,
+        const std::string & attrPath,
+        OutputsSpec outputsSpec)
+        : InstallableValue(state)
+        , cmd(cmd)
+        , v(allocRootValue(v))
+        , attrPath(attrPath)
+        , outputsSpec(std::move(outputsSpec))
     { }
 
     std::string what() const override { return attrPath; }
@@ -495,9 +505,15 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
         auto drvPath = drvInfo.queryDrvPath();
         if (!drvPath)
             throw Error("'%s' is not a derivation", what());
+
         std::set<std::string> outputsToInstall;
-        for (auto & output : drvInfo.queryOutputs(false, true))
+
-            outputsToInstall.insert(output.first);
+        if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+            outputsToInstall = *outputNames;
+        else
+            for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
+                outputsToInstall.insert(output.first);
+
         res.push_back(DerivationInfo {
             .drvPath = *drvPath,
             .outputsToInstall = std::move(outputsToInstall)
@@ -578,6 +594,7 @@ InstallableFlake::InstallableFlake(
     ref<EvalState> state,
     FlakeRef && flakeRef,
     std::string_view fragment,
+    OutputsSpec outputsSpec,
     Strings attrPaths,
     Strings prefixes,
     const flake::LockFlags & lockFlags)
@@ -585,6 +602,7 @@ InstallableFlake::InstallableFlake(
       flakeRef(flakeRef),
       attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
       prefixes(fragment == "" ? Strings{} : prefixes),
+      outputsSpec(std::move(outputsSpec)),
       lockFlags(lockFlags)
 {
     if (cmd && cmd->getAutoArgs(*state)->size())
@@ -609,14 +627,19 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
             for (auto & s : aOutputsToInstall->getListOfStrings())
                 outputsToInstall.insert(s);
 
-    if (outputsToInstall.empty())
+    if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
+        outputsToInstall.clear();
         if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
             for (auto & s : aOutputs->getListOfStrings())
                 outputsToInstall.insert(s);
+    }
 
     if (outputsToInstall.empty())
         outputsToInstall.insert("out");
 
+    if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
+        outputsToInstall = *outputNames;
+
     auto drvInfo = DerivationInfo {
         .drvPath = std::move(drvPath),
         .outputsToInstall = std::move(outputsToInstall),
@@ -742,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             state->eval(e, *vFile);
         }
 
-        for (auto & s : ss)
+        for (auto & s : ss) {
-            result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
+            auto [prefix, outputsSpec] = parseOutputsSpec(s);
+            result.push_back(
+                std::make_shared<InstallableAttrPath>(
+                    state, *this, vFile,
+                    prefix == "." ? "" : prefix,
+                    outputsSpec));
+        }
 
     } else {
 
@@ -762,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             }
 
             try {
-                auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
+                auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
                         this,
                         getEvalState(),
                         std::move(flakeRef),
                         fragment,
+                        outputsSpec,
                         getDefaultFlakeAttrPaths(),
                         getDefaultFlakeAttrPathPrefixes(),
                         lockFlags));

src/libcmd/installables.hh

@@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
     FlakeRef flakeRef;
     Strings attrPaths;
     Strings prefixes;
+    OutputsSpec outputsSpec;
     const flake::LockFlags & lockFlags;
     mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
 
@@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
         ref<EvalState> state,
         FlakeRef && flakeRef,
         std::string_view fragment,
+        OutputsSpec outputsSpec,
         Strings attrPaths,
         Strings prefixes,
         const flake::LockFlags & lockFlags);

src/libexpr/flake/flakeref.cc

@@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
     return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
 }
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+    const std::string & url,
+    const std::optional<Path> & baseDir,
+    bool allowMissing,
+    bool isFlake)
+{
+    auto [prefix, outputsSpec] = parseOutputsSpec(url);
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
+    return {std::move(flakeRef), fragment, outputsSpec};
+}
+
 }

src/libexpr/flake/flakeref.hh

@@ -3,6 +3,7 @@
 #include "types.hh"
 #include "hash.hh"
 #include "fetchers.hh"
+#include "path-with-outputs.hh"
 
 #include <variant>
 
@@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
 std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
     const std::string & url, const std::optional<Path> & baseDir = {});
 
+std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
+    const std::string & url,
+    const std::optional<Path> & baseDir = {},
+    bool allowMissing = false,
+    bool isFlake = true);
+
+
 }

src/libstore/path-with-outputs.cc

@@ -1,6 +1,8 @@
 #include "path-with-outputs.hh"
 #include "store-api.hh"
 
+#include <regex>
+
 namespace nix {
 
 std::string StorePathWithOutputs::to_string(const Store & store) const
@@ -68,4 +70,18 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
     return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
 }
 
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
+{
+    static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
+
+    std::smatch match;
+    if (!std::regex_match(s, match, regex))
+        return {s, DefaultOutputs()};
+
+    if (match[3].matched)
+        return {match[1], AllOutputs()};
+
+    return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
+}
+
 }

src/libstore/path-with-outputs.hh

@@ -32,4 +32,16 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
 
 StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
 
+typedef std::set<std::string> OutputNames;
+
+struct AllOutputs { };
+
+struct DefaultOutputs { };
+
+typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
+
+/* Parse a string of the form 'prefix^output1,...outputN' or
+   'prefix^*', returning the prefix and the outputs spec. */
+std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
+
 }

src/libstore/tests/path-with-outputs.cc

@@ -0,0 +1,46 @@
+#include "path-with-outputs.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+TEST(parseOutputsSpec, basic)
+{
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
+        ASSERT_EQ(prefix, "foo");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
+        ASSERT_EQ(prefix, "foo^bar");
+        ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
+    }
+
+    {
+        auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
+        ASSERT_EQ(prefix, "foo^&*()");
+        ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
+    }
+}
+
+}

src/nix/bundle.cc

@@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
 
         auto val = installable->toValue(*evalState).first;
 
-        auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+        auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
         const flake::LockFlags lockFlags{ .writeLockFile = false };
         InstallableFlake bundler{this,
-            evalState, std::move(bundlerFlakeRef), bundlerName,
+            evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
             {"bundlers." + settings.thisSystem.get() + ".default",
              "defaultBundler." + settings.thisSystem.get()
             },

src/nix/develop.cc

@@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
                 state,
                 installable->nixpkgsFlakeRef(),
                 "bashInteractive",
+                DefaultOutputs(),
                 Strings{},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 nixpkgsLockFlags);

src/nix/flake.cc

@@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
         auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
 
         auto installable = InstallableFlake(nullptr,
-            evalState, std::move(templateFlakeRef), templateName,
+            evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
             defaultTemplateAttrPaths,
             defaultTemplateAttrPathsPrefixes,
             lockFlags);

src/nix/nix.md

@@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`,
 i.e. Nix will operate on the default flake output attribute of the
 flake in the current directory.
 
+## Derivation output selection
+
+Derivations can have multiple outputs, each corresponding to a
+different store path. For instance, a package can have a `bin` output
+that contains programs, and a `dev` output that provides development
+artifacts like C/C++ header files. The outputs on which `nix` commands
+operate are determined as follows:
+
+* You can explicitly specify the desired outputs using the syntax
+  *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can
+  obtain the `dev` and `static` outputs of the `glibc` package:
+
+  ```console
+  # nix build 'nixpkgs#glibc^dev,static'
+  # ls ./result-dev/include/ ./result-static/lib/
+  …
+  ```
+
+* You can also specify that *all* outputs should be used using the
+  syntax *installable*`^*`. For example, the following shows the size
+  of all outputs of the `glibc` package in the binary cache:
+
+  ```console
+  # nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
+  /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123                 33208200
+  /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin             36142896
+  /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug          155787312
+  /nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static          42488328
+  /nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev             44200560
+  ```
+
+* If you didn't specify the desired outputs, but the derivation has an
+  attribute `meta.outputsToInstall`, Nix will use those outputs. For
+  example, since the package `nixpkgs#libxml2` has this attribute:
+
+  ```console
+  # nix eval 'nixpkgs#libxml2.meta.outputsToInstall'
+  [ "bin" "man" ]
+  ```
+
+  a command like `nix shell nixpkgs#libxml2` will provide only those
+  two outputs by default.
+
+* Otherwise, Nix will use all outputs of the derivation.
+
 # Nix stores
 
 Most `nix` subcommands operate on a *Nix store*.

src/nix/profile.cc

@@ -443,6 +443,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                     getEvalState(),
                     FlakeRef(element.source->originalRef),
                     "",
+                    DefaultOutputs(), // FIXME
                     Strings{element.source->attrPath},
                     Strings{},
                     lockFlags);

tests/build.sh

@@ -2,6 +2,8 @@ source common.sh
 
 clearStore
 
+set -o pipefail
+
 # Make sure that 'nix build' returns all outputs by default.
 nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
   (.[0] |
@@ -15,6 +17,45 @@ nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
     (.outputs.out | match(".*multiple-outputs-b")))
 '
 
+# Test output selection using the '^' syntax.
+nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys == ["first"]))
+'
+
+nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys == ["first", "second"]))
+'
+
+nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-a.drv")) and
+    (.outputs | keys == ["first", "second"]))
+'
+
+# Test that 'outputsToInstall' is respected by default.
+nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-e.drv")) and
+    (.outputs | keys == ["a", "b"]))
+'
+
+# But not when it's overriden.
+nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-e.drv")) and
+    (.outputs | keys == ["a"]))
+'
+
+nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
+  (.[0] |
+    (.drvPath | match(".*multiple-outputs-e.drv")) and
+    (.outputs | keys == ["a", "b", "c"]))
+'
+
 testNormalization () {
     clearStore
     outPath=$(nix-build ./simple.nix --no-out-link)

tests/multiple-outputs.nix

@@ -80,4 +80,11 @@ rec {
       '';
   }).a;
 
+  e = mkDerivation {
+    name = "multiple-outputs-e";
+    outputs = [ "a" "b" "c" ];
+    meta.outputsToInstall = [ "a" "b" ];
+    buildCommand = "mkdir $a $b $c";
+  };
+
 }

tests/shell-hello.nix

@@ -3,15 +3,24 @@ with import ./config.nix;
 {
   hello = mkDerivation {
     name = "hello";
+    outputs = [ "out" "dev" ];
+    meta.outputsToInstall = [ "out" ];
     buildCommand =
       ''
-        mkdir -p $out/bin
+        mkdir -p $out/bin $dev/bin
+
         cat > $out/bin/hello <<EOF
         #! ${shell}
         who=\$1
         echo "Hello \''${who:-World} from $out/bin/hello"
         EOF
         chmod +x $out/bin/hello
+
+        cat > $dev/bin/hello2 <<EOF
+        #! ${shell}
+        echo "Hello2"
+        EOF
+        chmod +x $dev/bin/hello2
       '';
   };
 }

tests/shell.sh

@@ -6,6 +6,10 @@ clearCache
 nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
 nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
 
+# Test output selection.
+nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
+nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
+
 if ! canUseSandbox; then exit 99; fi
 
 chmod -R u+w $TEST_ROOT/store0 || true