Allow selecting derivation outputs using 'installable!outputs'

E.g. 'nixpkgs#glibc^dev,static' or 'nixpkgs#glibc^*'.
This commit is contained in:
Eelco Dolstra 2022-04-22 15:17:01 +02:00
parent 404c222444
commit 4a79cba511
17 changed files with 255 additions and 12 deletions

View file

@ -14,3 +14,13 @@
* `nix build` has a new `--print-out-paths` flag to print the resulting output paths. * `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
This matches the default behaviour of `nix-build`. This matches the default behaviour of `nix-build`.
* You can now specify which outputs of a derivation `nix` should
operate on using the syntax `installable^outputs`,
e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
`nix` will use the outputs specified by the derivation's
`meta.outputsToInstall` attribute if it exists, or all outputs
otherwise.
Selecting derivation outputs using the attribute selection syntax
(e.g. `nixpkgs#glibc.dev`) no longer works.

View file

@ -464,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
SourceExprCommand & cmd; SourceExprCommand & cmd;
RootValue v; RootValue v;
std::string attrPath; std::string attrPath;
OutputsSpec outputsSpec;
InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath) InstallableAttrPath(
: InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath) ref<EvalState> state,
SourceExprCommand & cmd,
Value * v,
const std::string & attrPath,
OutputsSpec outputsSpec)
: InstallableValue(state)
, cmd(cmd)
, v(allocRootValue(v))
, attrPath(attrPath)
, outputsSpec(std::move(outputsSpec))
{ } { }
std::string what() const override { return attrPath; } std::string what() const override { return attrPath; }
@ -495,9 +505,15 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
auto drvPath = drvInfo.queryDrvPath(); auto drvPath = drvInfo.queryDrvPath();
if (!drvPath) if (!drvPath)
throw Error("'%s' is not a derivation", what()); throw Error("'%s' is not a derivation", what());
std::set<std::string> outputsToInstall; std::set<std::string> outputsToInstall;
for (auto & output : drvInfo.queryOutputs(false, true))
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first); outputsToInstall.insert(output.first);
res.push_back(DerivationInfo { res.push_back(DerivationInfo {
.drvPath = *drvPath, .drvPath = *drvPath,
.outputsToInstall = std::move(outputsToInstall) .outputsToInstall = std::move(outputsToInstall)
@ -578,6 +594,7 @@ InstallableFlake::InstallableFlake(
ref<EvalState> state, ref<EvalState> state,
FlakeRef && flakeRef, FlakeRef && flakeRef,
std::string_view fragment, std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths, Strings attrPaths,
Strings prefixes, Strings prefixes,
const flake::LockFlags & lockFlags) const flake::LockFlags & lockFlags)
@ -585,6 +602,7 @@ InstallableFlake::InstallableFlake(
flakeRef(flakeRef), flakeRef(flakeRef),
attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
prefixes(fragment == "" ? Strings{} : prefixes), prefixes(fragment == "" ? Strings{} : prefixes),
outputsSpec(std::move(outputsSpec)),
lockFlags(lockFlags) lockFlags(lockFlags)
{ {
if (cmd && cmd->getAutoArgs(*state)->size()) if (cmd && cmd->getAutoArgs(*state)->size())
@ -609,14 +627,19 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
for (auto & s : aOutputsToInstall->getListOfStrings()) for (auto & s : aOutputsToInstall->getListOfStrings())
outputsToInstall.insert(s); outputsToInstall.insert(s);
if (outputsToInstall.empty()) if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
outputsToInstall.clear();
if (auto aOutputs = attr->maybeGetAttr(state->sOutputs)) if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
for (auto & s : aOutputs->getListOfStrings()) for (auto & s : aOutputs->getListOfStrings())
outputsToInstall.insert(s); outputsToInstall.insert(s);
}
if (outputsToInstall.empty()) if (outputsToInstall.empty())
outputsToInstall.insert("out"); outputsToInstall.insert("out");
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
auto drvInfo = DerivationInfo { auto drvInfo = DerivationInfo {
.drvPath = std::move(drvPath), .drvPath = std::move(drvPath),
.outputsToInstall = std::move(outputsToInstall), .outputsToInstall = std::move(outputsToInstall),
@ -742,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
state->eval(e, *vFile); state->eval(e, *vFile);
} }
for (auto & s : ss) for (auto & s : ss) {
result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s)); auto [prefix, outputsSpec] = parseOutputsSpec(s);
result.push_back(
std::make_shared<InstallableAttrPath>(
state, *this, vFile,
prefix == "." ? "" : prefix,
outputsSpec));
}
} else { } else {
@ -762,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
} }
try { try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath(".")); auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>( result.push_back(std::make_shared<InstallableFlake>(
this, this,
getEvalState(), getEvalState(),
std::move(flakeRef), std::move(flakeRef),
fragment, fragment,
outputsSpec,
getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPaths(),
getDefaultFlakeAttrPathPrefixes(), getDefaultFlakeAttrPathPrefixes(),
lockFlags)); lockFlags));

View file

@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
FlakeRef flakeRef; FlakeRef flakeRef;
Strings attrPaths; Strings attrPaths;
Strings prefixes; Strings prefixes;
OutputsSpec outputsSpec;
const flake::LockFlags & lockFlags; const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
ref<EvalState> state, ref<EvalState> state,
FlakeRef && flakeRef, FlakeRef && flakeRef,
std::string_view fragment, std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths, Strings attrPaths,
Strings prefixes, Strings prefixes,
const flake::LockFlags & lockFlags); const flake::LockFlags & lockFlags);

View file

@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
} }
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
{
auto [prefix, outputsSpec] = parseOutputsSpec(url);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
return {std::move(flakeRef), fragment, outputsSpec};
}
} }

View file

@ -3,6 +3,7 @@
#include "types.hh" #include "types.hh"
#include "hash.hh" #include "hash.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "path-with-outputs.hh"
#include <variant> #include <variant>
@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {}); const std::string & url, const std::optional<Path> & baseDir = {});
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
} }

View file

@ -1,6 +1,8 @@
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "store-api.hh" #include "store-api.hh"
#include <regex>
namespace nix { namespace nix {
std::string StorePathWithOutputs::to_string(const Store & store) const std::string StorePathWithOutputs::to_string(const Store & store) const
@ -68,4 +70,18 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
} }
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
{
static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
std::smatch match;
if (!std::regex_match(s, match, regex))
return {s, DefaultOutputs()};
if (match[3].matched)
return {match[1], AllOutputs()};
return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
}
} }

View file

@ -32,4 +32,16 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
typedef std::set<std::string> OutputNames;
struct AllOutputs { };
struct DefaultOutputs { };
typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the outputs spec. */
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
} }

View file

@ -0,0 +1,46 @@
#include "path-with-outputs.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(parseOutputsSpec, basic)
{
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
ASSERT_EQ(prefix, "foo^&*()");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
}
}

View file

@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
auto val = installable->toValue(*evalState).first; auto val = installable->toValue(*evalState).first;
auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false }; const flake::LockFlags lockFlags{ .writeLockFile = false };
InstallableFlake bundler{this, InstallableFlake bundler{this,
evalState, std::move(bundlerFlakeRef), bundlerName, evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
{"bundlers." + settings.thisSystem.get() + ".default", {"bundlers." + settings.thisSystem.get() + ".default",
"defaultBundler." + settings.thisSystem.get() "defaultBundler." + settings.thisSystem.get()
}, },

View file

@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
state, state,
installable->nixpkgsFlakeRef(), installable->nixpkgsFlakeRef(),
"bashInteractive", "bashInteractive",
DefaultOutputs(),
Strings{}, Strings{},
Strings{"legacyPackages." + settings.thisSystem.get() + "."}, Strings{"legacyPackages." + settings.thisSystem.get() + "."},
nixpkgsLockFlags); nixpkgsLockFlags);

View file

@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
auto installable = InstallableFlake(nullptr, auto installable = InstallableFlake(nullptr,
evalState, std::move(templateFlakeRef), templateName, evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
defaultTemplateAttrPaths, defaultTemplateAttrPaths,
defaultTemplateAttrPathsPrefixes, defaultTemplateAttrPathsPrefixes,
lockFlags); lockFlags);

View file

@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`,
i.e. Nix will operate on the default flake output attribute of the i.e. Nix will operate on the default flake output attribute of the
flake in the current directory. flake in the current directory.
## Derivation output selection
Derivations can have multiple outputs, each corresponding to a
different store path. For instance, a package can have a `bin` output
that contains programs, and a `dev` output that provides development
artifacts like C/C++ header files. The outputs on which `nix` commands
operate are determined as follows:
* You can explicitly specify the desired outputs using the syntax
*installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can
obtain the `dev` and `static` outputs of the `glibc` package:
```console
# nix build 'nixpkgs#glibc^dev,static'
# ls ./result-dev/include/ ./result-static/lib/
```
* You can also specify that *all* outputs should be used using the
syntax *installable*`^*`. For example, the following shows the size
of all outputs of the `glibc` package in the binary cache:
```console
# nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
/nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200
/nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896
/nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312
/nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static 42488328
/nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560
```
* If you didn't specify the desired outputs, but the derivation has an
attribute `meta.outputsToInstall`, Nix will use those outputs. For
example, since the package `nixpkgs#libxml2` has this attribute:
```console
# nix eval 'nixpkgs#libxml2.meta.outputsToInstall'
[ "bin" "man" ]
```
a command like `nix shell nixpkgs#libxml2` will provide only those
two outputs by default.
* Otherwise, Nix will use all outputs of the derivation.
# Nix stores # Nix stores
Most `nix` subcommands operate on a *Nix store*. Most `nix` subcommands operate on a *Nix store*.

View file

@ -443,6 +443,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
getEvalState(), getEvalState(),
FlakeRef(element.source->originalRef), FlakeRef(element.source->originalRef),
"", "",
DefaultOutputs(), // FIXME
Strings{element.source->attrPath}, Strings{element.source->attrPath},
Strings{}, Strings{},
lockFlags); lockFlags);

View file

@ -2,6 +2,8 @@ source common.sh
clearStore clearStore
set -o pipefail
# Make sure that 'nix build' returns all outputs by default. # Make sure that 'nix build' returns all outputs by default.
nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
(.[0] | (.[0] |
@ -15,6 +17,45 @@ nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
(.outputs.out | match(".*multiple-outputs-b"))) (.outputs.out | match(".*multiple-outputs-b")))
' '
# Test output selection using the '^' syntax.
nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first"]))
'
nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
# Test that 'outputsToInstall' is respected by default.
nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b"]))
'
# But not when it's overriden.
nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a"]))
'
nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b", "c"]))
'
testNormalization () { testNormalization () {
clearStore clearStore
outPath=$(nix-build ./simple.nix --no-out-link) outPath=$(nix-build ./simple.nix --no-out-link)

View file

@ -80,4 +80,11 @@ rec {
''; '';
}).a; }).a;
e = mkDerivation {
name = "multiple-outputs-e";
outputs = [ "a" "b" "c" ];
meta.outputsToInstall = [ "a" "b" ];
buildCommand = "mkdir $a $b $c";
};
} }

View file

@ -3,15 +3,24 @@ with import ./config.nix;
{ {
hello = mkDerivation { hello = mkDerivation {
name = "hello"; name = "hello";
outputs = [ "out" "dev" ];
meta.outputsToInstall = [ "out" ];
buildCommand = buildCommand =
'' ''
mkdir -p $out/bin mkdir -p $out/bin $dev/bin
cat > $out/bin/hello <<EOF cat > $out/bin/hello <<EOF
#! ${shell} #! ${shell}
who=\$1 who=\$1
echo "Hello \''${who:-World} from $out/bin/hello" echo "Hello \''${who:-World} from $out/bin/hello"
EOF EOF
chmod +x $out/bin/hello chmod +x $out/bin/hello
cat > $dev/bin/hello2 <<EOF
#! ${shell}
echo "Hello2"
EOF
chmod +x $dev/bin/hello2
''; '';
}; };
} }

View file

@ -6,6 +6,10 @@ clearCache
nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
# Test output selection.
nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
if ! canUseSandbox; then exit 99; fi if ! canUseSandbox; then exit 99; fi
chmod -R u+w $TEST_ROOT/store0 || true chmod -R u+w $TEST_ROOT/store0 || true