Allow selecting derivation outputs using 'installable!outputs'

E.g. 'nixpkgs#glibc^dev,static' or 'nixpkgs#glibc^*'.
This commit is contained in:
Eelco Dolstra 2022-04-22 15:17:01 +02:00
parent 404c222444
commit 4a79cba511
17 changed files with 255 additions and 12 deletions

View file

@ -14,3 +14,13 @@
* `nix build` has a new `--print-out-paths` flag to print the resulting output paths.
This matches the default behaviour of `nix-build`.
* You can now specify which outputs of a derivation `nix` should
operate on using the syntax `installable^outputs`,
e.g. `nixpkgs#glibc^dev,static` or `nixpkgs#glibc^*`. By default,
`nix` will use the outputs specified by the derivation's
`meta.outputsToInstall` attribute if it exists, or all outputs
otherwise.
Selecting derivation outputs using the attribute selection syntax
(e.g. `nixpkgs#glibc.dev`) no longer works.

View file

@ -464,9 +464,19 @@ struct InstallableAttrPath : InstallableValue
SourceExprCommand & cmd;
RootValue v;
std::string attrPath;
OutputsSpec outputsSpec;
InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
: InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
InstallableAttrPath(
ref<EvalState> state,
SourceExprCommand & cmd,
Value * v,
const std::string & attrPath,
OutputsSpec outputsSpec)
: InstallableValue(state)
, cmd(cmd)
, v(allocRootValue(v))
, attrPath(attrPath)
, outputsSpec(std::move(outputsSpec))
{ }
std::string what() const override { return attrPath; }
@ -495,9 +505,15 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
auto drvPath = drvInfo.queryDrvPath();
if (!drvPath)
throw Error("'%s' is not a derivation", what());
std::set<std::string> outputsToInstall;
for (auto & output : drvInfo.queryOutputs(false, true))
outputsToInstall.insert(output.first);
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
else
for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec)))
outputsToInstall.insert(output.first);
res.push_back(DerivationInfo {
.drvPath = *drvPath,
.outputsToInstall = std::move(outputsToInstall)
@ -578,6 +594,7 @@ InstallableFlake::InstallableFlake(
ref<EvalState> state,
FlakeRef && flakeRef,
std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths,
Strings prefixes,
const flake::LockFlags & lockFlags)
@ -585,6 +602,7 @@ InstallableFlake::InstallableFlake(
flakeRef(flakeRef),
attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}),
prefixes(fragment == "" ? Strings{} : prefixes),
outputsSpec(std::move(outputsSpec)),
lockFlags(lockFlags)
{
if (cmd && cmd->getAutoArgs(*state)->size())
@ -609,14 +627,19 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
for (auto & s : aOutputsToInstall->getListOfStrings())
outputsToInstall.insert(s);
if (outputsToInstall.empty())
if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
outputsToInstall.clear();
if (auto aOutputs = attr->maybeGetAttr(state->sOutputs))
for (auto & s : aOutputs->getListOfStrings())
outputsToInstall.insert(s);
}
if (outputsToInstall.empty())
outputsToInstall.insert("out");
if (auto outputNames = std::get_if<OutputNames>(&outputsSpec))
outputsToInstall = *outputNames;
auto drvInfo = DerivationInfo {
.drvPath = std::move(drvPath),
.outputsToInstall = std::move(outputsToInstall),
@ -742,8 +765,14 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
state->eval(e, *vFile);
}
for (auto & s : ss)
result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
for (auto & s : ss) {
auto [prefix, outputsSpec] = parseOutputsSpec(s);
result.push_back(
std::make_shared<InstallableAttrPath>(
state, *this, vFile,
prefix == "." ? "" : prefix,
outputsSpec));
}
} else {
@ -762,12 +791,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
}
try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>(
this,
getEvalState(),
std::move(flakeRef),
fragment,
outputsSpec,
getDefaultFlakeAttrPaths(),
getDefaultFlakeAttrPathPrefixes(),
lockFlags));

View file

@ -156,6 +156,7 @@ struct InstallableFlake : InstallableValue
FlakeRef flakeRef;
Strings attrPaths;
Strings prefixes;
OutputsSpec outputsSpec;
const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
@ -164,6 +165,7 @@ struct InstallableFlake : InstallableValue
ref<EvalState> state,
FlakeRef && flakeRef,
std::string_view fragment,
OutputsSpec outputsSpec,
Strings attrPaths,
Strings prefixes,
const flake::LockFlags & lockFlags);

View file

@ -238,4 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
}
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir,
bool allowMissing,
bool isFlake)
{
auto [prefix, outputsSpec] = parseOutputsSpec(url);
auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake);
return {std::move(flakeRef), fragment, outputsSpec};
}
}

View file

@ -3,6 +3,7 @@
#include "types.hh"
#include "hash.hh"
#include "fetchers.hh"
#include "path-with-outputs.hh"
#include <variant>
@ -79,4 +80,11 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
const std::string & url, const std::optional<Path> & baseDir = {});
std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec(
const std::string & url,
const std::optional<Path> & baseDir = {},
bool allowMissing = false,
bool isFlake = true);
}

View file

@ -1,6 +1,8 @@
#include "path-with-outputs.hh"
#include "store-api.hh"
#include <regex>
namespace nix {
std::string StorePathWithOutputs::to_string(const Store & store) const
@ -68,4 +70,18 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std:
return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
}
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s)
{
static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))");
std::smatch match;
if (!std::regex_match(s, match, regex))
return {s, DefaultOutputs()};
if (match[3].matched)
return {match[1], AllOutputs()};
return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")};
}
}

View file

@ -32,4 +32,16 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view
StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
typedef std::set<std::string> OutputNames;
struct AllOutputs { };
struct DefaultOutputs { };
typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec;
/* Parse a string of the form 'prefix^output1,...outputN' or
'prefix^*', returning the prefix and the outputs spec. */
std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s);
}

View file

@ -0,0 +1,46 @@
#include "path-with-outputs.hh"
#include <gtest/gtest.h>
namespace nix {
TEST(parseOutputsSpec, basic)
{
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^*");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin");
ASSERT_EQ(prefix, "foo");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin");
ASSERT_EQ(prefix, "foo^bar");
ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"}));
}
{
auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()");
ASSERT_EQ(prefix, "foo^&*()");
ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec));
}
}
}

View file

@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand
auto val = installable->toValue(*evalState).first;
auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false };
InstallableFlake bundler{this,
evalState, std::move(bundlerFlakeRef), bundlerName,
evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec,
{"bundlers." + settings.thisSystem.get() + ".default",
"defaultBundler." + settings.thisSystem.get()
},

View file

@ -507,6 +507,7 @@ struct CmdDevelop : Common, MixEnvironment
state,
installable->nixpkgsFlakeRef(),
"bashInteractive",
DefaultOutputs(),
Strings{},
Strings{"legacyPackages." + settings.thisSystem.get() + "."},
nixpkgsLockFlags);

View file

@ -724,7 +724,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
auto installable = InstallableFlake(nullptr,
evalState, std::move(templateFlakeRef), templateName,
evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(),
defaultTemplateAttrPaths,
defaultTemplateAttrPathsPrefixes,
lockFlags);

View file

@ -146,6 +146,51 @@ For most commands, if no installable is specified, the default is `.`,
i.e. Nix will operate on the default flake output attribute of the
flake in the current directory.
## Derivation output selection
Derivations can have multiple outputs, each corresponding to a
different store path. For instance, a package can have a `bin` output
that contains programs, and a `dev` output that provides development
artifacts like C/C++ header files. The outputs on which `nix` commands
operate are determined as follows:
* You can explicitly specify the desired outputs using the syntax
*installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can
obtain the `dev` and `static` outputs of the `glibc` package:
```console
# nix build 'nixpkgs#glibc^dev,static'
# ls ./result-dev/include/ ./result-static/lib/
```
* You can also specify that *all* outputs should be used using the
syntax *installable*`^*`. For example, the following shows the size
of all outputs of the `glibc` package in the binary cache:
```console
# nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*'
/nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200
/nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896
/nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312
/nix/store/n4xa8h6pbmqmwnq0mmsz08l38abb06zc-glibc-2.33-123-static 42488328
/nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560
```
* If you didn't specify the desired outputs, but the derivation has an
attribute `meta.outputsToInstall`, Nix will use those outputs. For
example, since the package `nixpkgs#libxml2` has this attribute:
```console
# nix eval 'nixpkgs#libxml2.meta.outputsToInstall'
[ "bin" "man" ]
```
a command like `nix shell nixpkgs#libxml2` will provide only those
two outputs by default.
* Otherwise, Nix will use all outputs of the derivation.
# Nix stores
Most `nix` subcommands operate on a *Nix store*.

View file

@ -443,6 +443,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
getEvalState(),
FlakeRef(element.source->originalRef),
"",
DefaultOutputs(), // FIXME
Strings{element.source->attrPath},
Strings{},
lockFlags);

View file

@ -2,6 +2,8 @@ source common.sh
clearStore
set -o pipefail
# Make sure that 'nix build' returns all outputs by default.
nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
(.[0] |
@ -15,6 +17,45 @@ nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '
(.outputs.out | match(".*multiple-outputs-b")))
'
# Test output selection using the '^' syntax.
nix build -f multiple-outputs.nix --json a^first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first"]))
'
nix build -f multiple-outputs.nix --json a^second,first --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
nix build -f multiple-outputs.nix --json 'a^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-a.drv")) and
(.outputs | keys == ["first", "second"]))
'
# Test that 'outputsToInstall' is respected by default.
nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b"]))
'
# But not when it's overriden.
nix build -f multiple-outputs.nix --json e^a --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a"]))
'
nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a", "b", "c"]))
'
testNormalization () {
clearStore
outPath=$(nix-build ./simple.nix --no-out-link)

View file

@ -80,4 +80,11 @@ rec {
'';
}).a;
e = mkDerivation {
name = "multiple-outputs-e";
outputs = [ "a" "b" "c" ];
meta.outputsToInstall = [ "a" "b" ];
buildCommand = "mkdir $a $b $c";
};
}

View file

@ -3,15 +3,24 @@ with import ./config.nix;
{
hello = mkDerivation {
name = "hello";
outputs = [ "out" "dev" ];
meta.outputsToInstall = [ "out" ];
buildCommand =
''
mkdir -p $out/bin
mkdir -p $out/bin $dev/bin
cat > $out/bin/hello <<EOF
#! ${shell}
who=\$1
echo "Hello \''${who:-World} from $out/bin/hello"
EOF
chmod +x $out/bin/hello
cat > $dev/bin/hello2 <<EOF
#! ${shell}
echo "Hello2"
EOF
chmod +x $dev/bin/hello2
'';
};
}

View file

@ -6,6 +6,10 @@ clearCache
nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
# Test output selection.
nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2'
nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2'
if ! canUseSandbox; then exit 99; fi
chmod -R u+w $TEST_ROOT/store0 || true