Merge pull request #8813 from obsidiansystems/outputOf

Create (experimental) `outputOf` primop.
This commit is contained in:
Robert Hensing 2023-08-14 16:53:39 +02:00 committed by GitHub
commit 5542c1f87e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 268 additions and 77 deletions

View file

@ -19,3 +19,6 @@
- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. - The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field.
This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op.
- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin.
It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature.

View file

@ -1027,24 +1027,67 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
} }
std::string EvalState::mkOutputStringRaw(
const SingleDerivedPath::Built & b,
std::optional<StorePath> optStaticOutputPath,
const ExperimentalFeatureSettings & xpSettings)
{
/* In practice, this is testing for the case of CA derivations, or
dynamic derivations. */
return optStaticOutputPath
? store->printStorePath(*std::move(optStaticOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
}
void EvalState::mkOutputString( void EvalState::mkOutputString(
Value & value, Value & value,
const StorePath & drvPath, const SingleDerivedPath::Built & b,
const std::string outputName, std::optional<StorePath> optStaticOutputPath,
std::optional<StorePath> optOutputPath,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {
value.mkString( value.mkString(
optOutputPath mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
? store->printStorePath(*std::move(optOutputPath)) NixStringContext { b });
/* Downstream we would substitute this for an actual path once }
we build the floating CA derivation */
: DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(),
std::string EvalState::mkSingleDerivedPathStringRaw(
const SingleDerivedPath & p)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
return store->printStorePath(o.path);
},
[&](const SingleDerivedPath::Built & b) {
auto optStaticOutputPath = std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
auto drv = store->readDerivation(o.path);
auto i = drv.outputs.find(b.output);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
return i->second.path(*store, drv.name, b.output);
},
[&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
return std::nullopt;
},
}, b.drvPath->raw());
return mkOutputStringRaw(b, optStaticOutputPath);
}
}, p.raw());
}
void EvalState::mkSingleDerivedPathString(
const SingleDerivedPath & p,
Value & v)
{
v.mkString(
mkSingleDerivedPathStringRaw(p),
NixStringContext { NixStringContext {
NixStringContextElem::Built { std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
.drvPath = makeConstantStorePathRef(drvPath),
.output = outputName,
}
}); });
} }
@ -2333,39 +2376,25 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value &
{ {
auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_; auto s = s_;
std::visit(overloaded { auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
[&](const SingleDerivedPath::Opaque & o) { if (s != sExpected) {
auto sExpected = store->printStorePath(o.path); /* `std::visit` is used here just to provide a more precise
if (s != sExpected) error message. */
std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
error( error(
"path string '%s' has context with the different path '%s'", "path string '%s' has context with the different path '%s'",
s, sExpected) s, sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>(); .withTrace(pos, errorCtx).debugThrow<EvalError>();
}, },
[&](const SingleDerivedPath::Built & b) { [&](const SingleDerivedPath::Built & b) {
auto sExpected = std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
auto drv = store->readDerivation(o.path);
auto i = drv.outputs.find(b.output);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
auto optOutputPath = i->second.path(*store, drv.name, b.output);
// This is testing for the case of CA derivations
return optOutputPath
? store->printStorePath(*optOutputPath)
: DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
},
[&](const SingleDerivedPath::Built & o) {
return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
},
}, b.drvPath->raw());
if (s != sExpected)
error( error(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s, b.output, b.drvPath->to_string(*store), sExpected) s, b.output, b.drvPath->to_string(*store), sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>(); .withTrace(pos, errorCtx).debugThrow<EvalError>();
} }
}, derivedPath.raw()); }, derivedPath.raw());
}
return derivedPath; return derivedPath;
} }

View file

@ -668,37 +668,46 @@ public:
/** /**
* Create a string representing a store path. * Create a string representing a store path.
* *
* The string is the printed store path with a context containing a single * The string is the printed store path with a context containing a
* `NixStringContextElem::Opaque` element of that store path. * single `NixStringContextElem::Opaque` element of that store path.
*/ */
void mkStorePathString(const StorePath & storePath, Value & v); void mkStorePathString(const StorePath & storePath, Value & v);
/** /**
* Create a string representing a `DerivedPath::Built`. * Create a string representing a `SingleDerivedPath::Built`.
* *
* The string is the printed store path with a context containing a single * The string is the printed store path with a context containing a
* `NixStringContextElem::Built` element of the drv path and output name. * single `NixStringContextElem::Built` element of the drv path and
* output name.
* *
* @param value Value we are settings * @param value Value we are settings
* *
* @param drvPath Path the drv whose output we are making a string for * @param b the drv whose output we are making a string for, and the
* output
* *
* @param outputName Name of the output * @param optStaticOutputPath Optional output path for that string.
* * Must be passed if and only if output store object is
* @param optOutputPath Optional output path for that string. Must * input-addressed or fixed output. Will be printed to form string
* be passed if and only if output store object is input-addressed. * if passed, otherwise a placeholder will be used (see
* Will be printed to form string if passed, otherwise a placeholder * `DownstreamPlaceholder`).
* will be used (see `DownstreamPlaceholder`).
* *
* @param xpSettings Stop-gap to avoid globals during unit tests. * @param xpSettings Stop-gap to avoid globals during unit tests.
*/ */
void mkOutputString( void mkOutputString(
Value & value, Value & value,
const StorePath & drvPath, const SingleDerivedPath::Built & b,
const std::string outputName, std::optional<StorePath> optStaticOutputPath,
std::optional<StorePath> optOutputPath,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Create a string representing a `SingleDerivedPath`.
*
* A combination of `mkStorePathString` and `mkOutputString`.
*/
void mkSingleDerivedPathString(
const SingleDerivedPath & p,
Value & v);
void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
/** /**
@ -714,6 +723,22 @@ public:
private: private:
/**
* Like `mkOutputString` but just creates a raw string, not an
* string Value, which would also have a string context.
*/
std::string mkOutputStringRaw(
const SingleDerivedPath::Built & b,
std::optional<StorePath> optStaticOutputPath,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Like `mkSingleDerivedPathStringRaw` but just creates a raw string
* Value, which would also have a string context.
*/
std::string mkSingleDerivedPathStringRaw(
const SingleDerivedPath & p);
unsigned long nrEnvs = 0; unsigned long nrEnvs = 0;
unsigned long nrValuesInEnvs = 0; unsigned long nrValuesInEnvs = 0;
unsigned long nrValues = 0; unsigned long nrValues = 0;

View file

@ -156,8 +156,10 @@ static void mkOutputString(
{ {
state.mkOutputString( state.mkOutputString(
attrs.alloc(o.first), attrs.alloc(o.first),
drvPath, SingleDerivedPath::Built {
o.first, .drvPath = makeConstantStorePathRef(drvPath),
.output = o.first,
},
o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
} }
@ -1836,6 +1838,45 @@ static RegisterPrimOp primop_readDir({
.fun = prim_readDir, .fun = prim_readDir,
}); });
/* Extend single element string context with another output. */
static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
state.mkSingleDerivedPathString(
SingleDerivedPath::Built {
.drvPath = make_ref<SingleDerivedPath>(drvPath),
.output = std::string { outputName },
},
v);
}
static RegisterPrimOp primop_outputOf({
.name = "__outputOf",
.args = {"derivation-reference", "output-name"},
.doc = R"(
Return the output path of a derivation, literally or using a placeholder if needed.
If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned.
But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead.
*`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
This primop can be chained arbitrarily deeply.
For instance,
```nix
builtins.outputOf
(builtins.outputOf myDrv "out)
"out"
```
will return a placeholder for the output of the output of `myDrv`.
This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.
)",
.fun = prim_outputOf,
.experimentalFeature = Xp::DynamicDerivations,
});
/************************************************************* /*************************************************************
* Creating files * Creating files

View file

@ -34,8 +34,8 @@ RC_GTEST_FIXTURE_PROP(
RC_GTEST_FIXTURE_PROP( RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest, DerivedPathExpressionTest,
prop_built_path_placeholder_round_trip, prop_derived_path_built_placeholder_round_trip,
(const StorePath & drvPath, const StorePathName & outputName)) (const SingleDerivedPath::Built & b))
{ {
/** /**
* We set these in tests rather than the regular globals so we don't have * We set these in tests rather than the regular globals so we don't have
@ -45,27 +45,19 @@ RC_GTEST_FIXTURE_PROP(
mockXpSettings.set("experimental-features", "ca-derivations"); mockXpSettings.set("experimental-features", "ca-derivations");
auto * v = state.allocValue(); auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); state.mkOutputString(*v, b, std::nullopt, mockXpSettings);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
SingleDerivedPath::Built b {
.drvPath = makeConstantStorePathRef(drvPath),
.output = outputName.name,
};
RC_ASSERT(SingleDerivedPath { b } == d); RC_ASSERT(SingleDerivedPath { b } == d);
} }
RC_GTEST_FIXTURE_PROP( RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest, DerivedPathExpressionTest,
prop_built_path_out_path_round_trip, prop_derived_path_built_out_path_round_trip,
(const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath)) (const SingleDerivedPath::Built & b, const StorePath & outPath))
{ {
auto * v = state.allocValue(); auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, outPath); state.mkOutputString(*v, b, outPath);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
SingleDerivedPath::Built b {
.drvPath = makeConstantStorePathRef(drvPath),
.output = outputName.name,
};
RC_ASSERT(SingleDerivedPath { b } == d); RC_ASSERT(SingleDerivedPath { b } == d);
} }

View file

@ -39,16 +39,18 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
} }
DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
const SingleDerivedPath::Built & b) const SingleDerivedPath::Built & b,
const ExperimentalFeatureSettings & xpSettings)
{ {
return std::visit(overloaded { return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) { [&](const SingleDerivedPath::Opaque & o) {
return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings);
}, },
[&](const SingleDerivedPath::Built & b2) { [&](const SingleDerivedPath::Built & b2) {
return DownstreamPlaceholder::unknownDerivation( return DownstreamPlaceholder::unknownDerivation(
DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings),
b.output); b.output,
xpSettings);
}, },
}, b.drvPath->raw()); }, b.drvPath->raw());
} }

View file

@ -84,7 +84,8 @@ public:
* `SingleDerivedPath::Built.drvPath` chain. * `SingleDerivedPath::Built.drvPath` chain.
*/ */
static DownstreamPlaceholder fromSingleDerivedPathBuilt( static DownstreamPlaceholder fromSingleDerivedPathBuilt(
const SingleDerivedPath::Built & built); const SingleDerivedPath::Built & built,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
}; };
} }

View file

@ -359,6 +359,7 @@ void mainWrapped(int argc, char * * argv)
experimentalFeatureSettings.experimentalFeatures = { experimentalFeatureSettings.experimentalFeatures = {
Xp::Flakes, Xp::Flakes,
Xp::FetchClosure, Xp::FetchClosure,
Xp::DynamicDerivations,
}; };
evalSettings.pureEval = false; evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://")); EvalState state({}, openStore("dummy://"));

View file

@ -0,0 +1,9 @@
#!/usr/bin/env bash
source common.sh
out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link)
clearStore
expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported"

View file

@ -0,0 +1,80 @@
#!/usr/bin/env bash
source ./common.sh
# Without the dynamic-derivations XP feature, we don't have the builtin.
nix --experimental-features 'nix-command' eval --impure --expr \
'assert ! (builtins ? outputOf); ""'
# Test that a string is required.
#
# We currently require a string to be passed, rather than a derivation
# object that could be coerced to a string. We might liberalise this in
# the future so it does work, but there are some design questions to
# resolve first. Adding a test so we don't liberalise it by accident.
expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
'builtins.outputOf (import ../dependencies.nix) "out"' \
| grepQuiet "value is a set while a string was expected"
# Test that "DrvDeep" string contexts are not supported at this time
#
# Like the above, this is a restriction we could relax later.
expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \
'builtins.outputOf (import ../dependencies.nix).drvPath "out"' \
| grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time"
# Test using `builtins.outputOf` with static derivations
testStaticHello () {
nix eval --impure --expr \
'with (import ./text-hashed-output.nix); let
a = hello.outPath;
b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out";
in builtins.trace a
(builtins.trace b
(assert a == b; null))'
}
# Test with a regular old input-addresed derivation
#
# `builtins.outputOf` works without ca-derivations and doesn't create a
# placeholder but just returns the output path.
testStaticHello
# Test with content addressed derivation.
NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello
# Test with derivation-producing derivation
#
# This is hardly different from the preceding cases, except that we're
# only taking 1 outputOf out of 2 possible outputOfs. Note that
# `.outPath` could be defined as `outputOf drvPath`, which is what we're
# testing here. The other `outputOf` that we're not testing here is the
# use of _dynamic_ derivations.
nix eval --impure --expr \
'with (import ./text-hashed-output.nix); let
a = producingDrv.outPath;
b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out";
in builtins.trace a
(builtins.trace b
(assert a == b; null))'
# Test with unbuilt output of derivation-producing derivation.
#
# This function similar to `testStaticHello` used above, but instead of
# checking the property on a constant derivation, we check it on a
# derivation that's from another derivation's output (outPath).
testDynamicHello () {
nix eval --impure --expr \
'with (import ./text-hashed-output.nix); let
a = builtins.outputOf producingDrv.outPath "out";
b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out";
in builtins.trace a
(builtins.trace b
(assert a == b; null))'
}
# inner dynamic derivation is input-addressed
testDynamicHello
# inner dynamic derivation is content-addressed
NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello

View file

@ -1,7 +1,9 @@
dyn-drv-tests := \ dyn-drv-tests := \
$(d)/text-hashed-output.sh \ $(d)/text-hashed-output.sh \
$(d)/recursive-mod-json.sh \ $(d)/recursive-mod-json.sh \
$(d)/build-built-drv.sh $(d)/build-built-drv.sh \
$(d)/eval-outputOf.sh \
$(d)/dep-built-drv.sh
install-tests-groups += dyn-drv install-tests-groups += dyn-drv

View file

@ -3,6 +3,8 @@ source common.sh
# FIXME # FIXME
if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi
export NIX_TESTS_CA_BY_DEFAULT=1
enableFeatures 'recursive-nix' enableFeatures 'recursive-nix'
restartDaemon restartDaemon

View file

@ -12,9 +12,6 @@ rec {
mkdir -p $out mkdir -p $out
echo "Hello World" > $out/hello echo "Hello World" > $out/hello
''; '';
__contentAddressed = true;
outputHashMode = "recursive";
outputHashAlgo = "sha256";
}; };
producingDrv = mkDerivation { producingDrv = mkDerivation {
name = "hello.drv"; name = "hello.drv";
@ -26,4 +23,11 @@ rec {
outputHashMode = "text"; outputHashMode = "text";
outputHashAlgo = "sha256"; outputHashAlgo = "sha256";
}; };
wrapper = mkDerivation {
name = "use-dynamic-drv-in-non-dynamic-drv";
buildCommand = ''
echo "Copying the output of the dynamic derivation"
cp -r ${builtins.outputOf producingDrv.outPath "out"} $out
'';
};
} }