Merge pull request #7601 from obsidiansystems/string-installables

Make more string values work as installables
This commit is contained in:
Robert Hensing 2023-05-15 15:42:57 +02:00 committed by GitHub
commit 0c49c1af28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 343 additions and 60 deletions

View file

@ -46,7 +46,15 @@ std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state)
DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
{ {
auto v = toValue(*state).first; auto [v, pos] = toValue(*state);
if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
*v,
pos,
fmt("while evaluating the attribute '%s'", attrPath)))
{
return { *derivedPathWithInfo };
}
Bindings & autoArgs = *cmd.getAutoArgs(*state); Bindings & autoArgs = *cmd.getAutoArgs(*state);

View file

@ -95,31 +95,13 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
// FIXME: use eval cache? // FIXME: use eval cache?
auto v = attr->forceValue(); auto v = attr->forceValue();
if (v.type() == nPath) { if (std::optional derivedPathWithInfo = trySinglePathToDerivedPaths(
auto storePath = v.path().fetchToStore(state->store); v,
return {{ noPos,
.path = DerivedPath::Opaque { fmt("while evaluating the flake output attribute '%s'", attrPath)))
.path = std::move(storePath), {
}, return { *derivedPathWithInfo };
.info = make_ref<ExtraPathInfo>(),
}};
} }
else if (v.type() == nString) {
NixStringContext context;
auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath));
auto storePath = state->store->maybeParseStorePath(s);
if (storePath && context.count(NixStringContextElem::Opaque { .path = *storePath })) {
return {{
.path = DerivedPath::Opaque {
.path = std::move(*storePath),
},
.info = make_ref<ExtraPathInfo>(),
}};
} else
throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s);
}
else else
throw Error("flake output attribute '%s' is not a derivation or path", attrPath); throw Error("flake output attribute '%s' is not a derivation or path", attrPath);
} }

View file

@ -41,4 +41,26 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
return ref { castedInstallable }; return ref { castedInstallable };
} }
std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
{
if (v.type() == nPath) {
auto storePath = v.path().fetchToStore(state->store);
return {{
.path = DerivedPath::Opaque {
.path = std::move(storePath),
},
.info = make_ref<ExtraPathInfo>(),
}};
}
else if (v.type() == nString) {
return {{
.path = state->coerceToDerivedPath(pos, v, errorCtx),
.info = make_ref<ExtraPathInfo>(),
}};
}
else return std::nullopt;
}
} }

View file

@ -98,6 +98,24 @@ struct InstallableValue : Installable
static InstallableValue & require(Installable & installable); static InstallableValue & require(Installable & installable);
static ref<InstallableValue> require(ref<Installable> installable); static ref<InstallableValue> require(ref<Installable> installable);
protected:
/**
* Handles either a plain path, or a string with a single string
* context elem in the right format. The latter case is handled by
* `EvalState::coerceToDerivedPath()`; see it for details.
*
* @param v Value that is hopefully a string or path per the above.
*
* @param pos Position of value to aid with diagnostics.
*
* @param errorCtx Arbitrary message for use in potential error message when something is wrong with `v`.
*
* @result A derived path (with empty info, for now) if the value
* matched the above criteria.
*/
std::optional<DerivedPathWithInfo> trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx);
}; };
} }

View file

@ -1047,6 +1047,27 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
} }
void EvalState::mkOutputString(
Value & value,
const StorePath & drvPath,
const std::string outputName,
std::optional<StorePath> optOutputPath)
{
value.mkString(
optOutputPath
? store->printStorePath(*std::move(optOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: downstreamPlaceholder(*store, drvPath, outputName),
NixStringContext {
NixStringContextElem::Built {
.drvPath = drvPath,
.output = outputName,
}
});
}
/* Create a thunk for the delayed computation of the given expression /* Create a thunk for the delayed computation of the given expression
in the given environment. But if the expression is a variable, in the given environment. But if the expression is a variable,
then look it up right away. This significantly reduces the number then look it up right away. This significantly reduces the number
@ -2297,6 +2318,80 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
} }
std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
{
NixStringContext context;
auto s = forceString(v, context, pos, errorCtx);
auto csize = context.size();
if (csize != 1)
error(
"string '%s' has %d entries in its context. It should only have exactly one entry",
s, csize)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
auto derivedPath = std::visit(overloaded {
[&](NixStringContextElem::Opaque && o) -> DerivedPath {
return DerivedPath::Opaque {
.path = std::move(o.path),
};
},
[&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
error(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s).withTrace(pos, errorCtx).debugThrow<EvalError>();
},
[&](NixStringContextElem::Built && b) -> DerivedPath {
return DerivedPath::Built {
.drvPath = std::move(b.drvPath),
.outputs = OutputsSpec::Names { std::move(b.output) },
};
},
}, ((NixStringContextElem &&) *context.begin()).raw());
return {
std::move(derivedPath),
std::move(s),
};
}
DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
{
auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_;
std::visit(overloaded {
[&](const DerivedPath::Opaque & o) {
auto sExpected = store->printStorePath(o.path);
if (s != sExpected)
error(
"path string '%s' has context with the different path '%s'",
s, sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
},
[&](const DerivedPath::Built & b) {
// TODO need derived path with single output to make this
// total. Will add as part of RFC 92 work and then this is
// cleaned up.
auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
auto drv = store->readDerivation(b.drvPath);
auto i = drv.outputs.find(output);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
auto optOutputPath = i->second.path(*store, drv.name, output);
// This is testing for the case of CA derivations
auto sExpected = optOutputPath
? store->printStorePath(*optOutputPath)
: downstreamPlaceholder(*store, b.drvPath, output);
if (s != sExpected)
error(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s, output, store->printStorePath(b.drvPath), sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
}
}, derivedPath.raw());
return derivedPath;
}
bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{ {
forceValue(v1, noPos); forceValue(v1, noPos);

View file

@ -21,6 +21,7 @@ namespace nix {
class Store; class Store;
class EvalState; class EvalState;
class StorePath; class StorePath;
struct DerivedPath;
enum RepairFlag : bool; enum RepairFlag : bool;
@ -473,6 +474,28 @@ public:
*/ */
StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
/**
* Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
*/
std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
/**
* Coerce to `DerivedPath`.
*
* Must be a string which is either a literal store path or a
* "placeholder (see `downstreamPlaceholder()`).
*
* Even more importantly, the string context must be exactly one
* element, which is either a `NixStringContextElem::Opaque` or
* `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep`
* is not permitted).
*
* The string is parsed based on the context --- the context is the
* source of truth, and ultimately tells us what we want, and then
* we ensure the string corresponds to it.
*/
DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
public: public:
/** /**
@ -576,12 +599,37 @@ public:
void mkThunk_(Value & v, Expr * expr); void mkThunk_(Value & v, Expr * expr);
void mkPos(Value & v, PosIdx pos); void mkPos(Value & v, PosIdx pos);
/* Create a string representing a store path. /**
* Create a string representing a store path.
The string is the printed store path with a context containing a single *
`Opaque` element of that store path. */ * The string is the printed store path with a context containing a single
* `NixStringContextElem::Opaque` element of that store path.
*/
void mkStorePathString(const StorePath & storePath, Value & v); void mkStorePathString(const StorePath & storePath, Value & v);
/**
* Create a string representing a `DerivedPath::Built`.
*
* The string is the printed store path with a context containing a single
* `NixStringContextElem::Built` element of the drv path and output name.
*
* @param value Value we are settings
*
* @param drvPath Path the drv whose output we are making a string for
*
* @param outputName Name of the output
*
* @param optOutputPath Optional output path for that string. Must
* be passed if and only if output store object is input-addressed.
* Will be printed to form string if passed, otherwise a placeholder
* will be used (see `downstreamPlaceholder()`).
*/
void mkOutputString(
Value & value,
const StorePath & drvPath,
const std::string outputName,
std::optional<StorePath> optOutputPath);
void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
/** /**

View file

@ -129,40 +129,31 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
} }
} }
/* Add and attribute to the given attribute map from the output name to /**
the output path, or a placeholder. * Add and attribute to the given attribute map from the output name to
* the output path, or a placeholder.
Where possible the path is used, but for floating CA derivations we *
may not know it. For sake of determinism we always assume we don't * Where possible the path is used, but for floating CA derivations we
and instead put in a place holder. In either case, however, the * may not know it. For sake of determinism we always assume we don't
string context will contain the drv path and output name, so * and instead put in a place holder. In either case, however, the
downstream derivations will have the proper dependency, and in * string context will contain the drv path and output name, so
addition, before building, the placeholder will be rewritten to be * downstream derivations will have the proper dependency, and in
the actual path. * addition, before building, the placeholder will be rewritten to be
* the actual path.
The 'drv' and 'drvPath' outputs must correspond. */ *
* The 'drv' and 'drvPath' outputs must correspond.
*/
static void mkOutputString( static void mkOutputString(
EvalState & state, EvalState & state,
BindingsBuilder & attrs, BindingsBuilder & attrs,
const StorePath & drvPath, const StorePath & drvPath,
const BasicDerivation & drv,
const std::pair<std::string, DerivationOutput> & o) const std::pair<std::string, DerivationOutput> & o)
{ {
auto optOutputPath = o.second.path(*state.store, drv.name, o.first); state.mkOutputString(
attrs.alloc(o.first).mkString( attrs.alloc(o.first),
optOutputPath drvPath,
? state.store->printStorePath(*optOutputPath) o.first,
/* Downstream we would substitute this for an actual path once o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first));
we build the floating CA derivation */
/* FIXME: we need to depend on the basic derivation, not
derivation */
: downstreamPlaceholder(*state.store, drvPath, o.first),
NixStringContext {
NixStringContextElem::Built {
.drvPath = drvPath,
.output = o.first,
}
});
} }
/* Load and evaluate an expression from path specified by the /* Load and evaluate an expression from path specified by the
@ -193,7 +184,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.mkList(outputsVal, drv.outputs.size()); state.mkList(outputsVal, drv.outputs.size());
for (const auto & [i, o] : enumerate(drv.outputs)) { for (const auto & [i, o] : enumerate(drv.outputs)) {
mkOutputString(state, attrs, *storePath, drv, o); mkOutputString(state, attrs, *storePath, o);
(outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first);
} }
@ -1405,7 +1396,7 @@ drvName, Bindings * attrs, Value & v)
NixStringContextElem::DrvDeep { .drvPath = drvPath }, NixStringContextElem::DrvDeep { .drvPath = drvPath },
}); });
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
mkOutputString(state, result, drvPath, drv, i); mkOutputString(state, result, drvPath, i);
v.mkAttrs(result); v.mkAttrs(result);
} }

View file

@ -0,0 +1,65 @@
#include <nlohmann/json.hpp>
#include <gtest/gtest.h>
#include <rapidcheck/gtest.h>
#include "tests/derived-path.hh"
#include "tests/libexpr.hh"
namespace nix {
// Testing of trivial expressions
class DerivedPathExpressionTest : public LibExprTest {};
// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
// no a real fixture.
//
// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
TEST_F(DerivedPathExpressionTest, force_init)
{
}
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_opaque_path_round_trip,
(const DerivedPath::Opaque & o))
{
auto * v = state.allocValue();
state.mkStorePathString(o.path, *v);
auto d = state.coerceToDerivedPath(noPos, *v, "");
RC_ASSERT(DerivedPath { o } == d);
}
// TODO use DerivedPath::Built for parameter once it supports a single output
// path only.
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_built_path_placeholder_round_trip,
(const StorePath & drvPath, const StorePathName & outputName))
{
auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, std::nullopt);
auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
DerivedPath::Built b {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName.name },
};
RC_ASSERT(DerivedPath { b } == d);
}
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_built_path_out_path_round_trip,
(const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath))
{
auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, outPath);
auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
DerivedPath::Built b {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName.name },
};
RC_ASSERT(DerivedPath { b } == d);
}
} /* namespace nix */

View file

@ -57,6 +57,30 @@ nix build -f multiple-outputs.nix --json 'e^*' --no-link | jq --exit-status '
(.outputs | keys == ["a_a", "b", "c"])) (.outputs | keys == ["a_a", "b", "c"]))
' '
# test buidling from non-drv attr path
nix build -f multiple-outputs.nix --json 'e.a_a.outPath' --no-link | jq --exit-status '
(.[0] |
(.drvPath | match(".*multiple-outputs-e.drv")) and
(.outputs | keys == ["a_a"]))
'
# Illegal type of string context
expectStderr 1 nix build -f multiple-outputs.nix 'e.a_a.drvPath' \
| grepQuiet "has a context which refers to a complete source and binary closure."
# No string context
expectStderr 1 nix build --expr '""' --no-link \
| grepQuiet "has 0 entries in its context. It should only have exactly one entry"
# Too much string context
expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e.a_a; "${drvPath}${outPath}"' --no-link \
| grepQuiet "has 2 entries in its context. It should only have exactly one entry"
nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status '
(.[0] | .path | match(".*multiple-outputs-e.drv"))
'
# Test building from raw store path to drv not expression. # Test building from raw store path to drv not expression.
drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath) drv=$(nix eval -f multiple-outputs.nix --raw a.drvPath)

View file

@ -41,10 +41,27 @@ cat > $flake1Dir/flake.nix <<EOF
a8 = builtins.storePath $dep; a8 = builtins.storePath $dep;
a9 = "$dep"; a9 = "$dep";
drvCall = with import ./config.nix; mkDerivation {
name = "simple";
builder = ./simple.builder.sh;
PATH = "";
goodPath = path;
};
a10 = builtins.unsafeDiscardOutputDependency self.drvCall.drvPath;
a11 = self.drvCall.drvPath;
a12 = self.drvCall.outPath;
a13 = "\${self.drvCall.drvPath}\${self.drvCall.outPath}";
}; };
} }
EOF EOF
cp ../simple.nix ../simple.builder.sh ../config.nix $flake1Dir/
echo bar > $flake1Dir/foo echo bar > $flake1Dir/foo
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1
@ -63,4 +80,17 @@ nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6
nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8
diff common.sh $TEST_ROOT/result diff common.sh $TEST_ROOT/result
(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9) expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9 \
| grepQuiet "has 0 entries in its context. It should only have exactly one entry"
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a10
[[ $(readlink -e $TEST_ROOT/result) = *simple.drv ]]
expectStderr 1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a11 \
| grepQuiet "has a context which refers to a complete source and binary closure"
nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12
[[ -e $TEST_ROOT/result/hello ]]
expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \
| grepQuiet "has 2 entries in its context. It should only have exactly one entry"