forked from lix-project/lix
Merge remote-tracking branch 'upstream/master' into indexed-store-path-outputs
This commit is contained in:
commit
9c6be01d5f
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -90,6 +90,7 @@ perl/Makefile.config
|
||||||
|
|
||||||
/misc/systemd/nix-daemon.service
|
/misc/systemd/nix-daemon.service
|
||||||
/misc/systemd/nix-daemon.socket
|
/misc/systemd/nix-daemon.socket
|
||||||
|
/misc/systemd/nix-daemon.conf
|
||||||
/misc/upstart/nix-daemon.conf
|
/misc/upstart/nix-daemon.conf
|
||||||
|
|
||||||
/src/resolve-system-dependencies/resolve-system-dependencies
|
/src/resolve-system-dependencies/resolve-system-dependencies
|
||||||
|
|
|
@ -14,3 +14,21 @@
|
||||||
|
|
||||||
This function is only available if you enable the experimental
|
This function is only available if you enable the experimental
|
||||||
feature `fetch-closure`.
|
feature `fetch-closure`.
|
||||||
|
|
||||||
|
* New experimental feature: *impure derivations*. These are
|
||||||
|
derivations that can produce a different result every time they're
|
||||||
|
built. Here is an example:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
stdenv.mkDerivation {
|
||||||
|
name = "impure";
|
||||||
|
__impure = true; # marks this derivation as impure
|
||||||
|
buildCommand = "date > $out";
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Running `nix build` twice on this expression will build the
|
||||||
|
derivation twice, producing two different content-addressed store
|
||||||
|
paths. Like fixed-output derivations, impure derivations have access
|
||||||
|
to the network. Only fixed-output derivations and impure derivations
|
||||||
|
can depend on an impure derivation.
|
||||||
|
|
|
@ -3,6 +3,7 @@ Description=Nix Daemon
|
||||||
Documentation=man:nix-daemon https://nixos.org/manual
|
Documentation=man:nix-daemon https://nixos.org/manual
|
||||||
RequiresMountsFor=@storedir@
|
RequiresMountsFor=@storedir@
|
||||||
RequiresMountsFor=@localstatedir@
|
RequiresMountsFor=@localstatedir@
|
||||||
|
RequiresMountsFor=@localstatedir@/nix/db
|
||||||
ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
|
ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
|
|
|
@ -90,7 +90,7 @@ poly_configure_nix_daemon_service() {
|
||||||
ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
|
ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST
|
||||||
|
|
||||||
_sudo "to run systemd-tmpfiles once to pick that path up" \
|
_sudo "to run systemd-tmpfiles once to pick that path up" \
|
||||||
sytemd-tmpfiles create --prefix=/nix/var/nix
|
systemd-tmpfiles create --prefix=/nix/var/nix
|
||||||
|
|
||||||
_sudo "to set up the nix-daemon service" \
|
_sudo "to set up the nix-daemon service" \
|
||||||
systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
|
systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC"
|
||||||
|
|
|
@ -789,15 +789,50 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
|
||||||
return installables.front();
|
return installables.front();
|
||||||
}
|
}
|
||||||
|
|
||||||
BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
|
BuiltPaths Installable::build(
|
||||||
|
ref<Store> evalStore,
|
||||||
|
ref<Store> store,
|
||||||
|
Realise mode,
|
||||||
|
const std::vector<std::shared_ptr<Installable>> & installables,
|
||||||
|
BuildMode bMode)
|
||||||
{
|
{
|
||||||
BuiltPaths res;
|
BuiltPaths res;
|
||||||
for (const auto & b : hopefullyBuiltPaths)
|
for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
|
||||||
std::visit(
|
res.push_back(builtPath);
|
||||||
overloaded{
|
return res;
|
||||||
[&](const DerivedPath::Opaque & bo) {
|
}
|
||||||
res.push_back(BuiltPath::Opaque{bo.path});
|
|
||||||
},
|
std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
|
||||||
|
ref<Store> evalStore,
|
||||||
|
ref<Store> store,
|
||||||
|
Realise mode,
|
||||||
|
const std::vector<std::shared_ptr<Installable>> & installables,
|
||||||
|
BuildMode bMode)
|
||||||
|
{
|
||||||
|
if (mode == Realise::Nothing)
|
||||||
|
settings.readOnlyMode = true;
|
||||||
|
|
||||||
|
std::vector<DerivedPath> pathsToBuild;
|
||||||
|
std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap;
|
||||||
|
|
||||||
|
for (auto & i : installables) {
|
||||||
|
for (auto b : i->toDerivedPaths()) {
|
||||||
|
pathsToBuild.push_back(b);
|
||||||
|
backmap[b].push_back(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
|
||||||
|
|
||||||
|
switch (mode) {
|
||||||
|
|
||||||
|
case Realise::Nothing:
|
||||||
|
case Realise::Derivation:
|
||||||
|
printMissing(store, pathsToBuild, lvlError);
|
||||||
|
|
||||||
|
for (auto & path : pathsToBuild) {
|
||||||
|
for (auto & installable : backmap[path]) {
|
||||||
|
std::visit(overloaded {
|
||||||
[&](const DerivedPath::Built & bfd) {
|
[&](const DerivedPath::Built & bfd) {
|
||||||
OutputPathMap outputs;
|
OutputPathMap outputs;
|
||||||
auto drv = evalStore->readDerivation(bfd.drvPath);
|
auto drv = evalStore->readDerivation(bfd.drvPath);
|
||||||
|
@ -809,17 +844,14 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
|
||||||
"the derivation '%s' doesn't have an output named '%s'",
|
"the derivation '%s' doesn't have an output named '%s'",
|
||||||
store->printStorePath(bfd.drvPath), output);
|
store->printStorePath(bfd.drvPath), output);
|
||||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
||||||
auto outputId =
|
DrvOutput outputId { outputHashes.at(output), output };
|
||||||
DrvOutput{outputHashes.at(output), output};
|
auto realisation = store->queryRealisation(outputId);
|
||||||
auto realisation =
|
|
||||||
store->queryRealisation(outputId);
|
|
||||||
if (!realisation)
|
if (!realisation)
|
||||||
throw Error(
|
throw Error(
|
||||||
"cannot operate on an output of unbuilt "
|
"cannot operate on an output of unbuilt "
|
||||||
"content-addressed derivation '%s'",
|
"content-addressed derivation '%s'",
|
||||||
outputId.to_string());
|
outputId.to_string());
|
||||||
outputs.insert_or_assign(
|
outputs.insert_or_assign(output, realisation->outPath);
|
||||||
output, realisation->outPath);
|
|
||||||
} else {
|
} else {
|
||||||
// If ca-derivations isn't enabled, assume that
|
// If ca-derivations isn't enabled, assume that
|
||||||
// the output path is statically known.
|
// the output path is statically known.
|
||||||
|
@ -829,58 +861,45 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa
|
||||||
output, *drvOutputs.at(output).second);
|
output, *drvOutputs.at(output).second);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
|
res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
|
||||||
},
|
},
|
||||||
|
[&](const DerivedPath::Opaque & bo) {
|
||||||
|
res.push_back({installable, BuiltPath::Opaque { bo.path }});
|
||||||
},
|
},
|
||||||
b.raw());
|
}, path.raw());
|
||||||
|
}
|
||||||
return res;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
BuiltPaths Installable::build(
|
break;
|
||||||
ref<Store> evalStore,
|
|
||||||
ref<Store> store,
|
|
||||||
Realise mode,
|
|
||||||
const std::vector<std::shared_ptr<Installable>> & installables,
|
|
||||||
BuildMode bMode)
|
|
||||||
{
|
|
||||||
if (mode == Realise::Nothing)
|
|
||||||
settings.readOnlyMode = true;
|
|
||||||
|
|
||||||
std::vector<DerivedPath> pathsToBuild;
|
|
||||||
|
|
||||||
for (auto & i : installables) {
|
|
||||||
auto b = i->toDerivedPaths();
|
|
||||||
pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (mode) {
|
|
||||||
case Realise::Nothing:
|
|
||||||
case Realise::Derivation:
|
|
||||||
printMissing(store, pathsToBuild, lvlError);
|
|
||||||
return getBuiltPaths(evalStore, store, pathsToBuild);
|
|
||||||
case Realise::Outputs: {
|
case Realise::Outputs: {
|
||||||
BuiltPaths res;
|
|
||||||
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
|
for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) {
|
||||||
if (!buildResult.success())
|
if (!buildResult.success())
|
||||||
buildResult.rethrow();
|
buildResult.rethrow();
|
||||||
|
|
||||||
|
for (auto & installable : backmap[buildResult.path]) {
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const DerivedPath::Built & bfd) {
|
[&](const DerivedPath::Built & bfd) {
|
||||||
std::map<std::string, StorePath> outputs;
|
std::map<std::string, StorePath> outputs;
|
||||||
for (auto & path : buildResult.builtOutputs)
|
for (auto & path : buildResult.builtOutputs)
|
||||||
outputs.emplace(path.first.outputName, path.second.outPath);
|
outputs.emplace(path.first.outputName, path.second.outPath);
|
||||||
res.push_back(BuiltPath::Built { bfd.drvPath, outputs });
|
res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
|
||||||
},
|
},
|
||||||
[&](const DerivedPath::Opaque & bo) {
|
[&](const DerivedPath::Opaque & bo) {
|
||||||
res.push_back(BuiltPath::Opaque { bo.path });
|
res.push_back({installable, BuiltPath::Opaque { bo.path }});
|
||||||
},
|
},
|
||||||
}, buildResult.path.raw());
|
}, buildResult.path.raw());
|
||||||
}
|
}
|
||||||
return res;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
assert(false);
|
assert(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
BuiltPaths Installable::toBuiltPaths(
|
BuiltPaths Installable::toBuiltPaths(
|
||||||
|
|
|
@ -98,6 +98,13 @@ struct Installable
|
||||||
const std::vector<std::shared_ptr<Installable>> & installables,
|
const std::vector<std::shared_ptr<Installable>> & installables,
|
||||||
BuildMode bMode = bmNormal);
|
BuildMode bMode = bmNormal);
|
||||||
|
|
||||||
|
static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
|
||||||
|
ref<Store> evalStore,
|
||||||
|
ref<Store> store,
|
||||||
|
Realise mode,
|
||||||
|
const std::vector<std::shared_ptr<Installable>> & installables,
|
||||||
|
BuildMode bMode = bmNormal);
|
||||||
|
|
||||||
static std::set<StorePath> toStorePaths(
|
static std::set<StorePath> toStorePaths(
|
||||||
ref<Store> evalStore,
|
ref<Store> evalStore,
|
||||||
ref<Store> store,
|
ref<Store> store,
|
||||||
|
@ -185,9 +192,4 @@ ref<eval_cache::EvalCache> openEvalCache(
|
||||||
EvalState & state,
|
EvalState & state,
|
||||||
std::shared_ptr<flake::LockedFlake> lockedFlake);
|
std::shared_ptr<flake::LockedFlake> lockedFlake);
|
||||||
|
|
||||||
BuiltPaths getBuiltPaths(
|
|
||||||
ref<Store> evalStore,
|
|
||||||
ref<Store> store,
|
|
||||||
const DerivedPaths & hopefullyBuiltPaths);
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -436,6 +436,7 @@ EvalState::EvalState(
|
||||||
, sBuilder(symbols.create("builder"))
|
, sBuilder(symbols.create("builder"))
|
||||||
, sArgs(symbols.create("args"))
|
, sArgs(symbols.create("args"))
|
||||||
, sContentAddressed(symbols.create("__contentAddressed"))
|
, sContentAddressed(symbols.create("__contentAddressed"))
|
||||||
|
, sImpure(symbols.create("__impure"))
|
||||||
, sOutputHash(symbols.create("outputHash"))
|
, sOutputHash(symbols.create("outputHash"))
|
||||||
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
|
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
|
||||||
, sOutputHashMode(symbols.create("outputHashMode"))
|
, sOutputHashMode(symbols.create("outputHashMode"))
|
||||||
|
|
|
@ -78,7 +78,7 @@ public:
|
||||||
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
|
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
|
||||||
sFile, sLine, sColumn, sFunctor, sToString,
|
sFile, sLine, sColumn, sFunctor, sToString,
|
||||||
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
|
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
|
||||||
sContentAddressed,
|
sContentAddressed, sImpure,
|
||||||
sOutputHash, sOutputHashAlgo, sOutputHashMode,
|
sOutputHash, sOutputHashAlgo, sOutputHashMode,
|
||||||
sRecurseForDerivations,
|
sRecurseForDerivations,
|
||||||
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
|
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
|
||||||
|
|
|
@ -989,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||||
PathSet context;
|
PathSet context;
|
||||||
|
|
||||||
bool contentAddressed = false;
|
bool contentAddressed = false;
|
||||||
|
bool isImpure = false;
|
||||||
std::optional<std::string> outputHash;
|
std::optional<std::string> outputHash;
|
||||||
std::string outputHashAlgo;
|
std::optional<std::string> outputHashAlgo;
|
||||||
auto ingestionMethod = FileIngestionMethod::Flat;
|
std::optional<FileIngestionMethod> ingestionMethod;
|
||||||
|
|
||||||
StringSet outputs;
|
StringSet outputs;
|
||||||
outputs.insert("out");
|
outputs.insert("out");
|
||||||
|
@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
else if (i->name == state.sImpure) {
|
||||||
|
isImpure = state.forceBool(*i->value, pos);
|
||||||
|
if (isImpure)
|
||||||
|
settings.requireExperimentalFeature(Xp::ImpureDerivations);
|
||||||
|
}
|
||||||
|
|
||||||
/* The `args' attribute is special: it supplies the
|
/* The `args' attribute is special: it supplies the
|
||||||
command-line arguments to the builder. */
|
command-line arguments to the builder. */
|
||||||
else if (i->name == state.sArgs) {
|
else if (i->name == state.sArgs) {
|
||||||
|
@ -1183,27 +1190,43 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||||
.errPos = posDrvName
|
.errPos = posDrvName
|
||||||
});
|
});
|
||||||
|
|
||||||
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
|
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo.value_or("sha256"));
|
||||||
Hash h = newHashAllowEmpty(*outputHash, ht);
|
Hash h = newHashAllowEmpty(*outputHash, ht);
|
||||||
|
|
||||||
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
|
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
|
||||||
|
auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
|
||||||
drv.env["out"] = state.store->printStorePath(outPath);
|
drv.env["out"] = state.store->printStorePath(outPath);
|
||||||
drv.outputs.insert_or_assign("out",
|
drv.outputs.insert_or_assign("out",
|
||||||
DerivationOutput::CAFixed {
|
DerivationOutput::CAFixed {
|
||||||
.hash = FixedOutputHash {
|
.hash = FixedOutputHash {
|
||||||
.method = ingestionMethod,
|
.method = method,
|
||||||
.hash = std::move(h),
|
.hash = std::move(h),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
else if (contentAddressed) {
|
else if (contentAddressed || isImpure) {
|
||||||
HashType ht = parseHashType(outputHashAlgo);
|
if (contentAddressed && isImpure)
|
||||||
|
throw EvalError({
|
||||||
|
.msg = hintfmt("derivation cannot be both content-addressed and impure"),
|
||||||
|
.errPos = posDrvName
|
||||||
|
});
|
||||||
|
|
||||||
|
auto ht = parseHashType(outputHashAlgo.value_or("sha256"));
|
||||||
|
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
|
||||||
|
|
||||||
for (auto & i : outputs) {
|
for (auto & i : outputs) {
|
||||||
drv.env[i] = hashPlaceholder(i);
|
drv.env[i] = hashPlaceholder(i);
|
||||||
|
if (isImpure)
|
||||||
|
drv.outputs.insert_or_assign(i,
|
||||||
|
DerivationOutput::Impure {
|
||||||
|
.method = method,
|
||||||
|
.hashType = ht,
|
||||||
|
});
|
||||||
|
else
|
||||||
drv.outputs.insert_or_assign(i,
|
drv.outputs.insert_or_assign(i,
|
||||||
DerivationOutput::CAFloating {
|
DerivationOutput::CAFloating {
|
||||||
.method = ingestionMethod,
|
.method = method,
|
||||||
.hashType = ht,
|
.hashType = ht,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -1222,34 +1245,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||||
DerivationOutput::Deferred { });
|
DerivationOutput::Deferred { });
|
||||||
}
|
}
|
||||||
|
|
||||||
// Regular, non-CA derivation should always return a single hash and not
|
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
|
||||||
// hash per output.
|
switch (hashModulo.kind) {
|
||||||
auto hashModulo = hashDerivationModulo(*state.store, drv, true);
|
|
||||||
std::visit(overloaded {
|
|
||||||
[&](const DrvHash & drvHash) {
|
|
||||||
auto & h = drvHash.hash;
|
|
||||||
switch (drvHash.kind) {
|
|
||||||
case DrvHash::Kind::Deferred:
|
|
||||||
/* Outputs already deferred, nothing to do */
|
|
||||||
break;
|
|
||||||
case DrvHash::Kind::Regular:
|
case DrvHash::Kind::Regular:
|
||||||
for (auto & [outputName, output] : drv.outputs) {
|
for (auto & i : outputs) {
|
||||||
auto outPath = state.store->makeOutputPath(outputName, h, drvName);
|
auto h = hashModulo.hashes.at(i);
|
||||||
drv.env[outputName] = state.store->printStorePath(outPath);
|
auto outPath = state.store->makeOutputPath(i, h, drvName);
|
||||||
output = DerivationOutput::InputAddressed {
|
drv.env[i] = state.store->printStorePath(outPath);
|
||||||
|
drv.outputs.insert_or_assign(
|
||||||
|
i,
|
||||||
|
DerivationOutputInputAddressed {
|
||||||
.path = std::move(outPath),
|
.path = std::move(outPath),
|
||||||
};
|
});
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
;
|
||||||
|
case DrvHash::Kind::Deferred:
|
||||||
|
for (auto & i : outputs) {
|
||||||
|
drv.outputs.insert_or_assign(i, DerivationOutputDeferred {});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
[&](const CaOutputHashes &) {
|
|
||||||
// Shouldn't happen as the toplevel derivation is not CA.
|
|
||||||
assert(false);
|
|
||||||
},
|
|
||||||
},
|
|
||||||
hashModulo.raw());
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Write the resulting term into the Nix store directory. */
|
/* Write the resulting term into the Nix store directory. */
|
||||||
|
|
|
@ -84,7 +84,8 @@ void printValueAsJSON(EvalState & state, bool strict,
|
||||||
.msg = hintfmt("cannot convert %1% to JSON", showType(v)),
|
.msg = hintfmt("cannot convert %1% to JSON", showType(v)),
|
||||||
.errPos = v.determinePos(pos)
|
.errPos = v.determinePos(pos)
|
||||||
});
|
});
|
||||||
throw e.addTrace(pos, hintfmt("message for the trace"));
|
e.addTrace(pos, hintfmt("message for the trace"));
|
||||||
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
|
||||||
{
|
{
|
||||||
trace("have derivation");
|
trace("have derivation");
|
||||||
|
|
||||||
|
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
|
||||||
|
|
||||||
if (!drv->type().hasKnownOutputPaths())
|
if (!drv->type().hasKnownOutputPaths())
|
||||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||||
|
|
||||||
|
if (!drv->type().isPure()) {
|
||||||
|
settings.requireExperimentalFeature(Xp::ImpureDerivations);
|
||||||
|
|
||||||
|
for (auto & [outputName, output] : drv->outputs) {
|
||||||
|
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
|
||||||
|
assert(!worker.store.isValidPath(randomPath));
|
||||||
|
initialOutputs.insert({
|
||||||
|
outputName,
|
||||||
|
InitialOutput {
|
||||||
|
.wanted = true,
|
||||||
|
.outputHash = impureOutputHash,
|
||||||
|
.known = InitialOutputStatus {
|
||||||
|
.path = randomPath,
|
||||||
|
.status = PathStatus::Absent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
gaveUpOnSubstitution();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
for (auto & i : drv->outputsAndOptPaths(worker.store))
|
for (auto & i : drv->outputsAndOptPaths(worker.store))
|
||||||
if (i.second.second)
|
if (i.second.second)
|
||||||
worker.store.addTempRoot(*i.second.second);
|
worker.store.addTempRoot(*i.second.second);
|
||||||
|
@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
|
|
||||||
|
|
||||||
|
|
||||||
/* We are first going to try to create the invalid output paths
|
/* We are first going to try to create the invalid output paths
|
||||||
through substitutes. If that doesn't work, we'll build
|
through substitutes. If that doesn't work, we'll build
|
||||||
them. */
|
them. */
|
||||||
|
@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
|
||||||
{
|
{
|
||||||
trace("all outputs substituted (maybe)");
|
trace("all outputs substituted (maybe)");
|
||||||
|
|
||||||
|
assert(drv->type().isPure());
|
||||||
|
|
||||||
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
|
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
|
||||||
done(BuildResult::TransientFailure, {},
|
done(BuildResult::TransientFailure, {},
|
||||||
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
|
||||||
|
@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
|
||||||
void DerivationGoal::gaveUpOnSubstitution()
|
void DerivationGoal::gaveUpOnSubstitution()
|
||||||
{
|
{
|
||||||
/* The inputs must be built before we can build this goal. */
|
/* The inputs must be built before we can build this goal. */
|
||||||
|
inputDrvOutputs.clear();
|
||||||
if (useDerivation)
|
if (useDerivation)
|
||||||
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
|
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
|
||||||
|
/* Ensure that pure, non-fixed-output derivations don't
|
||||||
|
depend on impure derivations. */
|
||||||
|
if (drv->type().isPure() && !drv->type().isFixed()) {
|
||||||
|
auto inputDrv = worker.evalStore.readDerivation(i.first);
|
||||||
|
if (!inputDrv.type().isPure())
|
||||||
|
throw Error("pure derivation '%s' depends on impure derivation '%s'",
|
||||||
|
worker.store.printStorePath(drvPath),
|
||||||
|
worker.store.printStorePath(i.first));
|
||||||
|
}
|
||||||
|
|
||||||
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
|
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
|
||||||
|
}
|
||||||
|
|
||||||
/* Copy the input sources from the eval store to the build
|
/* Copy the input sources from the eval store to the build
|
||||||
store. */
|
store. */
|
||||||
|
@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
|
||||||
|
|
||||||
void DerivationGoal::repairClosure()
|
void DerivationGoal::repairClosure()
|
||||||
{
|
{
|
||||||
|
assert(drv->type().isPure());
|
||||||
|
|
||||||
/* If we're repairing, we now know that our own outputs are valid.
|
/* If we're repairing, we now know that our own outputs are valid.
|
||||||
Now check whether the other paths in the outputs closure are
|
Now check whether the other paths in the outputs closure are
|
||||||
good. If not, then start derivation goals for the derivations
|
good. If not, then start derivation goals for the derivations
|
||||||
|
@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
|
||||||
drvs. */
|
drvs. */
|
||||||
: true);
|
: true);
|
||||||
},
|
},
|
||||||
|
[&](const DerivationType::Impure &) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
}, drvType.raw());
|
}, drvType.raw());
|
||||||
|
|
||||||
if (resolveDrv)
|
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
|
||||||
{
|
|
||||||
settings.requireExperimentalFeature(Xp::CaDerivations);
|
settings.requireExperimentalFeature(Xp::CaDerivations);
|
||||||
|
|
||||||
/* We are be able to resolve this derivation based on the
|
/* We are be able to resolve this derivation based on the
|
||||||
now-known results of dependencies. If so, we become a stub goal
|
now-known results of dependencies. If so, we become a
|
||||||
aliasing that resolved derivation goal */
|
stub goal aliasing that resolved derivation goal. */
|
||||||
std::optional attempt = fullDrv.tryResolve(worker.store);
|
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
|
||||||
assert(attempt);
|
assert(attempt);
|
||||||
Derivation drvResolved { *std::move(attempt) };
|
Derivation drvResolved { *std::move(attempt) };
|
||||||
|
|
||||||
auto pathResolved = writeDerivation(worker.store, drvResolved);
|
auto pathResolved = writeDerivation(worker.store, drvResolved);
|
||||||
|
|
||||||
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
|
auto msg = fmt("resolved derivation: '%s' -> '%s'",
|
||||||
worker.store.printStorePath(drvPath),
|
worker.store.printStorePath(drvPath),
|
||||||
worker.store.printStorePath(pathResolved));
|
worker.store.printStorePath(pathResolved));
|
||||||
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
|
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
|
||||||
|
@ -488,23 +528,15 @@ void DerivationGoal::inputsRealised()
|
||||||
/* Add the relevant output closures of the input derivation
|
/* Add the relevant output closures of the input derivation
|
||||||
`i' as input paths. Only add the closures of output paths
|
`i' as input paths. Only add the closures of output paths
|
||||||
that are specified as inputs. */
|
that are specified as inputs. */
|
||||||
assert(worker.evalStore.isValidPath(drvPath));
|
for (auto & j : wantedDepOutputs)
|
||||||
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
|
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
|
||||||
for (auto & j : wantedDepOutputs) {
|
worker.store.computeFSClosure(*outPath, inputPaths);
|
||||||
if (outputs.count(j) > 0) {
|
else
|
||||||
auto optRealizedInput = outputs.at(j);
|
|
||||||
if (!optRealizedInput)
|
|
||||||
throw Error(
|
|
||||||
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
|
|
||||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
|
||||||
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
|
|
||||||
} else
|
|
||||||
throw Error(
|
throw Error(
|
||||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/* Second, the input sources. */
|
/* Second, the input sources. */
|
||||||
worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
|
worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
|
||||||
|
@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
|
||||||
st =
|
st =
|
||||||
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
|
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
|
||||||
statusOk(status) ? BuildResult::OutputRejected :
|
statusOk(status) ? BuildResult::OutputRejected :
|
||||||
derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
|
!derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
|
||||||
BuildResult::PermanentFailure;
|
BuildResult::PermanentFailure;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -934,9 +966,15 @@ void DerivationGoal::buildDone()
|
||||||
|
|
||||||
void DerivationGoal::resolvedFinished()
|
void DerivationGoal::resolvedFinished()
|
||||||
{
|
{
|
||||||
|
trace("resolved derivation finished");
|
||||||
|
|
||||||
assert(resolvedDrvGoal);
|
assert(resolvedDrvGoal);
|
||||||
auto resolvedDrv = *resolvedDrvGoal->drv;
|
auto resolvedDrv = *resolvedDrvGoal->drv;
|
||||||
|
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
||||||
|
|
||||||
|
DrvOutputs builtOutputs;
|
||||||
|
|
||||||
|
if (resolvedResult.success()) {
|
||||||
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
|
||||||
|
|
||||||
StorePathSet outputPaths;
|
StorePathSet outputPaths;
|
||||||
|
@ -946,30 +984,22 @@ void DerivationGoal::resolvedFinished()
|
||||||
if (realWantedOutputs.empty())
|
if (realWantedOutputs.empty())
|
||||||
realWantedOutputs = resolvedDrv.outputNames();
|
realWantedOutputs = resolvedDrv.outputNames();
|
||||||
|
|
||||||
DrvOutputs builtOutputs;
|
|
||||||
|
|
||||||
for (auto & wantedOutput : realWantedOutputs) {
|
for (auto & wantedOutput : realWantedOutputs) {
|
||||||
assert(initialOutputs.count(wantedOutput) != 0);
|
assert(initialOutputs.count(wantedOutput) != 0);
|
||||||
assert(resolvedHashes.count(wantedOutput) != 0);
|
assert(resolvedHashes.count(wantedOutput) != 0);
|
||||||
auto realisation = worker.store.queryRealisation(
|
auto realisation = resolvedResult.builtOutputs.at(
|
||||||
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
|
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
|
||||||
);
|
if (drv->type().isPure()) {
|
||||||
// We've just built it, but maybe the build failed, in which case the
|
auto newRealisation = realisation;
|
||||||
// realisation won't be there
|
|
||||||
if (realisation) {
|
|
||||||
auto newRealisation = *realisation;
|
|
||||||
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
|
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
|
||||||
newRealisation.signatures.clear();
|
newRealisation.signatures.clear();
|
||||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
|
if (!drv->type().isFixed())
|
||||||
|
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
|
||||||
signRealisation(newRealisation);
|
signRealisation(newRealisation);
|
||||||
worker.store.registerDrvOutput(newRealisation);
|
worker.store.registerDrvOutput(newRealisation);
|
||||||
outputPaths.insert(realisation->outPath);
|
|
||||||
builtOutputs.emplace(realisation->id, *realisation);
|
|
||||||
} else {
|
|
||||||
// If we don't have a realisation, then it must mean that something
|
|
||||||
// failed when building the resolved drv
|
|
||||||
assert(!buildResult.success());
|
|
||||||
}
|
}
|
||||||
|
outputPaths.insert(realisation.outPath);
|
||||||
|
builtOutputs.emplace(realisation.id, realisation);
|
||||||
}
|
}
|
||||||
|
|
||||||
runPostBuildHook(
|
runPostBuildHook(
|
||||||
|
@ -978,16 +1008,11 @@ void DerivationGoal::resolvedFinished()
|
||||||
drvPath,
|
drvPath,
|
||||||
outputPaths
|
outputPaths
|
||||||
);
|
);
|
||||||
|
|
||||||
auto status = [&]() {
|
|
||||||
auto & resolvedResult = resolvedDrvGoal->buildResult;
|
|
||||||
switch (resolvedResult.status) {
|
|
||||||
case BuildResult::AlreadyValid:
|
|
||||||
return BuildResult::ResolvesToAlreadyValid;
|
|
||||||
default:
|
|
||||||
return resolvedResult.status;
|
|
||||||
}
|
}
|
||||||
}();
|
|
||||||
|
auto status = resolvedResult.status;
|
||||||
|
if (status == BuildResult::AlreadyValid)
|
||||||
|
status = BuildResult::ResolvesToAlreadyValid;
|
||||||
|
|
||||||
done(status, std::move(builtOutputs));
|
done(status, std::move(builtOutputs));
|
||||||
}
|
}
|
||||||
|
@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine()
|
||||||
|
|
||||||
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
|
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
|
||||||
{
|
{
|
||||||
|
assert(drv->type().isPure());
|
||||||
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
||||||
std::map<std::string, std::optional<StorePath>> res;
|
std::map<std::string, std::optional<StorePath>> res;
|
||||||
for (auto & [name, output] : drv->outputs)
|
for (auto & [name, output] : drv->outputs)
|
||||||
|
@ -1248,6 +1274,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
|
||||||
|
|
||||||
OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||||
{
|
{
|
||||||
|
assert(drv->type().isPure());
|
||||||
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
|
||||||
OutputPathMap res;
|
OutputPathMap res;
|
||||||
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
|
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
|
||||||
|
@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||||
|
|
||||||
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
||||||
{
|
{
|
||||||
|
if (!drv->type().isPure()) return { false, {} };
|
||||||
|
|
||||||
bool checkHash = buildMode == bmRepair;
|
bool checkHash = buildMode == bmRepair;
|
||||||
auto wantedOutputsLeft = wantedOutputs;
|
auto wantedOutputsLeft = wantedOutputs;
|
||||||
DrvOutputs validOutputs;
|
DrvOutputs validOutputs;
|
||||||
|
@ -1304,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
|
||||||
if (info.wanted && info.known && info.known->isValid())
|
if (info.wanted && info.known && info.known->isValid())
|
||||||
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
|
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
|
||||||
}
|
}
|
||||||
|
|
||||||
// If we requested all the outputs via the empty set, we are always fine.
|
// If we requested all the outputs via the empty set, we are always fine.
|
||||||
// If we requested specific elements, the loop above removes all the valid
|
// If we requested specific elements, the loop above removes all the valid
|
||||||
// ones, so any that are left must be invalid.
|
// ones, so any that are left must be invalid.
|
||||||
|
@ -1343,7 +1373,6 @@ void DerivationGoal::done(
|
||||||
if (ex)
|
if (ex)
|
||||||
// FIXME: strip: "error: "
|
// FIXME: strip: "error: "
|
||||||
buildResult.errorMsg = ex->what();
|
buildResult.errorMsg = ex->what();
|
||||||
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
|
|
||||||
if (buildResult.status == BuildResult::TimedOut)
|
if (buildResult.status == BuildResult::TimedOut)
|
||||||
worker.timedOut = true;
|
worker.timedOut = true;
|
||||||
if (buildResult.status == BuildResult::PermanentFailure)
|
if (buildResult.status == BuildResult::PermanentFailure)
|
||||||
|
@ -1370,7 +1399,21 @@ void DerivationGoal::done(
|
||||||
fs.open(traceBuiltOutputsFile, std::fstream::out);
|
fs.open(traceBuiltOutputsFile, std::fstream::out);
|
||||||
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
|
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
|
||||||
|
{
|
||||||
|
Goal::waiteeDone(waitee, result);
|
||||||
|
|
||||||
|
if (waitee->buildResult.success())
|
||||||
|
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
|
||||||
|
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
|
||||||
|
inputDrvOutputs.insert_or_assign(
|
||||||
|
{ bfd->drvPath, output.outputName },
|
||||||
|
realisation.outPath);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
|
||||||
them. */
|
them. */
|
||||||
StringSet wantedOutputs;
|
StringSet wantedOutputs;
|
||||||
|
|
||||||
|
/* Mapping from input derivations + output names to actual store
|
||||||
|
paths. This is filled in by waiteeDone() as each dependency
|
||||||
|
finishes, before inputsRealised() is reached, */
|
||||||
|
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
||||||
|
|
||||||
/* Whether additional wanted outputs have been added. */
|
/* Whether additional wanted outputs have been added. */
|
||||||
bool needRestart = false;
|
bool needRestart = false;
|
||||||
|
|
||||||
|
@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
|
||||||
DrvOutputs builtOutputs = {},
|
DrvOutputs builtOutputs = {},
|
||||||
std::optional<Error> ex = {});
|
std::optional<Error> ex = {});
|
||||||
|
|
||||||
|
void waiteeDone(GoalPtr waitee, ExitCode result) override;
|
||||||
|
|
||||||
StorePathSet exportReferences(const StorePathSet & storePaths);
|
StorePathSet exportReferences(const StorePathSet & storePaths);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
|
||||||
else if (settings.sandboxMode == smDisabled)
|
else if (settings.sandboxMode == smDisabled)
|
||||||
useChroot = false;
|
useChroot = false;
|
||||||
else if (settings.sandboxMode == smRelaxed)
|
else if (settings.sandboxMode == smRelaxed)
|
||||||
useChroot = !(derivationType.isImpure()) && !noChroot;
|
useChroot = derivationType.isSandboxed() && !noChroot;
|
||||||
}
|
}
|
||||||
|
|
||||||
auto & localStore = getLocalStore();
|
auto & localStore = getLocalStore();
|
||||||
|
@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
|
||||||
"nogroup:x:65534:\n", sandboxGid()));
|
"nogroup:x:65534:\n", sandboxGid()));
|
||||||
|
|
||||||
/* Create /etc/hosts with localhost entry. */
|
/* Create /etc/hosts with localhost entry. */
|
||||||
if (!(derivationType.isImpure()))
|
if (derivationType.isSandboxed())
|
||||||
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
|
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
|
||||||
|
|
||||||
/* Make the closure of the inputs available in the chroot,
|
/* Make the closure of the inputs available in the chroot,
|
||||||
|
@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
|
||||||
us.
|
us.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
if (!(derivationType.isImpure()))
|
if (derivationType.isSandboxed())
|
||||||
privateNetwork = true;
|
privateNetwork = true;
|
||||||
|
|
||||||
userNamespaceSync.create();
|
userNamespaceSync.create();
|
||||||
|
@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
|
||||||
to the builder is generally impure, but the output of
|
to the builder is generally impure, but the output of
|
||||||
fixed-output derivations is by definition pure (since we
|
fixed-output derivations is by definition pure (since we
|
||||||
already know the cryptographic hash of the output). */
|
already know the cryptographic hash of the output). */
|
||||||
if (derivationType.isImpure()) {
|
if (!derivationType.isSandboxed()) {
|
||||||
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
|
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
|
||||||
env[i] = getEnv(i).value_or("");
|
env[i] = getEnv(i).value_or("");
|
||||||
}
|
}
|
||||||
|
@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild()
|
||||||
/* Fixed-output derivations typically need to access the
|
/* Fixed-output derivations typically need to access the
|
||||||
network, so give them access to /etc/resolv.conf and so
|
network, so give them access to /etc/resolv.conf and so
|
||||||
on. */
|
on. */
|
||||||
if (derivationType.isImpure()) {
|
if (!derivationType.isSandboxed()) {
|
||||||
// Only use nss functions to resolve hosts and
|
// Only use nss functions to resolve hosts and
|
||||||
// services. Don’t use it for anything else that may
|
// services. Don’t use it for anything else that may
|
||||||
// be configured for this system. This limits the
|
// be configured for this system. This limits the
|
||||||
|
@ -1918,7 +1918,7 @@ void LocalDerivationGoal::runChild()
|
||||||
|
|
||||||
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
|
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
|
||||||
|
|
||||||
if (derivationType.isImpure())
|
if (!derivationType.isSandboxed())
|
||||||
sandboxProfile += "(import \"sandbox-network.sb\")\n";
|
sandboxProfile += "(import \"sandbox-network.sb\")\n";
|
||||||
|
|
||||||
/* Add the output paths we'll use at build-time to the chroot */
|
/* Add the output paths we'll use at build-time to the chroot */
|
||||||
|
@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
assert(false);
|
assert(false);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
[&](const DerivationOutput::Impure & doi) {
|
||||||
|
return newInfoFromCA(DerivationOutput::CAFloating {
|
||||||
|
.method = doi.method,
|
||||||
|
.hashType = doi.hashType,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
}, output.raw());
|
}, output.raw());
|
||||||
|
|
||||||
/* FIXME: set proper permissions in restorePath() so
|
/* FIXME: set proper permissions in restorePath() so
|
||||||
|
@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
|
||||||
},
|
},
|
||||||
.outPath = newInfo.path
|
.outPath = newInfo.path
|
||||||
};
|
};
|
||||||
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
|
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
|
||||||
|
&& drv->type().isPure())
|
||||||
|
{
|
||||||
signRealisation(thisRealisation);
|
signRealisation(thisRealisation);
|
||||||
worker.store.registerDrvOutput(thisRealisation);
|
worker.store.registerDrvOutput(thisRealisation);
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
|
||||||
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
|
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
},
|
},
|
||||||
|
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
|
||||||
|
return std::nullopt;
|
||||||
|
},
|
||||||
}, raw());
|
}, raw());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
|
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
|
||||||
|
{
|
||||||
return store.makeFixedOutputPath(
|
return store.makeFixedOutputPath(
|
||||||
hash.method, hash.hash,
|
hash.method, hash.hash,
|
||||||
outputPathName(drvName, outputName));
|
outputPathName(drvName, outputName));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool DerivationType::isCA() const {
|
bool DerivationType::isCA() const
|
||||||
|
{
|
||||||
/* Normally we do the full `std::visit` to make sure we have
|
/* Normally we do the full `std::visit` to make sure we have
|
||||||
exhaustively handled all variants, but so long as there is a
|
exhaustively handled all variants, but so long as there is a
|
||||||
variant called `ContentAddressed`, it must be the only one for
|
variant called `ContentAddressed`, it must be the only one for
|
||||||
which `isCA` is true for this to make sense!. */
|
which `isCA` is true for this to make sense!. */
|
||||||
return std::holds_alternative<ContentAddressed>(raw());
|
return std::visit(overloaded {
|
||||||
|
[](const InputAddressed & ia) {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
[](const ContentAddressed & ca) {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
[](const Impure &) {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
}, raw());
|
||||||
}
|
}
|
||||||
|
|
||||||
bool DerivationType::isFixed() const {
|
bool DerivationType::isFixed() const
|
||||||
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[](const InputAddressed & ia) {
|
[](const InputAddressed & ia) {
|
||||||
return false;
|
return false;
|
||||||
|
@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
|
||||||
[](const ContentAddressed & ca) {
|
[](const ContentAddressed & ca) {
|
||||||
return ca.fixed;
|
return ca.fixed;
|
||||||
},
|
},
|
||||||
|
[](const Impure &) {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
}, raw());
|
}, raw());
|
||||||
}
|
}
|
||||||
|
|
||||||
bool DerivationType::hasKnownOutputPaths() const {
|
bool DerivationType::hasKnownOutputPaths() const
|
||||||
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[](const InputAddressed & ia) {
|
[](const InputAddressed & ia) {
|
||||||
return !ia.deferred;
|
return !ia.deferred;
|
||||||
|
@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const {
|
||||||
[](const ContentAddressed & ca) {
|
[](const ContentAddressed & ca) {
|
||||||
return ca.fixed;
|
return ca.fixed;
|
||||||
},
|
},
|
||||||
|
[](const Impure &) {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
}, raw());
|
}, raw());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool DerivationType::isImpure() const {
|
bool DerivationType::isSandboxed() const
|
||||||
|
{
|
||||||
return std::visit(overloaded {
|
return std::visit(overloaded {
|
||||||
[](const InputAddressed & ia) {
|
[](const InputAddressed & ia) {
|
||||||
return false;
|
return true;
|
||||||
},
|
},
|
||||||
[](const ContentAddressed & ca) {
|
[](const ContentAddressed & ca) {
|
||||||
return !ca.pure;
|
return ca.sandboxed;
|
||||||
|
},
|
||||||
|
[](const Impure &) {
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
}, raw());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
bool DerivationType::isPure() const
|
||||||
|
{
|
||||||
|
return std::visit(overloaded {
|
||||||
|
[](const InputAddressed & ia) {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
[](const ContentAddressed & ca) {
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
[](const Impure &) {
|
||||||
|
return false;
|
||||||
},
|
},
|
||||||
}, raw());
|
}, raw());
|
||||||
}
|
}
|
||||||
|
@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
|
||||||
hashAlgo = hashAlgo.substr(2);
|
hashAlgo = hashAlgo.substr(2);
|
||||||
}
|
}
|
||||||
const auto hashType = parseHashType(hashAlgo);
|
const auto hashType = parseHashType(hashAlgo);
|
||||||
if (hash != "") {
|
if (hash == "impure") {
|
||||||
|
settings.requireExperimentalFeature(Xp::ImpureDerivations);
|
||||||
|
assert(pathS == "");
|
||||||
|
return DerivationOutput::Impure {
|
||||||
|
.method = std::move(method),
|
||||||
|
.hashType = std::move(hashType),
|
||||||
|
};
|
||||||
|
} else if (hash != "") {
|
||||||
validatePath(pathS);
|
validatePath(pathS);
|
||||||
return DerivationOutput::CAFixed {
|
return DerivationOutput::CAFixed {
|
||||||
.hash = FixedOutputHash {
|
.hash = FixedOutputHash {
|
||||||
|
@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
|
||||||
s += ','; printUnquotedString(s, "");
|
s += ','; printUnquotedString(s, "");
|
||||||
s += ','; printUnquotedString(s, "");
|
s += ','; printUnquotedString(s, "");
|
||||||
s += ','; printUnquotedString(s, "");
|
s += ','; printUnquotedString(s, "");
|
||||||
|
},
|
||||||
|
[&](const DerivationOutputImpure & doi) {
|
||||||
|
// FIXME
|
||||||
|
s += ','; printUnquotedString(s, "");
|
||||||
|
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
|
||||||
|
s += ','; printUnquotedString(s, "impure");
|
||||||
}
|
}
|
||||||
}, i.second.raw());
|
}, i.second.raw());
|
||||||
s += ')';
|
s += ')';
|
||||||
|
@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
|
||||||
|
|
||||||
DerivationType BasicDerivation::type() const
|
DerivationType BasicDerivation::type() const
|
||||||
{
|
{
|
||||||
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
|
std::set<std::string_view>
|
||||||
|
inputAddressedOutputs,
|
||||||
|
fixedCAOutputs,
|
||||||
|
floatingCAOutputs,
|
||||||
|
deferredIAOutputs,
|
||||||
|
impureOutputs;
|
||||||
std::optional<HashType> floatingHashType;
|
std::optional<HashType> floatingHashType;
|
||||||
|
|
||||||
for (auto & i : outputs) {
|
for (auto & i : outputs) {
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const DerivationOutput::InputAddressed &) {
|
[&](const DerivationOutput::InputAddressed &) {
|
||||||
|
@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const
|
||||||
floatingHashType = dof.hashType;
|
floatingHashType = dof.hashType;
|
||||||
} else {
|
} else {
|
||||||
if (*floatingHashType != dof.hashType)
|
if (*floatingHashType != dof.hashType)
|
||||||
throw Error("All floating outputs must use the same hash type");
|
throw Error("all floating outputs must use the same hash type");
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
[&](const DerivationOutput::Deferred &) {
|
[&](const DerivationOutput::Deferred &) {
|
||||||
deferredIAOutputs.insert(i.first);
|
deferredIAOutputs.insert(i.first);
|
||||||
},
|
},
|
||||||
|
[&](const DerivationOutput::Impure &) {
|
||||||
|
impureOutputs.insert(i.first);
|
||||||
|
},
|
||||||
}, i.second.raw());
|
}, i.second.raw());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
if (inputAddressedOutputs.empty()
|
||||||
throw Error("Must have at least one output");
|
&& fixedCAOutputs.empty()
|
||||||
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
&& floatingCAOutputs.empty()
|
||||||
|
&& deferredIAOutputs.empty()
|
||||||
|
&& impureOutputs.empty())
|
||||||
|
throw Error("must have at least one output");
|
||||||
|
|
||||||
|
if (!inputAddressedOutputs.empty()
|
||||||
|
&& fixedCAOutputs.empty()
|
||||||
|
&& floatingCAOutputs.empty()
|
||||||
|
&& deferredIAOutputs.empty()
|
||||||
|
&& impureOutputs.empty())
|
||||||
return DerivationType::InputAddressed {
|
return DerivationType::InputAddressed {
|
||||||
.deferred = false,
|
.deferred = false,
|
||||||
};
|
};
|
||||||
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
|
||||||
|
if (inputAddressedOutputs.empty()
|
||||||
|
&& !fixedCAOutputs.empty()
|
||||||
|
&& floatingCAOutputs.empty()
|
||||||
|
&& deferredIAOutputs.empty()
|
||||||
|
&& impureOutputs.empty())
|
||||||
|
{
|
||||||
if (fixedCAOutputs.size() > 1)
|
if (fixedCAOutputs.size() > 1)
|
||||||
// FIXME: Experimental feature?
|
// FIXME: Experimental feature?
|
||||||
throw Error("Only one fixed output is allowed for now");
|
throw Error("only one fixed output is allowed for now");
|
||||||
if (*fixedCAOutputs.begin() != "out")
|
if (*fixedCAOutputs.begin() != "out")
|
||||||
throw Error("Single fixed output must be named \"out\"");
|
throw Error("single fixed output must be named \"out\"");
|
||||||
return DerivationType::ContentAddressed {
|
return DerivationType::ContentAddressed {
|
||||||
.pure = false,
|
.sandboxed = false,
|
||||||
.fixed = true,
|
.fixed = true,
|
||||||
};
|
};
|
||||||
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
|
}
|
||||||
|
|
||||||
|
if (inputAddressedOutputs.empty()
|
||||||
|
&& fixedCAOutputs.empty()
|
||||||
|
&& !floatingCAOutputs.empty()
|
||||||
|
&& deferredIAOutputs.empty()
|
||||||
|
&& impureOutputs.empty())
|
||||||
return DerivationType::ContentAddressed {
|
return DerivationType::ContentAddressed {
|
||||||
.pure = true,
|
.sandboxed = true,
|
||||||
.fixed = false,
|
.fixed = false,
|
||||||
};
|
};
|
||||||
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
|
|
||||||
|
if (inputAddressedOutputs.empty()
|
||||||
|
&& fixedCAOutputs.empty()
|
||||||
|
&& floatingCAOutputs.empty()
|
||||||
|
&& !deferredIAOutputs.empty()
|
||||||
|
&& impureOutputs.empty())
|
||||||
return DerivationType::InputAddressed {
|
return DerivationType::InputAddressed {
|
||||||
.deferred = true,
|
.deferred = true,
|
||||||
};
|
};
|
||||||
} else {
|
|
||||||
throw Error("Can't mix derivation output types");
|
if (inputAddressedOutputs.empty()
|
||||||
}
|
&& fixedCAOutputs.empty()
|
||||||
|
&& floatingCAOutputs.empty()
|
||||||
|
&& deferredIAOutputs.empty()
|
||||||
|
&& !impureOutputs.empty())
|
||||||
|
return DerivationType::Impure { };
|
||||||
|
|
||||||
|
throw Error("can't mix derivation output types");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -474,7 +571,7 @@ Sync<DrvHashes> drvHashes;
|
||||||
/* Look up the derivation by value and memoize the
|
/* Look up the derivation by value and memoize the
|
||||||
`hashDerivationModulo` call.
|
`hashDerivationModulo` call.
|
||||||
*/
|
*/
|
||||||
static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
|
static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
auto hashes = drvHashes.lock();
|
auto hashes = drvHashes.lock();
|
||||||
|
@ -509,7 +606,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
|
||||||
don't leak the provenance of fixed outputs, reducing pointless cache
|
don't leak the provenance of fixed outputs, reducing pointless cache
|
||||||
misses as the build itself won't know this.
|
misses as the build itself won't know this.
|
||||||
*/
|
*/
|
||||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
|
||||||
{
|
{
|
||||||
auto type = drv.type();
|
auto type = drv.type();
|
||||||
|
|
||||||
|
@ -524,7 +621,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
||||||
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
+ store.printStorePath(dof.path(store, drv.name, i.first)));
|
||||||
outputHashes.insert_or_assign(i.first, std::move(hash));
|
outputHashes.insert_or_assign(i.first, std::move(hash));
|
||||||
}
|
}
|
||||||
return outputHashes;
|
return DrvHash {
|
||||||
|
.hashes = outputHashes,
|
||||||
|
.kind = DrvHash::Kind::Regular,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!type.isPure()) {
|
||||||
|
std::map<std::string, Hash> outputHashes;
|
||||||
|
for (const auto & [outputName, _] : drv.outputs)
|
||||||
|
outputHashes.insert_or_assign(outputName, impureOutputHash);
|
||||||
|
return DrvHash {
|
||||||
|
.hashes = outputHashes,
|
||||||
|
.kind = DrvHash::Kind::Deferred,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
auto kind = std::visit(overloaded {
|
auto kind = std::visit(overloaded {
|
||||||
|
@ -538,67 +648,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
|
||||||
? DrvHash::Kind::Regular
|
? DrvHash::Kind::Regular
|
||||||
: DrvHash::Kind::Deferred;
|
: DrvHash::Kind::Deferred;
|
||||||
},
|
},
|
||||||
|
[](const DerivationType::Impure &) -> DrvHash::Kind {
|
||||||
|
assert(false);
|
||||||
|
}
|
||||||
}, drv.type().raw());
|
}, drv.type().raw());
|
||||||
|
|
||||||
/* For other derivations, replace the inputs paths with recursive
|
|
||||||
calls to this function. */
|
|
||||||
std::map<std::string, StringSet> inputs2;
|
std::map<std::string, StringSet> inputs2;
|
||||||
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
|
for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
|
||||||
// Avoid lambda capture restriction with standard / Clang
|
// Avoid lambda capture restriction with standard / Clang
|
||||||
auto & inputOutputs = inputOutputs0;
|
auto & inputOutputs = inputOutputs0;
|
||||||
const auto & res = pathDerivationModulo(store, drvPath);
|
const auto & res = pathDerivationModulo(store, drvPath);
|
||||||
std::visit(overloaded {
|
if (res.kind == DrvHash::Kind::Deferred)
|
||||||
// Regular non-CA derivation, replace derivation
|
kind = DrvHash::Kind::Deferred;
|
||||||
[&](const DrvHash & drvHash) {
|
for (auto & outputName : inputOutputs) {
|
||||||
kind |= drvHash.kind;
|
const auto h = res.hashes.at(outputName);
|
||||||
inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
|
inputs2[h.to_string(Base16, false)].insert(outputName);
|
||||||
},
|
|
||||||
// CA derivation's output hashes
|
|
||||||
[&](const CaOutputHashes & outputHashes) {
|
|
||||||
std::set<std::string> justOut = { "out" };
|
|
||||||
for (auto & output : inputOutputs) {
|
|
||||||
/* Put each one in with a single "out" output.. */
|
|
||||||
const auto h = outputHashes.at(output);
|
|
||||||
inputs2.insert_or_assign(
|
|
||||||
h.to_string(Base16, false),
|
|
||||||
justOut);
|
|
||||||
}
|
}
|
||||||
},
|
|
||||||
}, res.raw());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
|
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
|
||||||
|
|
||||||
return DrvHash { .hash = hash, .kind = kind };
|
std::map<std::string, Hash> outputHashes;
|
||||||
|
for (const auto & [outputName, _] : drv.outputs) {
|
||||||
|
outputHashes.insert_or_assign(outputName, hash);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return DrvHash {
|
||||||
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
|
.hashes = outputHashes,
|
||||||
{
|
.kind = kind,
|
||||||
switch (other) {
|
};
|
||||||
case DrvHash::Kind::Regular:
|
|
||||||
break;
|
|
||||||
case DrvHash::Kind::Deferred:
|
|
||||||
self = other;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
|
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
|
||||||
{
|
{
|
||||||
std::map<std::string, Hash> res;
|
return hashDerivationModulo(store, drv, true).hashes;
|
||||||
std::visit(overloaded {
|
|
||||||
[&](const DrvHash & drvHash) {
|
|
||||||
for (auto & outputName : drv.outputNames()) {
|
|
||||||
res.insert({outputName, drvHash.hash});
|
|
||||||
}
|
|
||||||
},
|
|
||||||
[&](const CaOutputHashes & outputHashes) {
|
|
||||||
res = outputHashes;
|
|
||||||
},
|
|
||||||
}, hashDerivationModulo(store, drv, true).raw());
|
|
||||||
return res;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -625,7 +709,8 @@ StringSet BasicDerivation::outputNames() const
|
||||||
return names;
|
return names;
|
||||||
}
|
}
|
||||||
|
|
||||||
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
|
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
|
||||||
|
{
|
||||||
DerivationOutputsAndOptPaths outsAndOptPaths;
|
DerivationOutputsAndOptPaths outsAndOptPaths;
|
||||||
for (auto output : outputs)
|
for (auto output : outputs)
|
||||||
outsAndOptPaths.insert(std::make_pair(
|
outsAndOptPaths.insert(std::make_pair(
|
||||||
|
@ -636,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
|
||||||
return outsAndOptPaths;
|
return outsAndOptPaths;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
|
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
|
||||||
|
{
|
||||||
auto nameWithSuffix = drvPath.name();
|
auto nameWithSuffix = drvPath.name();
|
||||||
constexpr std::string_view extension = ".drv";
|
constexpr std::string_view extension = ".drv";
|
||||||
assert(hasSuffix(nameWithSuffix, extension));
|
assert(hasSuffix(nameWithSuffix, extension));
|
||||||
|
@ -698,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
||||||
<< ""
|
<< ""
|
||||||
<< "";
|
<< "";
|
||||||
},
|
},
|
||||||
|
[&](const DerivationOutput::Impure & doi) {
|
||||||
|
out << ""
|
||||||
|
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
|
||||||
|
<< "impure";
|
||||||
|
},
|
||||||
}, i.second.raw());
|
}, i.second.raw());
|
||||||
}
|
}
|
||||||
worker_proto::write(store, out, drv.inputSrcs);
|
worker_proto::write(store, out, drv.inputSrcs);
|
||||||
|
@ -723,10 +814,8 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
|
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
|
||||||
|
{
|
||||||
debug("Rewriting the derivation");
|
|
||||||
|
|
||||||
for (auto & rewrite : rewrites) {
|
for (auto & rewrite : rewrites) {
|
||||||
debug("rewriting %s as %s", rewrite.first, rewrite.second);
|
debug("rewriting %s as %s", rewrite.first, rewrite.second);
|
||||||
}
|
}
|
||||||
|
@ -747,7 +836,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
|
||||||
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
|
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
|
||||||
for (auto & [outputName, output] : drv.outputs) {
|
for (auto & [outputName, output] : drv.outputs) {
|
||||||
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
|
if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
|
||||||
auto & h = hashModulo.requireNoFixedNonDeferred();
|
auto & h = hashModulo.hashes.at(outputName);
|
||||||
auto outPath = store.makeOutputPath(outputName, h, drv.name);
|
auto outPath = store.makeOutputPath(outputName, h, drv.name);
|
||||||
drv.env[outputName] = store.printStorePath(outPath);
|
drv.env[outputName] = store.printStorePath(outPath);
|
||||||
output = DerivationOutput::InputAddressed {
|
output = DerivationOutput::InputAddressed {
|
||||||
|
@ -758,55 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
|
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
|
||||||
auto * drvHashOpt = std::get_if<DrvHash>(&raw());
|
|
||||||
assert(drvHashOpt);
|
|
||||||
assert(drvHashOpt->kind == DrvHash::Kind::Regular);
|
|
||||||
return drvHashOpt->hash;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool tryResolveInput(
|
|
||||||
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
|
|
||||||
const StorePath & inputDrv, const StringSet & inputOutputs)
|
|
||||||
{
|
{
|
||||||
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
|
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
|
||||||
|
|
||||||
auto getOutput = [&](const std::string & outputName) {
|
for (auto & input : inputDrvs)
|
||||||
auto & actualPathOpt = inputDrvOutputs.at(outputName);
|
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
|
||||||
if (!actualPathOpt)
|
if (outputPath)
|
||||||
warn("output %s of input %s missing, aborting the resolving",
|
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
|
||||||
outputName,
|
|
||||||
store.printStorePath(inputDrv)
|
|
||||||
);
|
|
||||||
return actualPathOpt;
|
|
||||||
};
|
|
||||||
|
|
||||||
for (auto & outputName : inputOutputs) {
|
return tryResolve(store, inputDrvOutputs);
|
||||||
auto actualPathOpt = getOutput(outputName);
|
|
||||||
if (!actualPathOpt) return false;
|
|
||||||
auto actualPath = *actualPathOpt;
|
|
||||||
inputRewrites.emplace(
|
|
||||||
downstreamPlaceholder(store, inputDrv, outputName),
|
|
||||||
store.printStorePath(actualPath));
|
|
||||||
inputSrcs.insert(std::move(actualPath));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
std::optional<BasicDerivation> Derivation::tryResolve(
|
||||||
}
|
Store & store,
|
||||||
|
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
|
||||||
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
|
{
|
||||||
BasicDerivation resolved { *this };
|
BasicDerivation resolved { *this };
|
||||||
|
|
||||||
// Input paths that we'll want to rewrite in the derivation
|
// Input paths that we'll want to rewrite in the derivation
|
||||||
StringMap inputRewrites;
|
StringMap inputRewrites;
|
||||||
|
|
||||||
for (auto & [inputDrv, inputOutputs] : inputDrvs)
|
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
|
||||||
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
|
for (auto & outputName : inputOutputs) {
|
||||||
return std::nullopt;
|
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
|
||||||
|
inputRewrites.emplace(
|
||||||
|
downstreamPlaceholder(store, inputDrv, outputName),
|
||||||
|
store.printStorePath(*actualPath));
|
||||||
|
resolved.inputSrcs.insert(*actualPath);
|
||||||
|
} else {
|
||||||
|
warn("output '%s' of input '%s' missing, aborting the resolving",
|
||||||
|
outputName,
|
||||||
|
store.printStorePath(inputDrv));
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
rewriteDerivation(store, resolved, inputRewrites);
|
rewriteDerivation(store, resolved, inputRewrites);
|
||||||
|
|
||||||
return resolved;
|
return resolved;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const Hash impureOutputHash = hashString(htSHA256, "impure");
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
|
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
|
||||||
* known atm
|
* known yet.
|
||||||
*/
|
*/
|
||||||
struct DerivationOutputDeferred {};
|
struct DerivationOutputDeferred {};
|
||||||
|
|
||||||
|
/* Impure output which is moved to a content-addressed location (like
|
||||||
|
CAFloating) but isn't registered as a realization.
|
||||||
|
*/
|
||||||
|
struct DerivationOutputImpure
|
||||||
|
{
|
||||||
|
/* information used for expected hash computation */
|
||||||
|
FileIngestionMethod method;
|
||||||
|
HashType hashType;
|
||||||
|
};
|
||||||
|
|
||||||
typedef std::variant<
|
typedef std::variant<
|
||||||
DerivationOutputInputAddressed,
|
DerivationOutputInputAddressed,
|
||||||
DerivationOutputCAFixed,
|
DerivationOutputCAFixed,
|
||||||
DerivationOutputCAFloating,
|
DerivationOutputCAFloating,
|
||||||
DerivationOutputDeferred
|
DerivationOutputDeferred,
|
||||||
|
DerivationOutputImpure
|
||||||
> _DerivationOutputRaw;
|
> _DerivationOutputRaw;
|
||||||
|
|
||||||
struct DerivationOutput : _DerivationOutputRaw
|
struct DerivationOutput : _DerivationOutputRaw
|
||||||
|
@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
|
||||||
using CAFixed = DerivationOutputCAFixed;
|
using CAFixed = DerivationOutputCAFixed;
|
||||||
using CAFloating = DerivationOutputCAFloating;
|
using CAFloating = DerivationOutputCAFloating;
|
||||||
using Deferred = DerivationOutputDeferred;
|
using Deferred = DerivationOutputDeferred;
|
||||||
|
using Impure = DerivationOutputImpure;
|
||||||
|
|
||||||
/* Note, when you use this function you should make sure that you're passing
|
/* Note, when you use this function you should make sure that you're passing
|
||||||
the right derivation name. When in doubt, you should use the safer
|
the right derivation name. When in doubt, you should use the safer
|
||||||
|
@ -90,13 +102,17 @@ struct DerivationType_InputAddressed {
|
||||||
};
|
};
|
||||||
|
|
||||||
struct DerivationType_ContentAddressed {
|
struct DerivationType_ContentAddressed {
|
||||||
bool pure;
|
bool sandboxed;
|
||||||
bool fixed;
|
bool fixed;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct DerivationType_Impure {
|
||||||
|
};
|
||||||
|
|
||||||
typedef std::variant<
|
typedef std::variant<
|
||||||
DerivationType_InputAddressed,
|
DerivationType_InputAddressed,
|
||||||
DerivationType_ContentAddressed
|
DerivationType_ContentAddressed,
|
||||||
|
DerivationType_Impure
|
||||||
> _DerivationTypeRaw;
|
> _DerivationTypeRaw;
|
||||||
|
|
||||||
struct DerivationType : _DerivationTypeRaw {
|
struct DerivationType : _DerivationTypeRaw {
|
||||||
|
@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
|
||||||
using Raw::Raw;
|
using Raw::Raw;
|
||||||
using InputAddressed = DerivationType_InputAddressed;
|
using InputAddressed = DerivationType_InputAddressed;
|
||||||
using ContentAddressed = DerivationType_ContentAddressed;
|
using ContentAddressed = DerivationType_ContentAddressed;
|
||||||
|
using Impure = DerivationType_Impure;
|
||||||
|
|
||||||
/* Do the outputs of the derivation have paths calculated from their content,
|
/* Do the outputs of the derivation have paths calculated from their content,
|
||||||
or from the derivation itself? */
|
or from the derivation itself? */
|
||||||
|
@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw {
|
||||||
non-CA derivations. */
|
non-CA derivations. */
|
||||||
bool isFixed() const;
|
bool isFixed() const;
|
||||||
|
|
||||||
/* Is the derivation impure and needs to access non-deterministic resources, or
|
/* Whether the derivation is fully sandboxed. If false, the
|
||||||
pure and can be sandboxed? Note that whether or not we actually sandbox the
|
sandbox is opened up, e.g. the derivation has access to the
|
||||||
derivation is controlled separately. Never true for non-CA derivations. */
|
network. Note that whether or not we actually sandbox the
|
||||||
bool isImpure() const;
|
derivation is controlled separately. Always true for non-CA
|
||||||
|
derivations. */
|
||||||
|
bool isSandboxed() const;
|
||||||
|
|
||||||
|
/* Whether the derivation is expected to produce the same result
|
||||||
|
every time, and therefore it only needs to be built once. This
|
||||||
|
is only false for derivations that have the attribute '__impure
|
||||||
|
= true'. */
|
||||||
|
bool isPure() const;
|
||||||
|
|
||||||
/* Does the derivation knows its own output paths?
|
/* Does the derivation knows its own output paths?
|
||||||
Only true when there's no floating-ca derivation involved in the
|
Only true when there's no floating-ca derivation involved in the
|
||||||
|
@ -173,7 +197,14 @@ struct Derivation : BasicDerivation
|
||||||
added directly to input sources.
|
added directly to input sources.
|
||||||
|
|
||||||
2. Input placeholders are replaced with realized input store paths. */
|
2. Input placeholders are replaced with realized input store paths. */
|
||||||
std::optional<BasicDerivation> tryResolve(Store & store);
|
std::optional<BasicDerivation> tryResolve(Store & store) const;
|
||||||
|
|
||||||
|
/* Like the above, but instead of querying the Nix database for
|
||||||
|
realisations, uses a given mapping from input derivation paths
|
||||||
|
+ output names to actual output store paths. */
|
||||||
|
std::optional<BasicDerivation> tryResolve(
|
||||||
|
Store & store,
|
||||||
|
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
|
||||||
|
|
||||||
Derivation() = default;
|
Derivation() = default;
|
||||||
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
|
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
|
||||||
|
@ -202,12 +233,14 @@ bool isDerivation(const std::string & fileName);
|
||||||
the output name is "out". */
|
the output name is "out". */
|
||||||
std::string outputPathName(std::string_view drvName, std::string_view outputName);
|
std::string outputPathName(std::string_view drvName, std::string_view outputName);
|
||||||
|
|
||||||
// known CA drv's output hashes, current just for fixed-output derivations
|
|
||||||
// whose output hashes are always known since they are fixed up-front.
|
|
||||||
typedef std::map<std::string, Hash> CaOutputHashes;
|
|
||||||
|
|
||||||
|
// The hashes modulo of a derivation.
|
||||||
|
//
|
||||||
|
// Each output is given a hash, although in practice only the content-addressed
|
||||||
|
// derivations (fixed-output or not) will have a different hash for each
|
||||||
|
// output.
|
||||||
struct DrvHash {
|
struct DrvHash {
|
||||||
Hash hash;
|
std::map<std::string, Hash> hashes;
|
||||||
|
|
||||||
enum struct Kind : bool {
|
enum struct Kind : bool {
|
||||||
// Statically determined derivations.
|
// Statically determined derivations.
|
||||||
|
@ -222,28 +255,6 @@ struct DrvHash {
|
||||||
|
|
||||||
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
|
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
|
||||||
|
|
||||||
typedef std::variant<
|
|
||||||
// Regular normalized derivation hash, and whether it was deferred (because
|
|
||||||
// an ancestor derivation is a floating content addressed derivation).
|
|
||||||
DrvHash,
|
|
||||||
// Fixed-output derivation hashes
|
|
||||||
CaOutputHashes
|
|
||||||
> _DrvHashModuloRaw;
|
|
||||||
|
|
||||||
struct DrvHashModulo : _DrvHashModuloRaw {
|
|
||||||
using Raw = _DrvHashModuloRaw;
|
|
||||||
using Raw::Raw;
|
|
||||||
|
|
||||||
/* Get hash, throwing if it is per-output CA hashes or a
|
|
||||||
deferred Drv hash.
|
|
||||||
*/
|
|
||||||
const Hash & requireNoFixedNonDeferred() const;
|
|
||||||
|
|
||||||
inline const Raw & raw() const {
|
|
||||||
return static_cast<const Raw &>(*this);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Returns hashes with the details of fixed-output subderivations
|
/* Returns hashes with the details of fixed-output subderivations
|
||||||
expunged.
|
expunged.
|
||||||
|
|
||||||
|
@ -267,16 +278,18 @@ struct DrvHashModulo : _DrvHashModuloRaw {
|
||||||
ATerm, after subderivations have been likewise expunged from that
|
ATerm, after subderivations have been likewise expunged from that
|
||||||
derivation.
|
derivation.
|
||||||
*/
|
*/
|
||||||
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Return a map associating each output to a hash that uniquely identifies its
|
Return a map associating each output to a hash that uniquely identifies its
|
||||||
derivation (modulo the self-references).
|
derivation (modulo the self-references).
|
||||||
|
|
||||||
|
FIXME: what is the Hash in this map?
|
||||||
*/
|
*/
|
||||||
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
|
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
|
||||||
|
|
||||||
/* Memoisation of hashDerivationModulo(). */
|
/* Memoisation of hashDerivationModulo(). */
|
||||||
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
|
typedef std::map<StorePath, DrvHash> DrvHashes;
|
||||||
|
|
||||||
// FIXME: global, though at least thread-safe.
|
// FIXME: global, though at least thread-safe.
|
||||||
extern Sync<DrvHashes> drvHashes;
|
extern Sync<DrvHashes> drvHashes;
|
||||||
|
@ -306,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName);
|
||||||
dependency which is a CA derivation. */
|
dependency which is a CA derivation. */
|
||||||
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
|
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
|
||||||
|
|
||||||
|
extern const Hash impureOutputHash;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,9 @@ struct DerivedPathOpaque {
|
||||||
nlohmann::json toJSON(ref<Store> store) const;
|
nlohmann::json toJSON(ref<Store> store) const;
|
||||||
std::string to_string(const Store & store) const;
|
std::string to_string(const Store & store) const;
|
||||||
static DerivedPathOpaque parse(const Store & store, std::string_view);
|
static DerivedPathOpaque parse(const Store & store, std::string_view);
|
||||||
|
|
||||||
|
bool operator < (const DerivedPathOpaque & b) const
|
||||||
|
{ return path < b.path; }
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -46,6 +49,9 @@ struct DerivedPathBuilt {
|
||||||
std::string to_string(const Store & store) const;
|
std::string to_string(const Store & store) const;
|
||||||
static DerivedPathBuilt parse(const Store & store, std::string_view);
|
static DerivedPathBuilt parse(const Store & store, std::string_view);
|
||||||
nlohmann::json toJSON(ref<Store> store) const;
|
nlohmann::json toJSON(ref<Store> store) const;
|
||||||
|
|
||||||
|
bool operator < (const DerivedPathBuilt & b) const
|
||||||
|
{ return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); }
|
||||||
};
|
};
|
||||||
|
|
||||||
using _DerivedPathRaw = std::variant<
|
using _DerivedPathRaw = std::variant<
|
||||||
|
|
|
@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
||||||
// combinations that are currently prohibited.
|
// combinations that are currently prohibited.
|
||||||
drv.type();
|
drv.type();
|
||||||
|
|
||||||
std::optional<Hash> h;
|
std::optional<DrvHash> hashesModulo;
|
||||||
for (auto & i : drv.outputs) {
|
for (auto & i : drv.outputs) {
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const DerivationOutput::InputAddressed & doia) {
|
[&](const DerivationOutput::InputAddressed & doia) {
|
||||||
if (!h) {
|
if (!hashesModulo) {
|
||||||
// somewhat expensive so we do lazily
|
// somewhat expensive so we do lazily
|
||||||
auto h0 = hashDerivationModulo(*this, drv, true);
|
hashesModulo = hashDerivationModulo(*this, drv, true);
|
||||||
h = h0.requireNoFixedNonDeferred();
|
|
||||||
}
|
}
|
||||||
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
|
StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName);
|
||||||
if (doia.path != recomputed)
|
if (doia.path != recomputed)
|
||||||
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
|
throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
|
||||||
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
|
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
|
||||||
|
@ -720,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
||||||
[&](const DerivationOutput::Deferred &) {
|
[&](const DerivationOutput::Deferred &) {
|
||||||
/* Nothing to check */
|
/* Nothing to check */
|
||||||
},
|
},
|
||||||
|
[&](const DerivationOutput::Impure &) {
|
||||||
|
/* Nothing to check */
|
||||||
|
},
|
||||||
}, i.second.raw());
|
}, i.second.raw());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -285,7 +285,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
DrvOutput{outputHashes.at(outputName), outputName});
|
DrvOutput{outputHashes.at(outputName), outputName});
|
||||||
if (!thisRealisation)
|
if (!thisRealisation)
|
||||||
throw Error(
|
throw Error(
|
||||||
"output '%s' of derivation '%s' isn’t built", outputName,
|
"output '%s' of derivation '%s' isn't built", outputName,
|
||||||
store.printStorePath(inputDrv));
|
store.printStorePath(inputDrv));
|
||||||
inputRealisations.insert(*thisRealisation);
|
inputRealisations.insert(*thisRealisation);
|
||||||
}
|
}
|
||||||
|
@ -295,4 +295,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
|
|
||||||
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
|
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
|
|
||||||
|
#include <sodium.h>
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
static void checkName(std::string_view path, std::string_view name)
|
static void checkName(std::string_view path, std::string_view name)
|
||||||
|
@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
|
||||||
|
|
||||||
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
|
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
|
||||||
|
|
||||||
|
StorePath StorePath::random(std::string_view name)
|
||||||
|
{
|
||||||
|
Hash hash(htSHA1);
|
||||||
|
randombytes_buf(hash.hash, hash.hashSize);
|
||||||
|
return StorePath(hash, name);
|
||||||
|
}
|
||||||
|
|
||||||
StorePath Store::parseStorePath(std::string_view path) const
|
StorePath Store::parseStorePath(std::string_view path) const
|
||||||
{
|
{
|
||||||
auto p = canonPath(std::string(path));
|
auto p = canonPath(std::string(path));
|
||||||
|
|
|
@ -58,6 +58,8 @@ public:
|
||||||
}
|
}
|
||||||
|
|
||||||
static StorePath dummy;
|
static StorePath dummy;
|
||||||
|
|
||||||
|
static StorePath random(std::string_view name);
|
||||||
};
|
};
|
||||||
|
|
||||||
typedef std::set<StorePath> StorePathSet;
|
typedef std::set<StorePath> StorePathSet;
|
||||||
|
|
|
@ -9,10 +9,9 @@ namespace nix {
|
||||||
|
|
||||||
const std::string nativeSystem = SYSTEM;
|
const std::string nativeSystem = SYSTEM;
|
||||||
|
|
||||||
BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
|
void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
|
||||||
{
|
{
|
||||||
err.traces.push_front(Trace { .pos = e, .hint = hint });
|
err.traces.push_front(Trace { .pos = e, .hint = hint });
|
||||||
return *this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// c++ std::exception descendants must have a 'const char* what()' function.
|
// c++ std::exception descendants must have a 'const char* what()' function.
|
||||||
|
|
|
@ -175,12 +175,12 @@ public:
|
||||||
const ErrorInfo & info() const { calcWhat(); return err; }
|
const ErrorInfo & info() const { calcWhat(); return err; }
|
||||||
|
|
||||||
template<typename... Args>
|
template<typename... Args>
|
||||||
BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
|
void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
|
||||||
{
|
{
|
||||||
return addTrace(e, hintfmt(fs, args...));
|
addTrace(e, hintfmt(fs, args...));
|
||||||
}
|
}
|
||||||
|
|
||||||
BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
|
void addTrace(std::optional<ErrPos> e, hintformat hint);
|
||||||
|
|
||||||
bool hasTrace() const { return !err.traces.empty(); }
|
bool hasTrace() const { return !err.traces.empty(); }
|
||||||
};
|
};
|
||||||
|
|
|
@ -7,6 +7,7 @@ namespace nix {
|
||||||
|
|
||||||
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
|
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
|
||||||
{ Xp::CaDerivations, "ca-derivations" },
|
{ Xp::CaDerivations, "ca-derivations" },
|
||||||
|
{ Xp::ImpureDerivations, "impure-derivations" },
|
||||||
{ Xp::Flakes, "flakes" },
|
{ Xp::Flakes, "flakes" },
|
||||||
{ Xp::NixCommand, "nix-command" },
|
{ Xp::NixCommand, "nix-command" },
|
||||||
{ Xp::RecursiveNix, "recursive-nix" },
|
{ Xp::RecursiveNix, "recursive-nix" },
|
||||||
|
|
|
@ -16,6 +16,7 @@ namespace nix {
|
||||||
enum struct ExperimentalFeature
|
enum struct ExperimentalFeature
|
||||||
{
|
{
|
||||||
CaDerivations,
|
CaDerivations,
|
||||||
|
ImpureDerivations,
|
||||||
Flakes,
|
Flakes,
|
||||||
NixCommand,
|
NixCommand,
|
||||||
RecursiveNix,
|
RecursiveNix,
|
||||||
|
|
|
@ -71,14 +71,12 @@ void clearEnv()
|
||||||
unsetenv(name.first.c_str());
|
unsetenv(name.first.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
void replaceEnv(std::map<std::string, std::string> newEnv)
|
void replaceEnv(const std::map<std::string, std::string> & newEnv)
|
||||||
{
|
{
|
||||||
clearEnv();
|
clearEnv();
|
||||||
for (auto newEnvVar : newEnv)
|
for (auto & newEnvVar : newEnv)
|
||||||
{
|
|
||||||
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
|
setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
|
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
|
||||||
|
|
|
@ -9,7 +9,7 @@ using namespace nix;
|
||||||
|
|
||||||
struct CmdBundle : InstallableCommand
|
struct CmdBundle : InstallableCommand
|
||||||
{
|
{
|
||||||
std::string bundler = "github:matthewbauer/nix-bundle";
|
std::string bundler = "github:NixOS/bundlers";
|
||||||
std::optional<Path> outLink;
|
std::optional<Path> outLink;
|
||||||
|
|
||||||
CmdBundle()
|
CmdBundle()
|
||||||
|
|
|
@ -204,10 +204,10 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
|
||||||
output.second = DerivationOutput::Deferred { };
|
output.second = DerivationOutput::Deferred { };
|
||||||
drv.env[output.first] = "";
|
drv.env[output.first] = "";
|
||||||
}
|
}
|
||||||
auto h0 = hashDerivationModulo(*evalStore, drv, true);
|
auto hashesModulo = hashDerivationModulo(*evalStore, drv, true);
|
||||||
const Hash & h = h0.requireNoFixedNonDeferred();
|
|
||||||
|
|
||||||
for (auto & output : drv.outputs) {
|
for (auto & output : drv.outputs) {
|
||||||
|
Hash h = hashesModulo.hashes.at(output.first);
|
||||||
auto outPath = store->makeOutputPath(output.first, h, drv.name);
|
auto outPath = store->makeOutputPath(output.first, h, drv.name);
|
||||||
output.second = DerivationOutput::InputAddressed {
|
output.second = DerivationOutput::InputAddressed {
|
||||||
.path = outPath,
|
.path = outPath,
|
||||||
|
|
|
@ -62,22 +62,21 @@ struct ProfileElement
|
||||||
return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
|
return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
|
||||||
}
|
}
|
||||||
|
|
||||||
void updateStorePaths(ref<Store> evalStore, ref<Store> store, Installable & installable)
|
void updateStorePaths(
|
||||||
|
ref<Store> evalStore,
|
||||||
|
ref<Store> store,
|
||||||
|
const BuiltPaths & builtPaths)
|
||||||
{
|
{
|
||||||
// FIXME: respect meta.outputsToInstall
|
// FIXME: respect meta.outputsToInstall
|
||||||
storePaths.clear();
|
storePaths.clear();
|
||||||
for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) {
|
for (auto & buildable : builtPaths) {
|
||||||
std::visit(overloaded {
|
std::visit(overloaded {
|
||||||
[&](const BuiltPath::Opaque & bo) {
|
[&](const BuiltPath::Opaque & bo) {
|
||||||
storePaths.insert(bo.path);
|
storePaths.insert(bo.path);
|
||||||
},
|
},
|
||||||
[&](const BuiltPath::Built & bfd) {
|
[&](const BuiltPath::Built & bfd) {
|
||||||
// TODO: Why are we querying if we know the output
|
for (auto & output : bfd.outputs)
|
||||||
// names already? Is it just to figure out what the
|
|
||||||
// default one is?
|
|
||||||
for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
|
|
||||||
storePaths.insert(output.second);
|
storePaths.insert(output.second);
|
||||||
}
|
|
||||||
},
|
},
|
||||||
}, buildable.raw());
|
}, buildable.raw());
|
||||||
}
|
}
|
||||||
|
@ -236,6 +235,16 @@ struct ProfileManifest
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static std::map<Installable *, BuiltPaths>
|
||||||
|
builtPathsPerInstallable(
|
||||||
|
const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
|
||||||
|
{
|
||||||
|
std::map<Installable *, BuiltPaths> res;
|
||||||
|
for (auto & [installable, builtPath] : builtPaths)
|
||||||
|
res[installable.get()].push_back(builtPath);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
|
struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
|
||||||
{
|
{
|
||||||
std::string description() override
|
std::string description() override
|
||||||
|
@ -254,7 +263,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
|
||||||
{
|
{
|
||||||
ProfileManifest manifest(*getEvalState(), *profile);
|
ProfileManifest manifest(*getEvalState(), *profile);
|
||||||
|
|
||||||
auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal);
|
auto builtPaths = builtPathsPerInstallable(
|
||||||
|
Installable::build2(
|
||||||
|
getEvalStore(), store, Realise::Outputs, installables, bmNormal));
|
||||||
|
|
||||||
for (auto & installable : installables) {
|
for (auto & installable : installables) {
|
||||||
ProfileElement element;
|
ProfileElement element;
|
||||||
|
@ -269,7 +280,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
element.updateStorePaths(getEvalStore(), store, *installable);
|
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
|
||||||
|
|
||||||
manifest.elements.push_back(std::move(element));
|
manifest.elements.push_back(std::move(element));
|
||||||
}
|
}
|
||||||
|
@ -457,12 +468,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
|
||||||
warn ("Use 'nix profile list' to see the current profile.");
|
warn ("Use 'nix profile list' to see the current profile.");
|
||||||
}
|
}
|
||||||
|
|
||||||
auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal);
|
auto builtPaths = builtPathsPerInstallable(
|
||||||
|
Installable::build2(
|
||||||
|
getEvalStore(), store, Realise::Outputs, installables, bmNormal));
|
||||||
|
|
||||||
for (size_t i = 0; i < installables.size(); ++i) {
|
for (size_t i = 0; i < installables.size(); ++i) {
|
||||||
auto & installable = installables.at(i);
|
auto & installable = installables.at(i);
|
||||||
auto & element = manifest.elements[indices.at(i)];
|
auto & element = manifest.elements[indices.at(i)];
|
||||||
element.updateStorePaths(getEvalStore(), store, *installable);
|
element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
updateProfile(manifest.build(store));
|
updateProfile(manifest.build(store));
|
||||||
|
|
|
@ -38,9 +38,12 @@ void runProgramInStore(ref<Store> store,
|
||||||
unshare(CLONE_NEWUSER) doesn't work in a multithreaded program
|
unshare(CLONE_NEWUSER) doesn't work in a multithreaded program
|
||||||
(which "nix" is), so we exec() a single-threaded helper program
|
(which "nix" is), so we exec() a single-threaded helper program
|
||||||
(chrootHelper() below) to do the work. */
|
(chrootHelper() below) to do the work. */
|
||||||
auto store2 = store.dynamic_pointer_cast<LocalStore>();
|
auto store2 = store.dynamic_pointer_cast<LocalFSStore>();
|
||||||
|
|
||||||
if (store2 && store->storeDir != store2->getRealStoreDir()) {
|
if (!store2)
|
||||||
|
throw Error("store '%s' is not a local store so it does not support command execution", store->getUri());
|
||||||
|
|
||||||
|
if (store->storeDir != store2->getRealStoreDir()) {
|
||||||
Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
|
Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program };
|
||||||
for (auto & arg : args) helperArgs.push_back(arg);
|
for (auto & arg : args) helperArgs.push_back(arg);
|
||||||
|
|
||||||
|
|
|
@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
|
||||||
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
|
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
|
||||||
},
|
},
|
||||||
[&](const DerivationOutput::Deferred &) {},
|
[&](const DerivationOutput::Deferred &) {},
|
||||||
|
[&](const DerivationOutput::Impure & doi) {
|
||||||
|
outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
|
||||||
|
outputObj.attr("impure", true);
|
||||||
|
},
|
||||||
}, output.raw());
|
}, output.raw());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
touch foo -t 202211111111
|
touch $TEST_ROOT/foo -t 202211111111
|
||||||
# We only check whether 2022-11-1* **:**:** is the last modified date since
|
# We only check whether 2022-11-1* **:**:** is the last modified date since
|
||||||
# `lastModified` is transformed into UTC in `builtins.fetchTarball`.
|
# `lastModified` is transformed into UTC in `builtins.fetchTarball`.
|
||||||
[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]]
|
[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]]
|
||||||
|
|
63
tests/impure-derivations.nix
Normal file
63
tests/impure-derivations.nix
Normal file
|
@ -0,0 +1,63 @@
|
||||||
|
with import ./config.nix;
|
||||||
|
|
||||||
|
rec {
|
||||||
|
|
||||||
|
impure = mkDerivation {
|
||||||
|
name = "impure";
|
||||||
|
outputs = [ "out" "stuff" ];
|
||||||
|
buildCommand =
|
||||||
|
''
|
||||||
|
echo impure
|
||||||
|
x=$(< $TEST_ROOT/counter)
|
||||||
|
mkdir $out $stuff
|
||||||
|
echo $x > $out/n
|
||||||
|
ln -s $out/n $stuff/bla
|
||||||
|
printf $((x + 1)) > $TEST_ROOT/counter
|
||||||
|
'';
|
||||||
|
__impure = true;
|
||||||
|
impureEnvVars = [ "TEST_ROOT" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
impureOnImpure = mkDerivation {
|
||||||
|
name = "impure-on-impure";
|
||||||
|
buildCommand =
|
||||||
|
''
|
||||||
|
echo impure-on-impure
|
||||||
|
x=$(< ${impure}/n)
|
||||||
|
mkdir $out
|
||||||
|
printf X$x > $out/n
|
||||||
|
ln -s ${impure.stuff} $out/symlink
|
||||||
|
ln -s $out $out/self
|
||||||
|
'';
|
||||||
|
__impure = true;
|
||||||
|
};
|
||||||
|
|
||||||
|
# This is not allowed.
|
||||||
|
inputAddressed = mkDerivation {
|
||||||
|
name = "input-addressed";
|
||||||
|
buildCommand =
|
||||||
|
''
|
||||||
|
cat ${impure} > $out
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
contentAddressed = mkDerivation {
|
||||||
|
name = "content-addressed";
|
||||||
|
buildCommand =
|
||||||
|
''
|
||||||
|
echo content-addressed
|
||||||
|
x=$(< ${impureOnImpure}/n)
|
||||||
|
printf ''${x:0:1} > $out
|
||||||
|
'';
|
||||||
|
outputHashMode = "recursive";
|
||||||
|
outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ=";
|
||||||
|
};
|
||||||
|
|
||||||
|
inputAddressedAfterCA = mkDerivation {
|
||||||
|
name = "input-addressed-after-ca";
|
||||||
|
buildCommand =
|
||||||
|
''
|
||||||
|
cat ${contentAddressed} > $out
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
57
tests/impure-derivations.sh
Normal file
57
tests/impure-derivations.sh
Normal file
|
@ -0,0 +1,57 @@
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
requireDaemonNewerThan "2.8pre20220311"
|
||||||
|
|
||||||
|
enableFeatures "ca-derivations ca-references impure-derivations"
|
||||||
|
restartDaemon
|
||||||
|
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
clearStore
|
||||||
|
|
||||||
|
# Basic test of impure derivations: building one a second time should not use the previous result.
|
||||||
|
printf 0 > $TEST_ROOT/counter
|
||||||
|
|
||||||
|
json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
|
||||||
|
path1=$(echo $json | jq -r .[].outputs.out)
|
||||||
|
path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
|
||||||
|
[[ $(< $path1/n) = 0 ]]
|
||||||
|
[[ $(< $path1_stuff/bla) = 0 ]]
|
||||||
|
|
||||||
|
[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
|
||||||
|
|
||||||
|
path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
|
||||||
|
[[ $(< $path2/n) = 1 ]]
|
||||||
|
|
||||||
|
# Test impure derivations that depend on impure derivations.
|
||||||
|
path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
|
||||||
|
[[ $(< $path3/n) = X2 ]]
|
||||||
|
|
||||||
|
path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
|
||||||
|
[[ $(< $path4/n) = X3 ]]
|
||||||
|
|
||||||
|
# Test that (self-)references work.
|
||||||
|
[[ $(< $path4/symlink/bla) = 3 ]]
|
||||||
|
[[ $(< $path4/self/n) = X3 ]]
|
||||||
|
|
||||||
|
# Input-addressed derivations cannot depend on impure derivations directly.
|
||||||
|
(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
|
||||||
|
|
||||||
|
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
|
||||||
|
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
|
||||||
|
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]
|
||||||
|
|
||||||
|
# Fixed-output derivations *can* depend on impure derivations.
|
||||||
|
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
|
||||||
|
[[ $(< $path5) = X ]]
|
||||||
|
[[ $(< $TEST_ROOT/counter) = 5 ]]
|
||||||
|
|
||||||
|
# And they should not be rebuilt.
|
||||||
|
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
|
||||||
|
[[ $(< $path5) = X ]]
|
||||||
|
[[ $(< $TEST_ROOT/counter) = 5 ]]
|
||||||
|
|
||||||
|
# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations.
|
||||||
|
path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out)
|
||||||
|
[[ $(< $path6) = X ]]
|
||||||
|
[[ $(< $TEST_ROOT/counter) = 5 ]]
|
|
@ -98,7 +98,8 @@ nix_tests = \
|
||||||
nix-profile.sh \
|
nix-profile.sh \
|
||||||
suggestions.sh \
|
suggestions.sh \
|
||||||
store-ping.sh \
|
store-ping.sh \
|
||||||
fetchClosure.sh
|
fetchClosure.sh \
|
||||||
|
impure-derivations.sh
|
||||||
|
|
||||||
ifeq ($(HAVE_LIBCPUID), 1)
|
ifeq ($(HAVE_LIBCPUID), 1)
|
||||||
nix_tests += compute-levels.sh
|
nix_tests += compute-levels.sh
|
||||||
|
|
Loading…
Reference in a new issue