lehmanator/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Merge pull request #6227 from NixOS/impure-derivations-ng

Browse source 
Impure derivations
This commit is contained in:
Eelco Dolstra 2022-03-31 19:58:35 +02:00 committed by GitHub
parent 0fe8849914 6377442c98
commit c9a29d0d92
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 562 additions and 169 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
doc/manual/src/release-notes/rl-next.md
View file

@ -14,3 +14,21 @@
This function is only available if you enable the experimental
feature `fetch-closure`.
* New experimental feature: *impure derivations*. These are
derivations that can produce a different result every time they're
built. Here is an example:
```nix
stdenv.mkDerivation {
name = "impure";
__impure = true; # marks this derivation as impure
buildCommand = "date > $out";
}
```
Running `nix build` twice on this expression will build the
derivation twice, producing two different content-addressed store
paths. Like fixed-output derivations, impure derivations have access
to the network. Only fixed-output derivations and impure derivations
can depend on an impure derivation.

1
src/libexpr/eval.cc
View file

@ -436,6 +436,7 @@ EvalState::EvalState(
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed"))
, sImpure(symbols.create("__impure"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))

2
src/libexpr/eval.hh
View file

@ -78,7 +78,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sContentAddressed,
sContentAddressed, sImpure,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,

47
src/libexpr/primops.cc
View file

@ -989,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context;
bool contentAddressed = false;
bool isImpure = false;
std::optional<std::string> outputHash;
std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat;
std::optional<std::string> outputHashAlgo;
std::optional<FileIngestionMethod> ingestionMethod;
StringSet outputs;
outputs.insert("out");
@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
settings.requireExperimentalFeature(Xp::CaDerivations);
}
else if (i->name == state.sImpure) {
isImpure = state.forceBool(*i->value, pos);
if (isImpure)
settings.requireExperimentalFeature(Xp::ImpureDerivations);
}
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
else if (i->name == state.sArgs) {
@ -1183,29 +1190,45 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
.errPos = posDrvName
});
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo.value_or("sha256"));
Hash h = newHashAllowEmpty(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out",
DerivationOutput::CAFixed {
.hash = FixedOutputHash {
.method = ingestionMethod,
.method = method,
.hash = std::move(h),
},
});
}
else if (contentAddressed) {
HashType ht = parseHashType(outputHashAlgo);
else if (contentAddressed || isImpure) {
if (contentAddressed && isImpure)
throw EvalError({
.msg = hintfmt("derivation cannot be both content-addressed and impure"),
.errPos = posDrvName
});
auto ht = parseHashType(outputHashAlgo.value_or("sha256"));
auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i);
drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating {
.method = ingestionMethod,
.hashType = ht,
});
if (isImpure)
drv.outputs.insert_or_assign(i,
DerivationOutput::Impure {
.method = method,
.hashType = ht,
});
else
drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating {
.method = method,
.hashType = ht,
});
}
}

183
src/libstore/build/derivation-goal.cc
View file

@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
{
trace("have derivation");
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
if (!drv->type().isPure()) {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
assert(!worker.store.isValidPath(randomPath));
initialOutputs.insert({
outputName,
InitialOutput {
.wanted = true,
.outputHash = impureOutputHash,
.known = InitialOutputStatus {
.path = randomPath,
.status = PathStatus::Absent
}
}
});
}
gaveUpOnSubstitution();
return;
}
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
return;
}
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
{
trace("all outputs substituted (maybe)");
assert(drv->type().isPure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
void DerivationGoal::gaveUpOnSubstitution()
{
/* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear();
if (useDerivation)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
if (drv->type().isPure() && !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(i.first);
if (!inputDrv.type().isPure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(i.first));
}
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
}
/* Copy the input sources from the eval store to the build
store. */
@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
void DerivationGoal::repairClosure()
{
assert(drv->type().isPure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
drvs. */
: true);
},
[&](const DerivationType::Impure &) {
return true;
}
}, drvType.raw());
if (resolveDrv)
{
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
settings.requireExperimentalFeature(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
std::optional attempt = fullDrv.tryResolve(worker.store);
now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
assert(attempt);
Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
auto msg = fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@ -488,21 +528,13 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.evalStore.isValidPath(drvPath));
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
for (auto & j : wantedDepOutputs) {
if (outputs.count(j) > 0) {
auto optRealizedInput = outputs.at(j);
if (!optRealizedInput)
throw Error(
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
} else
for (auto & j : wantedDepOutputs)
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
worker.store.computeFSClosure(*outPath, inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
}
}
}
@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
!derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@ -934,60 +966,53 @@ void DerivationGoal::buildDone()
void DerivationGoal::resolvedFinished()
{
trace("resolved derivation finished");
assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv;
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames();
auto & resolvedResult = resolvedDrvGoal->buildResult;
DrvOutputs builtOutputs;
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = worker.store.queryRealisation(
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
);
// We've just built it, but maybe the build failed, in which case the
// realisation won't be there
if (realisation) {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
assert(!buildResult.success());
if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths;
// `wantedOutputs` might be empty, which means “all the outputs”
auto realWantedOutputs = wantedOutputs;
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames();
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = resolvedResult.builtOutputs.at(
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
if (drv->type().isPure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
newRealisation.signatures.clear();
if (!drv->type().isFixed())
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
}
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation);
}
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
}
runPostBuildHook(
worker.store,
*logger,
drvPath,
outputPaths
);
auto status = [&]() {
auto & resolvedResult = resolvedDrvGoal->buildResult;
switch (resolvedResult.status) {
case BuildResult::AlreadyValid:
return BuildResult::ResolvesToAlreadyValid;
default:
return resolvedResult.status;
}
}();
auto status = resolvedResult.status;
if (status == BuildResult::AlreadyValid)
status = BuildResult::ResolvesToAlreadyValid;
done(status, std::move(builtOutputs));
}
@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
@ -1248,6 +1274,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{
if (!drv->type().isPure()) return { false, {} };
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs;
@ -1304,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
}
// If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
@ -1343,7 +1373,6 @@ void DerivationGoal::done(
if (ex)
// FIXME: strip: "error: "
buildResult.errorMsg = ex->what();
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure)
@ -1370,7 +1399,21 @@ void DerivationGoal::done(
fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
}
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
}
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{
Goal::waiteeDone(waitee, result);
if (waitee->buildResult.success())
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
inputDrvOutputs.insert_or_assign(
{ bfd->drvPath, output.outputName },
realisation.outPath);
}
}

7
src/libstore/build/derivation-goal.hh
View file

@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
them. */
StringSet wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
finishes, before inputsRealised() is reached, */
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */
bool needRestart = false;
@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
DrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths);
};

23
src/libstore/build/local-derivation-goal.cc
View file

@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = !(derivationType.isImpure()) && !noChroot;
useChroot = derivationType.isSandboxed() && !noChroot;
}
auto & localStore = getLocalStore();
@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */
if (!(derivationType.isImpure()))
if (derivationType.isSandboxed())
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
us.
*/
if (!(derivationType.isImpure()))
if (derivationType.isSandboxed())
privateNetwork = true;
userNamespaceSync.create();
@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
if (derivationType.isImpure()) {
if (!derivationType.isSandboxed()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
if (derivationType.isImpure()) {
if (!derivationType.isSandboxed()) {
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
@ -1918,7 +1918,7 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
if (derivationType.isImpure())
if (!derivationType.isSandboxed())
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Add the output paths we'll use at build-time to the chroot */
@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
assert(false);
},
[&](const DerivationOutput::Impure & doi) {
return newInfoFromCA(DerivationOutput::CAFloating {
.method = doi.method,
.hashType = doi.hashType,
});
},
}, output.raw());
/* FIXME: set proper permissions in restorePath() so
@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
},
.outPath = newInfo.path
};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
&& drv->type().isPure())
{
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}

241
src/libstore/derivations.cc
View file

@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt;
},
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
return std::nullopt;
},
}, raw());
}
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool DerivationType::isCA() const {
bool DerivationType::isCA() const
{
/* Normally we do the full `std::visit` to make sure we have
exhaustively handled all variants, but so long as there is a
variant called `ContentAddressed`, it must be the only one for
which `isCA` is true for this to make sense!. */
return std::holds_alternative<ContentAddressed>(raw());
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return true;
},
}, raw());
}
bool DerivationType::isFixed() const {
bool DerivationType::isFixed() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::hasKnownOutputPaths() const {
bool DerivationType::hasKnownOutputPaths() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return !ia.deferred;
@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::isImpure() const {
bool DerivationType::isSandboxed() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
return true;
},
[](const ContentAddressed & ca) {
return !ca.pure;
return ca.sandboxed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::isPure() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return true;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return false;
},
}, raw());
}
@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
if (hash == "impure") {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
assert(pathS == "");
return DerivationOutput::Impure {
.method = std::move(method),
.hashType = std::move(hashType),
};
} else if (hash != "") {
validatePath(pathS);
return DerivationOutput::CAFixed {
.hash = FixedOutputHash {
@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutputImpure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw());
s += ')';
@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::set<std::string_view>
inputAddressedOutputs,
fixedCAOutputs,
floatingCAOutputs,
deferredIAOutputs,
impureOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed &) {
@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const
floatingHashType = dof.hashType;
} else {
if (*floatingHashType != dof.hashType)
throw Error("All floating outputs must use the same hash type");
throw Error("all floating outputs must use the same hash type");
}
},
[&](const DerivationOutput::Deferred &) {
deferredIAOutputs.insert(i.first);
deferredIAOutputs.insert(i.first);
},
[&](const DerivationOutput::Impure &) {
impureOutputs.insert(i.first);
},
}, i.second.raw());
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
throw Error("must have at least one output");
if (!inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = false,
};
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& !fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
{
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed {
.pure = false,
.sandboxed = false,
.fixed = true,
};
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
}
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& !floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::ContentAddressed {
.pure = true,
.sandboxed = true,
.fixed = false,
};
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& !deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = true,
};
} else {
throw Error("Can't mix derivation output types");
}
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& !impureOutputs.empty())
return DerivationType::Impure { };
throw Error("can't mix derivation output types");
}
@ -524,12 +621,22 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return DrvHash{
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
}
if (!type.isPure()) {
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs)
outputHashes.insert_or_assign(outputName, impureOutputHash);
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Deferred,
};
}
auto kind = std::visit(overloaded {
[](const DerivationType::InputAddressed & ia) {
/* This might be a "pesimistically" deferred output, so we don't
@ -541,6 +648,9 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
? DrvHash::Kind::Regular
: DrvHash::Kind::Deferred;
},
[](const DerivationType::Impure &) -> DrvHash::Kind {
assert(false);
}
}, drv.type().raw());
std::map<std::string, StringSet> inputs2;
@ -599,7 +709,8 @@ StringSet BasicDerivation::outputNames() const
return names;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
{
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair(
@ -610,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
return outsAndOptPaths;
}
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
{
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
@ -672,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< ""
<< "";
},
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
<< "impure";
},
}, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
@ -697,21 +814,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
for (auto &rewrite: rewrites) {
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
{
for (auto & rewrite : rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second);
}
drv.builder = rewriteStrings(drv.builder, rewrites);
for (auto & arg: drv.args) {
for (auto & arg : drv.args) {
arg = rewriteStrings(arg, rewrites);
}
StringPairs newEnv;
for (auto & envVar: drv.env) {
for (auto & envVar : drv.env) {
auto envName = rewriteStrings(envVar.first, rewrites);
auto envValue = rewriteStrings(envVar.second, rewrites);
newEnv.emplace(envName, envValue);
@ -732,48 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
static bool tryResolveInput(
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
const StorePath & inputDrv, const StringSet & inputOutputs)
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
{
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
auto getOutput = [&](const std::string & outputName) {
auto & actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt)
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv)
);
return actualPathOpt;
};
for (auto & input : inputDrvs)
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
if (outputPath)
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
for (auto & outputName : inputOutputs) {
auto actualPathOpt = getOutput(outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(actualPath));
inputSrcs.insert(std::move(actualPath));
}
return true;
return tryResolve(store, inputDrvOutputs);
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
std::optional<BasicDerivation> Derivation::tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
{
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
for (auto & [inputDrv, inputOutputs] : inputDrvs)
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
return std::nullopt;
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
for (auto & outputName : inputOutputs) {
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(*actualPath));
resolved.inputSrcs.insert(*actualPath);
} else {
warn("output '%s' of input '%s' missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv));
return {};
}
}
}
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
}
const Hash impureOutputHash = hashString(htSHA256, "impure");
}

59
src/libstore/derivations.hh
View file

@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
};
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm
* known yet.
*/
struct DerivationOutputDeferred {};
/* Impure output which is moved to a content-addressed location (like
CAFloating) but isn't registered as a realization.
*/
struct DerivationOutputImpure
{
/* information used for expected hash computation */
FileIngestionMethod method;
HashType hashType;
};
typedef std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating,
DerivationOutputDeferred
DerivationOutputDeferred,
DerivationOutputImpure
> _DerivationOutputRaw;
struct DerivationOutput : _DerivationOutputRaw
@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
using CAFixed = DerivationOutputCAFixed;
using CAFloating = DerivationOutputCAFloating;
using Deferred = DerivationOutputDeferred;
using Impure = DerivationOutputImpure;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
@ -90,13 +102,17 @@ struct DerivationType_InputAddressed {
};
struct DerivationType_ContentAddressed {
bool pure;
bool sandboxed;
bool fixed;
};
struct DerivationType_Impure {
};
typedef std::variant<
DerivationType_InputAddressed,
DerivationType_ContentAddressed
DerivationType_ContentAddressed,
DerivationType_Impure
> _DerivationTypeRaw;
struct DerivationType : _DerivationTypeRaw {
@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
using Raw::Raw;
using InputAddressed = DerivationType_InputAddressed;
using ContentAddressed = DerivationType_ContentAddressed;
using Impure = DerivationType_Impure;
/* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */
@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw {
non-CA derivations. */
bool isFixed() const;
/* Is the derivation impure and needs to access non-deterministic resources, or
pure and can be sandboxed? Note that whether or not we actually sandbox the
derivation is controlled separately. Never true for non-CA derivations. */
bool isImpure() const;
/* Whether the derivation is fully sandboxed. If false, the
sandbox is opened up, e.g. the derivation has access to the
network. Note that whether or not we actually sandbox the
derivation is controlled separately. Always true for non-CA
derivations. */
bool isSandboxed() const;
/* Whether the derivation is expected to produce the same result
every time, and therefore it only needs to be built once. This
is only false for derivations that have the attribute '__impure
= true'. */
bool isPure() const;
/* Does the derivation knows its own output paths?
Only true when there's no floating-ca derivation involved in the
@ -173,7 +197,14 @@ struct Derivation : BasicDerivation
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);
std::optional<BasicDerivation> tryResolve(Store & store) const;
/* Like the above, but instead of querying the Nix database for
realisations, uses a given mapping from input derivation paths
+ output names to actual output store paths. */
std::optional<BasicDerivation> tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
@ -211,7 +242,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
struct DrvHash {
std::map<std::string, Hash> hashes;
enum struct Kind: bool {
enum struct Kind : bool {
// Statically determined derivations.
// This hash will be directly used to compute the output paths
Regular,
@ -252,8 +283,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
FIXME: what is the Hash in this map?
*/
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHash> DrvHashes;
@ -286,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName);
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
extern const Hash impureOutputHash;
}

3
src/libstore/local-store.cc
View file

@ -719,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
}, i.second.raw());
}
}

7
src/libstore/misc.cc
View file

@ -277,15 +277,15 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
{
std::set<Realisation> inputRealisations;
for (const auto& [inputDrv, outputNames] : drv.inputDrvs) {
for (const auto & [inputDrv, outputNames] : drv.inputDrvs) {
auto outputHashes =
staticOutputHashes(store, store.readDerivation(inputDrv));
for (const auto& outputName : outputNames) {
for (const auto & outputName : outputNames) {
auto thisRealisation = store.queryRealisation(
DrvOutput{outputHashes.at(outputName), outputName});
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName,
"output '%s' of derivation '%s' isn't built", outputName,
store.printStorePath(inputDrv));
inputRealisations.insert(*thisRealisation);
}
@ -295,4 +295,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
}

9
src/libstore/path.cc
View file

@ -1,5 +1,7 @@
#include "store-api.hh"
#include <sodium.h>
namespace nix {
static void checkName(std::string_view path, std::string_view name)
@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
StorePath StorePath::random(std::string_view name)
{
Hash hash(htSHA1);
randombytes_buf(hash.hash, hash.hashSize);
return StorePath(hash, name);
}
StorePath Store::parseStorePath(std::string_view path) const
{
auto p = canonPath(std::string(path));

2
src/libstore/path.hh
View file

@ -58,6 +58,8 @@ public:
}
static StorePath dummy;
static StorePath random(std::string_view name);
};
typedef std::set<StorePath> StorePathSet;

1
src/libutil/experimental-features.cc
View file

@ -7,6 +7,7 @@ namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
{ Xp::ImpureDerivations, "impure-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },

1
src/libutil/experimental-features.hh
View file

@ -16,6 +16,7 @@ namespace nix {
enum struct ExperimentalFeature
{
CaDerivations,
ImpureDerivations,
Flakes,
NixCommand,
RecursiveNix,

4
src/nix/show-derivation.cc
View file

@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
outputObj.attr("impure", true);
},
}, output.raw());
}
}

63
tests/impure-derivations.nix Normal file
View file

@ -0,0 +1,63 @@
with import ./config.nix;
rec {
impure = mkDerivation {
name = "impure";
outputs = [ "out" "stuff" ];
buildCommand =
''
echo impure
x=$(< $TEST_ROOT/counter)
mkdir $out $stuff
echo $x > $out/n
ln -s $out/n $stuff/bla
printf $((x + 1)) > $TEST_ROOT/counter
'';
__impure = true;
impureEnvVars = [ "TEST_ROOT" ];
};
impureOnImpure = mkDerivation {
name = "impure-on-impure";
buildCommand =
''
echo impure-on-impure
x=$(< ${impure}/n)
mkdir $out
printf X$x > $out/n
ln -s ${impure.stuff} $out/symlink
ln -s $out $out/self
'';
__impure = true;
};
# This is not allowed.
inputAddressed = mkDerivation {
name = "input-addressed";
buildCommand =
''
cat ${impure} > $out
'';
};
contentAddressed = mkDerivation {
name = "content-addressed";
buildCommand =
''
echo content-addressed
x=$(< ${impureOnImpure}/n)
printf ''${x:0:1} > $out
'';
outputHashMode = "recursive";
outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ=";
};
inputAddressedAfterCA = mkDerivation {
name = "input-addressed-after-ca";
buildCommand =
''
cat ${contentAddressed} > $out
'';
};
}

57
tests/impure-derivations.sh Normal file
View file

@ -0,0 +1,57 @@
source common.sh
requireDaemonNewerThan "2.8pre20220311"
enableFeatures "ca-derivations ca-references impure-derivations"
restartDaemon
set -o pipefail
clearStore
# Basic test of impure derivations: building one a second time should not use the previous result.
printf 0 > $TEST_ROOT/counter
json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
path1=$(echo $json | jq -r .[].outputs.out)
path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
[[ $(< $path1/n) = 0 ]]
[[ $(< $path1_stuff/bla) = 0 ]]
[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
[[ $(< $path2/n) = 1 ]]
# Test impure derivations that depend on impure derivations.
path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
[[ $(< $path3/n) = X2 ]]
path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
[[ $(< $path4/n) = X3 ]]
# Test that (self-)references work.
[[ $(< $path4/symlink/bla) = 3 ]]
[[ $(< $path4/self/n) = X3 ]]
# Input-addressed derivations cannot depend on impure derivations directly.
(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]
# Fixed-output derivations *can* depend on impure derivations.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
[[ $(< $path5) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]
# And they should not be rebuilt.
path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
[[ $(< $path5) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]
# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations.
path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out)
[[ $(< $path6) = X ]]
[[ $(< $TEST_ROOT/counter) = 5 ]]

3
tests/local.mk
View file

@ -97,7 +97,8 @@ nix_tests = \
nix-profile.sh \
suggestions.sh \
store-ping.sh \
fetchClosure.sh
fetchClosure.sh \
impure-derivations.sh
ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh