scriptis/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

Add support for impure derivations

Browse source 
Impure derivations are derivations that can produce a different result
every time they're built. Example:

  stdenv.mkDerivation {
    name = "impure";
    __impure = true; # marks this derivation as impure
    outputHashAlgo = "sha256";
    outputHashMode = "recursive";
    buildCommand = "date > $out";
  };

Some important characteristics:

* This requires the 'impure-derivations' experimental feature.

* Impure derivations are not "cached". Thus, running "nix-build" on
  the example above multiple times will cause a rebuild every time.

* They are implemented similar to CA derivations, i.e. the output is
  moved to a content-addressed path in the store. The difference is
  that we don't register a realisation in the Nix database.

* Pure derivations are not allowed to depend on impure derivations. In
  the future fixed-output derivations will be allowed to depend on
  impure derivations, thus forming an "impurity barrier" in the
  dependency graph.

* When sandboxing is enabled, impure derivations can access the
  network in the same way as fixed-output derivations. In relaxed
  sandboxing mode, they can access the local filesystem.
This commit is contained in:
Eelco Dolstra 2022-03-30 16:31:01 +02:00
parent 28309352d9
commit 5cd72598fe
17 changed files with 486 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/libexpr/eval.cc
View file

@ -436,6 +436,7 @@ EvalState::EvalState(
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed"))
, sImpure(symbols.create("__impure"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))

2
src/libexpr/eval.hh
View file

@ -78,7 +78,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sContentAddressed,
sContentAddressed, sImpure,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,

22
src/libexpr/primops.cc
View file

@ -989,6 +989,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context;
bool contentAddressed = false;
bool isImpure = false;
std::optional<std::string> outputHash;
std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat;
@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
settings.requireExperimentalFeature(Xp::CaDerivations);
}
else if (i->name == state.sImpure) {
isImpure = state.forceBool(*i->value, pos);
if (isImpure)
settings.requireExperimentalFeature(Xp::ImpureDerivations);
}
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
else if (i->name == state.sArgs) {
@ -1197,10 +1204,23 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
else if (contentAddressed) {
else if (contentAddressed || isImpure) {
if (contentAddressed && isImpure)
throw EvalError({
.msg = hintfmt("derivation cannot be both content-addressed and impure"),
.errPos = posDrvName
});
HashType ht = parseHashType(outputHashAlgo);
for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i);
if (isImpure)
drv.outputs.insert_or_assign(i,
DerivationOutput::Impure {
.method = ingestionMethod,
.hashType = ht,
});
else
drv.outputs.insert_or_assign(i,
DerivationOutput::CAFloating {
.method = ingestionMethod,

142
src/libstore/build/derivation-goal.cc
View file

@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
{
trace("have derivation");
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
if (!drv->type().isPure()) {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
for (auto & [outputName, output] : drv->outputs) {
auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
assert(!worker.store.isValidPath(randomPath));
initialOutputs.insert({
outputName,
InitialOutput {
.wanted = true,
.outputHash = impureOutputHash,
.known = InitialOutputStatus {
.path = randomPath,
.status = PathStatus::Absent
}
}
});
}
gaveUpOnSubstitution();
return;
}
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
return;
}
parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
{
trace("all outputs substituted (maybe)");
assert(drv->type().isPure());
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
void DerivationGoal::gaveUpOnSubstitution()
{
/* The inputs must be built before we can build this goal. */
inputDrvOutputs.clear();
if (useDerivation)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
/* Ensure that pure derivations don't depend on impure
derivations. */
if (drv->type().isPure()) {
auto inputDrv = worker.evalStore.readDerivation(i.first);
if (!inputDrv.type().isPure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(i.first));
}
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
}
/* Copy the input sources from the eval store to the build
store. */
@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
void DerivationGoal::repairClosure()
{
assert(drv->type().isPure());
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
drvs. */
: true);
},
[&](const DerivationType::Impure &) {
return true;
}
}, drvType.raw());
if (resolveDrv)
{
if (resolveDrv && !fullDrv.inputDrvs.empty()) {
settings.requireExperimentalFeature(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
std::optional attempt = fullDrv.tryResolve(worker.store);
now-known results of dependencies. If so, we become a
stub goal aliasing that resolved derivation goal. */
std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
assert(attempt);
Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
auto msg = fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@ -488,23 +528,15 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
assert(worker.evalStore.isValidPath(drvPath));
auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
for (auto & j : wantedDepOutputs) {
if (outputs.count(j) > 0) {
auto optRealizedInput = outputs.at(j);
if (!optRealizedInput)
throw Error(
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
} else
for (auto & j : wantedDepOutputs)
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
worker.store.computeFSClosure(*outPath, inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
}
}
}
/* Second, the input sources. */
worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
derivationType.needsNetworkAccess() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@ -934,9 +966,15 @@ void DerivationGoal::buildDone()
void DerivationGoal::resolvedFinished()
{
trace("resolved derivation finished");
assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv;
auto & resolvedResult = resolvedDrvGoal->buildResult;
DrvOutputs builtOutputs;
if (resolvedResult.success()) {
auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
StorePathSet outputPaths;
@ -946,30 +984,21 @@ void DerivationGoal::resolvedFinished()
if (realWantedOutputs.empty())
realWantedOutputs = resolvedDrv.outputNames();
DrvOutputs builtOutputs;
for (auto & wantedOutput : realWantedOutputs) {
assert(initialOutputs.count(wantedOutput) != 0);
assert(resolvedHashes.count(wantedOutput) != 0);
auto realisation = worker.store.queryRealisation(
DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
);
// We've just built it, but maybe the build failed, in which case the
// realisation won't be there
if (realisation) {
auto newRealisation = *realisation;
newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
auto realisation = resolvedResult.builtOutputs.at(
DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
if (drv->type().isPure()) {
auto newRealisation = realisation;
newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
newRealisation.signatures.clear();
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
outputPaths.insert(realisation->outPath);
builtOutputs.emplace(realisation->id, *realisation);
} else {
// If we don't have a realisation, then it must mean that something
// failed when building the resolved drv
assert(!buildResult.success());
}
outputPaths.insert(realisation.outPath);
builtOutputs.emplace(realisation.id, realisation);
}
runPostBuildHook(
@ -978,16 +1007,11 @@ void DerivationGoal::resolvedFinished()
drvPath,
outputPaths
);
auto status = [&]() {
auto & resolvedResult = resolvedDrvGoal->buildResult;
switch (resolvedResult.status) {
case BuildResult::AlreadyValid:
return BuildResult::ResolvesToAlreadyValid;
default:
return resolvedResult.status;
}
}();
auto status = resolvedResult.status;
if (status == BuildResult::AlreadyValid)
status = BuildResult::ResolvesToAlreadyValid;
done(status, std::move(builtOutputs));
}
@ -1236,6 +1260,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
@ -1248,6 +1273,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@ -1261,6 +1287,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{
if (!drv->type().isPure()) return { false, {} };
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs;
@ -1304,6 +1332,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
}
// If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
@ -1343,7 +1372,6 @@ void DerivationGoal::done(
if (ex)
// FIXME: strip: "error: "
buildResult.errorMsg = ex->what();
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure)
@ -1370,7 +1398,21 @@ void DerivationGoal::done(
fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
}
amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
}
void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
{
Goal::waiteeDone(waitee, result);
if (waitee->buildResult.success())
if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
inputDrvOutputs.insert_or_assign(
{ bfd->drvPath, output.outputName },
realisation.outPath);
}
}

7
src/libstore/build/derivation-goal.hh
View file

@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
them. */
StringSet wantedOutputs;
/* Mapping from input derivations + output names to actual store
paths. This is filled in by waiteeDone() as each dependency
finishes, before inputsRealised() is reached, */
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
/* Whether additional wanted outputs have been added. */
bool needRestart = false;
@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
DrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
void waiteeDone(GoalPtr waitee, ExitCode result) override;
StorePathSet exportReferences(const StorePathSet & storePaths);
};

21
src/libstore/build/local-derivation-goal.cc
View file

@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
useChroot = !(derivationType.isImpure()) && !noChroot;
useChroot = !derivationType.needsNetworkAccess() && !noChroot;
}
auto & localStore = getLocalStore();
@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */
if (!(derivationType.isImpure()))
if (!derivationType.needsNetworkAccess())
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
us.
*/
if (!(derivationType.isImpure()))
if (!derivationType.needsNetworkAccess())
privateNetwork = true;
userNamespaceSync.create();
@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
if (derivationType.isImpure()) {
if (derivationType.needsNetworkAccess()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
if (derivationType.isImpure()) {
if (derivationType.needsNetworkAccess()) {
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
assert(false);
},
[&](const DerivationOutput::Impure & doi) {
return newInfoFromCA(DerivationOutput::CAFloating {
.method = doi.method,
.hashType = doi.hashType,
});
},
}, output.raw());
/* FIXME: set proper permissions in restorePath() so
@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
},
.outPath = newInfo.path
};
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
&& drv->type().isPure())
{
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}

233
src/libstore/derivations.cc
View file

@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt;
},
[](const DerivationOutput::Impure &) -> std::optional<StorePath> {
return std::nullopt;
},
}, raw());
}
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
bool DerivationType::isCA() const {
bool DerivationType::isCA() const
{
/* Normally we do the full `std::visit` to make sure we have
exhaustively handled all variants, but so long as there is a
variant called `ContentAddressed`, it must be the only one for
which `isCA` is true for this to make sense!. */
return std::holds_alternative<ContentAddressed>(raw());
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return true;
},
}, raw());
}
bool DerivationType::isFixed() const {
bool DerivationType::isFixed() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::hasKnownOutputPaths() const {
bool DerivationType::hasKnownOutputPaths() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return !ia.deferred;
@ -63,11 +83,15 @@ bool DerivationType::hasKnownOutputPaths() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
[](const Impure &) {
return false;
},
}, raw());
}
bool DerivationType::isImpure() const {
bool DerivationType::needsNetworkAccess() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
@ -75,6 +99,25 @@ bool DerivationType::isImpure() const {
[](const ContentAddressed & ca) {
return !ca.pure;
},
[](const Impure &) {
return true;
},
}, raw());
}
bool DerivationType::isPure() const
{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return true;
},
[](const ContentAddressed & ca) {
return true;
},
[](const Impure &) {
return false;
},
}, raw());
}
@ -176,7 +219,16 @@ static DerivationOutput parseDerivationOutput(const Store & store,
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo);
if (hash != "") {
if (hash == "impure") {
settings.requireExperimentalFeature(Xp::ImpureDerivations);
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputImpure {
.method = std::move(method),
.hashType = std::move(hashType),
},
};
} else if (hash != "") {
validatePath(pathS);
return DerivationOutput::CAFixed {
.hash = FixedOutputHash {
@ -345,6 +397,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
[&](const DerivationOutputImpure & doi) {
// FIXME
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw());
s += ')';
@ -410,8 +468,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::set<std::string_view>
inputAddressedOutputs,
fixedCAOutputs,
floatingCAOutputs,
deferredIAOutputs,
impureOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed &) {
@ -426,43 +490,78 @@ DerivationType BasicDerivation::type() const
floatingHashType = dof.hashType;
} else {
if (*floatingHashType != dof.hashType)
throw Error("All floating outputs must use the same hash type");
throw Error("all floating outputs must use the same hash type");
}
},
[&](const DerivationOutput::Deferred &) {
deferredIAOutputs.insert(i.first);
},
[&](const DerivationOutput::Impure &) {
impureOutputs.insert(i.first);
},
}, i.second.raw());
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
throw Error("must have at least one output");
if (!inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = false,
};
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& !fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
{
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed {
.pure = false,
.fixed = true,
};
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
}
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& !floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::ContentAddressed {
.pure = true,
.fixed = false,
};
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& !deferredIAOutputs.empty()
&& impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = true,
};
} else {
throw Error("Can't mix derivation output types");
}
if (inputAddressedOutputs.empty()
&& fixedCAOutputs.empty()
&& floatingCAOutputs.empty()
&& deferredIAOutputs.empty()
&& !impureOutputs.empty())
return DerivationType::Impure { };
throw Error("can't mix derivation output types");
}
@ -524,12 +623,22 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
return DrvHash{
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
}
if (!type.isPure()) {
std::map<std::string, Hash> outputHashes;
for (const auto & [outputName, _] : drv.outputs)
outputHashes.insert_or_assign(outputName, impureOutputHash);
return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Deferred,
};
}
auto kind = std::visit(overloaded {
[](const DerivationType::InputAddressed & ia) {
/* This might be a "pesimistically" deferred output, so we don't
@ -541,6 +650,9 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
? DrvHash::Kind::Regular
: DrvHash::Kind::Deferred;
},
[](const DerivationType::Impure &) -> DrvHash::Kind {
assert(false);
}
}, drv.type().raw());
std::map<std::string, StringSet> inputs2;
@ -599,7 +711,8 @@ StringSet BasicDerivation::outputNames() const
return names;
}
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
{
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair(
@ -610,7 +723,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
return outsAndOptPaths;
}
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
{
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
@ -672,6 +786,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< ""
<< "";
},
[&](const DerivationOutput::Impure & doi) {
out << ""
<< (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
<< "impure";
},
}, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
@ -697,21 +816,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
for (auto &rewrite: rewrites) {
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
{
for (auto & rewrite : rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second);
}
drv.builder = rewriteStrings(drv.builder, rewrites);
for (auto & arg: drv.args) {
for (auto & arg : drv.args) {
arg = rewriteStrings(arg, rewrites);
}
StringPairs newEnv;
for (auto & envVar: drv.env) {
for (auto & envVar : drv.env) {
auto envName = rewriteStrings(envVar.first, rewrites);
auto envValue = rewriteStrings(envVar.second, rewrites);
newEnv.emplace(envName, envValue);
@ -732,48 +849,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
static bool tryResolveInput(
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
const StorePath & inputDrv, const StringSet & inputOutputs)
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
{
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
auto getOutput = [&](const std::string & outputName) {
auto & actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt)
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv)
);
return actualPathOpt;
};
for (auto & input : inputDrvs)
for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
if (outputPath)
inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
for (auto & outputName : inputOutputs) {
auto actualPathOpt = getOutput(outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(actualPath));
inputSrcs.insert(std::move(actualPath));
}
return true;
return tryResolve(store, inputDrvOutputs);
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
std::optional<BasicDerivation> Derivation::tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
{
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
for (auto & [inputDrv, inputOutputs] : inputDrvs)
if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
return std::nullopt;
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
for (auto & outputName : inputOutputs) {
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(*actualPath));
resolved.inputSrcs.insert(*actualPath);
} else {
warn("output '%s' of input '%s' missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv));
return {};
}
}
}
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
}
const Hash impureOutputHash = hashString(htSHA256, "impure");
}

52
src/libstore/derivations.hh
View file

@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
};
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm
* known yet.
*/
struct DerivationOutputDeferred {};
/* Impure output which is moved to a content-addressed location (like
CAFloating) but isn't registered as a realization.
*/
struct DerivationOutputImpure
{
/* information used for expected hash computation */
FileIngestionMethod method;
HashType hashType;
};
typedef std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating,
DerivationOutputDeferred
DerivationOutputDeferred,
DerivationOutputImpure
> _DerivationOutputRaw;
struct DerivationOutput : _DerivationOutputRaw
@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
using CAFixed = DerivationOutputCAFixed;
using CAFloating = DerivationOutputCAFloating;
using Deferred = DerivationOutputDeferred;
using Impure = DerivationOutputImpure;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
@ -94,9 +106,13 @@ struct DerivationType_ContentAddressed {
bool fixed;
};
struct DerivationType_Impure {
};
typedef std::variant<
DerivationType_InputAddressed,
DerivationType_ContentAddressed
DerivationType_ContentAddressed,
DerivationType_Impure
> _DerivationTypeRaw;
struct DerivationType : _DerivationTypeRaw {
@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
using Raw::Raw;
using InputAddressed = DerivationType_InputAddressed;
using ContentAddressed = DerivationType_ContentAddressed;
using Impure = DerivationType_Impure;
/* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */
@ -114,10 +130,13 @@ struct DerivationType : _DerivationTypeRaw {
non-CA derivations. */
bool isFixed() const;
/* Is the derivation impure and needs to access non-deterministic resources, or
pure and can be sandboxed? Note that whether or not we actually sandbox the
derivation is controlled separately. Never true for non-CA derivations. */
bool isImpure() const;
/* Whether the derivation needs to access the network. Note that
whether or not we actually sandbox the derivation is controlled
separately. Never true for non-CA derivations. */
bool needsNetworkAccess() const;
/* FIXME */
bool isPure() const;
/* Does the derivation knows its own output paths?
Only true when there's no floating-ca derivation involved in the
@ -173,7 +192,14 @@ struct Derivation : BasicDerivation
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);
std::optional<BasicDerivation> tryResolve(Store & store) const;
/* Like the above, but instead of querying the Nix database for
realisations, uses a given mapping from input derivation paths
+ output names to actual output store paths. */
std::optional<BasicDerivation> tryResolve(
Store & store,
const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
@ -211,7 +237,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
struct DrvHash {
std::map<std::string, Hash> hashes;
enum struct Kind: bool {
enum struct Kind : bool {
// Statically determined derivations.
// This hash will be directly used to compute the output paths
Regular,
@ -252,8 +278,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
FIXME: what is the Hash in this map?
*/
std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHash> DrvHashes;
@ -286,4 +314,6 @@ std::string hashPlaceholder(const std::string_view outputName);
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
extern const Hash impureOutputHash;
}

3
src/libstore/local-store.cc
View file

@ -719,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
[&](const DerivationOutput::Impure &) {
/* Nothing to check */
},
}, i.second.raw());
}
}

9
src/libstore/path.cc
View file

@ -1,5 +1,7 @@
#include "store-api.hh"
#include <sodium.h>
namespace nix {
static void checkName(std::string_view path, std::string_view name)
@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
StorePath StorePath::random(std::string_view name)
{
Hash hash(htSHA1);
randombytes_buf(hash.hash, hash.hashSize);
return StorePath(hash, name);
}
StorePath Store::parseStorePath(std::string_view path) const
{
auto p = canonPath(std::string(path));

2
src/libstore/path.hh
View file

@ -58,6 +58,8 @@ public:
}
static StorePath dummy;
static StorePath random(std::string_view name);
};
typedef std::set<StorePath> StorePathSet;

1
src/libutil/experimental-features.cc
View file

@ -7,6 +7,7 @@ namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
{ Xp::ImpureDerivations, "impure-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },

1
src/libutil/experimental-features.hh
View file

@ -16,6 +16,7 @@ namespace nix {
enum struct ExperimentalFeature
{
CaDerivations,
ImpureDerivations,
Flakes,
NixCommand,
RecursiveNix,

4
src/nix/show-derivation.cc
View file

@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
[&](const DerivationOutput::Deferred &) {},
[&](const DerivationOutput::Impure & doi) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
outputObj.attr("impure", true);
},
}, output.raw());
}
}

46
tests/impure-derivations.nix Normal file
View file

@ -0,0 +1,46 @@
with import ./config.nix;
rec {
impure = mkDerivation {
name = "impure";
outputs = [ "out" "stuff" ];
buildCommand =
''
x=$(< $TEST_ROOT/counter)
mkdir $out $stuff
echo $x > $out/n
ln -s $out/n $stuff/bla
printf $((x + 1)) > $TEST_ROOT/counter
'';
__impure = true;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
impureEnvVars = [ "TEST_ROOT" ];
};
impureOnImpure = mkDerivation {
name = "impure-on-impure";
buildCommand =
''
x=$(< ${impure}/n)
mkdir $out
printf X$x > $out/n
ln -s ${impure.stuff} $out/symlink
ln -s $out $out/self
'';
__impure = true;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
};
# This is not allowed.
inputAddressed = mkDerivation {
name = "input-addressed";
buildCommand =
''
cat ${impure} > $out
'';
};
}

39
tests/impure-derivations.sh Normal file
View file

@ -0,0 +1,39 @@
source common.sh
requireDaemonNewerThan "2.8pre20220311"
enableFeatures "ca-derivations ca-references impure-derivations"
clearStore
# Basic test of impure derivations: building one a second time should not use the previous result.
printf 0 > $TEST_ROOT/counter
json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure)
path1=$(echo $json | jq -r .[].outputs.out)
path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
[[ $(< $path1/n) = 0 ]]
[[ $(< $path1_stuff/bla) = 0 ]]
[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
[[ $(< $path2/n) = 1 ]]
# Test impure derivations that depend on impure derivations.
path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure -vvvvv | jq -r .[].outputs.out)
[[ $(< $path3/n) = X2 ]]
path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure -vvvvv | jq -r .[].outputs.out)
[[ $(< $path4/n) = X3 ]]
# Test that (self-)references work.
[[ $(< $path4/symlink/bla) = 3 ]]
[[ $(< $path4/self/n) = X3 ]]
# Input-addressed derivations cannot depend on impure derivations directly.
nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1 | grep 'depends on impure derivation'
drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]

3
tests/local.mk
View file

@ -97,7 +97,8 @@ nix_tests = \
nix-profile.sh \
suggestions.sh \
store-ping.sh \
fetchClosure.sh
fetchClosure.sh \
impure-derivations.sh
ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh