diff --git a/.gitignore b/.gitignore index 4b290425a..58e7377fb 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,7 @@ perl/Makefile.config /misc/systemd/nix-daemon.service /misc/systemd/nix-daemon.socket +/misc/systemd/nix-daemon.conf /misc/upstart/nix-daemon.conf /src/resolve-system-dependencies/resolve-system-dependencies diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2ec864ee4..4f3c9ce41 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -14,3 +14,21 @@ This function is only available if you enable the experimental feature `fetch-closure`. + +* New experimental feature: *impure derivations*. These are + derivations that can produce a different result every time they're + built. Here is an example: + + ```nix + stdenv.mkDerivation { + name = "impure"; + __impure = true; # marks this derivation as impure + buildCommand = "date > $out"; + } + ``` + + Running `nix build` twice on this expression will build the + derivation twice, producing two different content-addressed store + paths. Like fixed-output derivations, impure derivations have access + to the network. Only fixed-output derivations and impure derivations + can depend on an impure derivation. diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index b4badf2ba..24d894898 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -3,6 +3,7 @@ Description=Nix Daemon Documentation=man:nix-daemon https://nixos.org/manual RequiresMountsFor=@storedir@ RequiresMountsFor=@localstatedir@ +RequiresMountsFor=@localstatedir@/nix/db ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 24884a023..1d92c5388 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -90,7 +90,7 @@ poly_configure_nix_daemon_service() { ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST _sudo "to run systemd-tmpfiles once to pick that path up" \ - sytemd-tmpfiles create --prefix=/nix/var/nix + systemd-tmpfiles create --prefix=/nix/var/nix _sudo "to set up the nix-daemon service" \ systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a9923331e..1d44ffe84 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -789,55 +789,20 @@ std::shared_ptr SourceExprCommand::parseInstallable( return installables.front(); } -BuiltPaths getBuiltPaths(ref evalStore, ref store, const DerivedPaths & hopefullyBuiltPaths) +BuiltPaths Installable::build( + ref evalStore, + ref store, + Realise mode, + const std::vector> & installables, + BuildMode bMode) { BuiltPaths res; - for (const auto & b : hopefullyBuiltPaths) - std::visit( - overloaded{ - [&](const DerivedPath::Opaque & bo) { - res.push_back(BuiltPath::Opaque{bo.path}); - }, - [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - if (!outputHashes.count(output)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - auto outputId = - DrvOutput{outputHashes.at(output), output}; - auto realisation = - store->queryRealisation(outputId); - if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); - outputs.insert_or_assign( - output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - assert(drvOutputs.count(output)); - assert(drvOutputs.at(output).second); - outputs.insert_or_assign( - output, *drvOutputs.at(output).second); - } - } - res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); - }, - }, - b.raw()); - + for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode)) + res.push_back(builtPath); return res; } -BuiltPaths Installable::build( +std::vector, BuiltPath>> Installable::build2( ref evalStore, ref store, Realise mode, @@ -848,39 +813,93 @@ BuiltPaths Installable::build( settings.readOnlyMode = true; std::vector pathsToBuild; + std::map>> backmap; for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); + for (auto b : i->toDerivedPaths()) { + pathsToBuild.push_back(b); + backmap[b].push_back(i); + } } + std::vector, BuiltPath>> res; + switch (mode) { + case Realise::Nothing: case Realise::Derivation: printMissing(store, pathsToBuild, lvlError); - return getBuiltPaths(evalStore, store, pathsToBuild); + + for (auto & path : pathsToBuild) { + for (auto & installable : backmap[path]) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + OutputPathMap outputs; + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*store); + for (auto & output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store->printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + DrvOutput outputId { outputHashes.at(output), output }; + auto realisation = store->queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + outputs.insert_or_assign(output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + outputs.insert_or_assign( + output, *drvOutputs.at(output).second); + } + } + res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back({installable, BuiltPath::Opaque { bo.path }}); + }, + }, path.raw()); + } + } + + break; + case Realise::Outputs: { - BuiltPaths res; for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { if (!buildResult.success()) buildResult.rethrow(); - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - std::map outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back(BuiltPath::Built { bfd.drvPath, outputs }); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back(BuiltPath::Opaque { bo.path }); - }, - }, buildResult.path.raw()); + + for (auto & installable : backmap[buildResult.path]) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + std::map outputs; + for (auto & path : buildResult.builtOutputs) + outputs.emplace(path.first.outputName, path.second.outPath); + res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back({installable, BuiltPath::Opaque { bo.path }}); + }, + }, buildResult.path.raw()); + } } - return res; + + break; } + default: assert(false); } + + return res; } BuiltPaths Installable::toBuiltPaths( diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e172b71b0..f4bf0d406 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -98,6 +98,13 @@ struct Installable const std::vector> & installables, BuildMode bMode = bmNormal); + static std::vector, BuiltPath>> build2( + ref evalStore, + ref store, + Realise mode, + const std::vector> & installables, + BuildMode bMode = bmNormal); + static std::set toStorePaths( ref evalStore, ref store, @@ -185,9 +192,4 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake); -BuiltPaths getBuiltPaths( - ref evalStore, - ref store, - const DerivedPaths & hopefullyBuiltPaths); - } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 437c7fc53..b87e06ef5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -436,6 +436,7 @@ EvalState::EvalState( , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) + , sImpure(symbols.create("__impure")) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e7915dd99..7ed376e8d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -78,7 +78,7 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sContentAddressed, + sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f3eb5e925..969391725 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -989,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; bool contentAddressed = false; + bool isImpure = false; std::optional outputHash; - std::string outputHashAlgo; - auto ingestionMethod = FileIngestionMethod::Flat; + std::optional outputHashAlgo; + std::optional ingestionMethod; StringSet outputs; outputs.insert("out"); @@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * settings.requireExperimentalFeature(Xp::CaDerivations); } + else if (i->name == state.sImpure) { + isImpure = state.forceBool(*i->value, pos); + if (isImpure) + settings.requireExperimentalFeature(Xp::ImpureDerivations); + } + /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { @@ -1183,29 +1190,45 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .errPos = posDrvName }); - std::optional ht = parseHashTypeOpt(outputHashAlgo); + std::optional ht = parseHashTypeOpt(outputHashAlgo.value_or("sha256")); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); + auto outPath = state.store->makeFixedOutputPath(method, h, drvName); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput::CAFixed { .hash = FixedOutputHash { - .method = ingestionMethod, + .method = method, .hash = std::move(h), }, }); } - else if (contentAddressed) { - HashType ht = parseHashType(outputHashAlgo); + else if (contentAddressed || isImpure) { + if (contentAddressed && isImpure) + throw EvalError({ + .msg = hintfmt("derivation cannot be both content-addressed and impure"), + .errPos = posDrvName + }); + + auto ht = parseHashType(outputHashAlgo.value_or("sha256")); + auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); - drv.outputs.insert_or_assign(i, - DerivationOutput::CAFloating { - .method = ingestionMethod, - .hashType = ht, - }); + if (isImpure) + drv.outputs.insert_or_assign(i, + DerivationOutput::Impure { + .method = method, + .hashType = ht, + }); + else + drv.outputs.insert_or_assign(i, + DerivationOutput::CAFloating { + .method = method, + .hashType = ht, + }); } } @@ -1222,34 +1245,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * DerivationOutput::Deferred { }); } - // Regular, non-CA derivation should always return a single hash and not - // hash per output. - auto hashModulo = hashDerivationModulo(*state.store, drv, true); - std::visit(overloaded { - [&](const DrvHash & drvHash) { - auto & h = drvHash.hash; - switch (drvHash.kind) { - case DrvHash::Kind::Deferred: - /* Outputs already deferred, nothing to do */ - break; - case DrvHash::Kind::Regular: - for (auto & [outputName, output] : drv.outputs) { - auto outPath = state.store->makeOutputPath(outputName, h, drvName); - drv.env[outputName] = state.store->printStorePath(outPath); - output = DerivationOutput::InputAddressed { - .path = std::move(outPath), - }; - } - break; - } - }, - [&](const CaOutputHashes &) { - // Shouldn't happen as the toplevel derivation is not CA. - assert(false); - }, - }, - hashModulo.raw()); - + auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); + switch (hashModulo.kind) { + case DrvHash::Kind::Regular: + for (auto & i : outputs) { + auto h = hashModulo.hashes.at(i); + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign( + i, + DerivationOutputInputAddressed { + .path = std::move(outPath), + }); + } + break; + ; + case DrvHash::Kind::Deferred: + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, DerivationOutputDeferred {}); + } + } } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 517da4c01..7b35abca2 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -84,7 +84,8 @@ void printValueAsJSON(EvalState & state, bool strict, .msg = hintfmt("cannot convert %1% to JSON", showType(v)), .errPos = v.determinePos(pos) }); - throw e.addTrace(pos, hintfmt("message for the trace")); + e.addTrace(pos, hintfmt("message for the trace")); + throw e; } } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3d1c4fbc1..6582497bd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); + parsedDrv = std::make_unique(drvPath, *drv); + if (!drv->type().hasKnownOutputPaths()) settings.requireExperimentalFeature(Xp::CaDerivations); + if (!drv->type().isPure()) { + settings.requireExperimentalFeature(Xp::ImpureDerivations); + + for (auto & [outputName, output] : drv->outputs) { + auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); + assert(!worker.store.isValidPath(randomPath)); + initialOutputs.insert({ + outputName, + InitialOutput { + .wanted = true, + .outputHash = impureOutputHash, + .known = InitialOutputStatus { + .path = randomPath, + .status = PathStatus::Absent + } + } + }); + } + + gaveUpOnSubstitution(); + return; + } + for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) worker.store.addTempRoot(*i.second.second); @@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique(drvPath, *drv); - - /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); + assert(drv->type().isPure()); + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", @@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried() void DerivationGoal::gaveUpOnSubstitution() { /* The inputs must be built before we can build this goal. */ + inputDrvOutputs.clear(); if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) + for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (drv->type().isPure() && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(i.first); + if (!inputDrv.type().isPure()) + throw Error("pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(i.first)); + } + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + } /* Copy the input sources from the eval store to the build store. */ @@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution() void DerivationGoal::repairClosure() { + assert(drv->type().isPure()); + /* If we're repairing, we now know that our own outputs are valid. Now check whether the other paths in the outputs closure are good. If not, then start derivation goals for the derivations @@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised() drvs. */ : true); }, + [&](const DerivationType::Impure &) { + return true; + } }, drvType.raw()); - if (resolveDrv) - { + if (resolveDrv && !fullDrv.inputDrvs.empty()) { settings.requireExperimentalFeature(Xp::CaDerivations); /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); + now-known results of dependencies. If so, we become a + stub goal aliasing that resolved derivation goal. */ + std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); assert(attempt); Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - auto msg = fmt("Resolved derivation: '%s' -> '%s'", + auto msg = fmt("resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), worker.store.printStorePath(pathResolved)); act = std::make_unique(*logger, lvlInfo, actBuildWaiting, msg, @@ -488,21 +528,13 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.evalStore.isValidPath(drvPath)); - auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else + for (auto & j : wantedDepOutputs) + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + worker.store.computeFSClosure(*outPath, inputPaths); + else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - } } } @@ -923,7 +955,7 @@ void DerivationGoal::buildDone() st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - derivationType.isImpure() || diskFull ? BuildResult::TransientFailure : + !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -934,60 +966,53 @@ void DerivationGoal::buildDone() void DerivationGoal::resolvedFinished() { + trace("resolved derivation finished"); + assert(resolvedDrvGoal); auto resolvedDrv = *resolvedDrvGoal->drv; - - auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); - - StorePathSet outputPaths; - - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); + auto & resolvedResult = resolvedDrvGoal->buildResult; DrvOutputs builtOutputs; - for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} - ); - // We've just built it, but maybe the build failed, in which case the - // realisation won't be there - if (realisation) { - auto newRealisation = *realisation; - newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; - newRealisation.signatures.clear(); - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); - signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - outputPaths.insert(realisation->outPath); - builtOutputs.emplace(realisation->id, *realisation); - } else { - // If we don't have a realisation, then it must mean that something - // failed when building the resolved drv - assert(!buildResult.success()); + if (resolvedResult.success()) { + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); + + StorePathSet outputPaths; + + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv.outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(initialOutputs.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = resolvedResult.builtOutputs.at( + DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput }); + if (drv->type().isPure()) { + auto newRealisation = realisation; + newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; + newRealisation.signatures.clear(); + if (!drv->type().isFixed()) + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + signRealisation(newRealisation); + worker.store.registerDrvOutput(newRealisation); + } + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(realisation.id, realisation); } + + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); } - runPostBuildHook( - worker.store, - *logger, - drvPath, - outputPaths - ); - - auto status = [&]() { - auto & resolvedResult = resolvedDrvGoal->buildResult; - switch (resolvedResult.status) { - case BuildResult::AlreadyValid: - return BuildResult::ResolvesToAlreadyValid; - default: - return resolvedResult.status; - } - }(); + auto status = resolvedResult.status; + if (status == BuildResult::AlreadyValid) + status = BuildResult::ResolvesToAlreadyValid; done(status, std::move(builtOutputs)); } @@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine() std::map> DerivationGoal::queryPartialDerivationOutputMap() { + assert(drv->type().isPure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { std::map> res; for (auto & [name, output] : drv->outputs) @@ -1248,6 +1274,7 @@ std::map> DerivationGoal::queryPartialDeri OutputPathMap DerivationGoal::queryDerivationOutputMap() { + assert(drv->type().isPure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) @@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() std::pair DerivationGoal::checkPathValidity() { + if (!drv->type().isPure()) return { false, {} }; + bool checkHash = buildMode == bmRepair; auto wantedOutputsLeft = wantedOutputs; DrvOutputs validOutputs; @@ -1304,6 +1333,7 @@ std::pair DerivationGoal::checkPathValidity() if (info.wanted && info.known && info.known->isValid()) validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } + // If we requested all the outputs via the empty set, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. @@ -1343,7 +1373,6 @@ void DerivationGoal::done( if (ex) // FIXME: strip: "error: " buildResult.errorMsg = ex->what(); - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; if (buildResult.status == BuildResult::PermanentFailure) @@ -1370,7 +1399,21 @@ void DerivationGoal::done( fs.open(traceBuiltOutputsFile, std::fstream::out); fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } + + amDone(buildResult.success() ? ecSuccess : ecFailed, ex); } +void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + Goal::waiteeDone(waitee, result); + + if (waitee->buildResult.success()) + if (auto bfd = std::get_if(&waitee->buildResult.path)) + for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + inputDrvOutputs.insert_or_assign( + { bfd->drvPath, output.outputName }, + realisation.outPath); +} + } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index f556b6f25..2d8bfd592 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -57,6 +57,11 @@ struct DerivationGoal : public Goal them. */ StringSet wantedOutputs; + /* Mapping from input derivations + output names to actual store + paths. This is filled in by waiteeDone() as each dependency + finishes, before inputsRealised() is reached, */ + std::map, StorePath> inputDrvOutputs; + /* Whether additional wanted outputs have been added. */ bool needRestart = false; @@ -224,6 +229,8 @@ struct DerivationGoal : public Goal DrvOutputs builtOutputs = {}, std::optional ex = {}); + void waiteeDone(GoalPtr waitee, ExitCode result) override; + StorePathSet exportReferences(const StorePathSet & storePaths); }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b176f318b..40ef706a6 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationType.isImpure()) && !noChroot; + useChroot = derivationType.isSandboxed() && !noChroot; } auto & localStore = getLocalStore(); @@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (!(derivationType.isImpure())) + if (derivationType.isSandboxed()) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder() us. */ - if (!(derivationType.isImpure())) + if (derivationType.isSandboxed()) privateNetwork = true; userNamespaceSync.create(); @@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (derivationType.isImpure()) { + if (!derivationType.isSandboxed()) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (derivationType.isImpure()) { + if (!derivationType.isSandboxed()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -1918,7 +1918,7 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (derivationType.isImpure()) + if (!derivationType.isSandboxed()) sandboxProfile += "(import \"sandbox-network.sb\")\n"; /* Add the output paths we'll use at build-time to the chroot */ @@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() assert(false); }, + [&](const DerivationOutput::Impure & doi) { + return newInfoFromCA(DerivationOutput::CAFloating { + .method = doi.method, + .hashType = doi.hashType, + }); + }, + }, output.raw()); /* FIXME: set proper permissions in restorePath() so @@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + && drv->type().isPure()) + { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7fed80387..1c695de82 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -25,26 +25,42 @@ std::optional DerivationOutput::path(const Store & store, std::string [](const DerivationOutput::Deferred &) -> std::optional { return std::nullopt; }, + [](const DerivationOutput::Impure &) -> std::optional { + return std::nullopt; + }, }, raw()); } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +{ return store.makeFixedOutputPath( hash.method, hash.hash, outputPathName(drvName, outputName)); } -bool DerivationType::isCA() const { +bool DerivationType::isCA() const +{ /* Normally we do the full `std::visit` to make sure we have exhaustively handled all variants, but so long as there is a variant called `ContentAddressed`, it must be the only one for which `isCA` is true for this to make sense!. */ - return std::holds_alternative(raw()); + return std::visit(overloaded { + [](const InputAddressed & ia) { + return false; + }, + [](const ContentAddressed & ca) { + return true; + }, + [](const Impure &) { + return true; + }, + }, raw()); } -bool DerivationType::isFixed() const { +bool DerivationType::isFixed() const +{ return std::visit(overloaded { [](const InputAddressed & ia) { return false; @@ -52,10 +68,14 @@ bool DerivationType::isFixed() const { [](const ContentAddressed & ca) { return ca.fixed; }, + [](const Impure &) { + return false; + }, }, raw()); } -bool DerivationType::hasKnownOutputPaths() const { +bool DerivationType::hasKnownOutputPaths() const +{ return std::visit(overloaded { [](const InputAddressed & ia) { return !ia.deferred; @@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const { [](const ContentAddressed & ca) { return ca.fixed; }, + [](const Impure &) { + return false; + }, }, raw()); } -bool DerivationType::isImpure() const { +bool DerivationType::isSandboxed() const +{ return std::visit(overloaded { [](const InputAddressed & ia) { - return false; + return true; }, [](const ContentAddressed & ca) { - return !ca.pure; + return ca.sandboxed; + }, + [](const Impure &) { + return false; + }, + }, raw()); +} + + +bool DerivationType::isPure() const +{ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return true; + }, + [](const ContentAddressed & ca) { + return true; + }, + [](const Impure &) { + return false; }, }, raw()); } @@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store, hashAlgo = hashAlgo.substr(2); } const auto hashType = parseHashType(hashAlgo); - if (hash != "") { + if (hash == "impure") { + settings.requireExperimentalFeature(Xp::ImpureDerivations); + assert(pathS == ""); + return DerivationOutput::Impure { + .method = std::move(method), + .hashType = std::move(hashType), + }; + } else if (hash != "") { validatePath(pathS); return DerivationOutput::CAFixed { .hash = FixedOutputHash { @@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); + }, + [&](const DerivationOutputImpure & doi) { + // FIXME + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, "impure"); } }, i.second.raw()); s += ')'; @@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName DerivationType BasicDerivation::type() const { - std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; + std::set + inputAddressedOutputs, + fixedCAOutputs, + floatingCAOutputs, + deferredIAOutputs, + impureOutputs; std::optional floatingHashType; + for (auto & i : outputs) { std::visit(overloaded { [&](const DerivationOutput::InputAddressed &) { @@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const floatingHashType = dof.hashType; } else { if (*floatingHashType != dof.hashType) - throw Error("All floating outputs must use the same hash type"); + throw Error("all floating outputs must use the same hash type"); } }, [&](const DerivationOutput::Deferred &) { - deferredIAOutputs.insert(i.first); + deferredIAOutputs.insert(i.first); + }, + [&](const DerivationOutput::Impure &) { + impureOutputs.insert(i.first); }, }, i.second.raw()); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + throw Error("must have at least one output"); + + if (!inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) return DerivationType::InputAddressed { .deferred = false, }; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + + if (inputAddressedOutputs.empty() + && !fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? - throw Error("Only one fixed output is allowed for now"); + throw Error("only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") - throw Error("Single fixed output must be named \"out\""); + throw Error("single fixed output must be named \"out\""); return DerivationType::ContentAddressed { - .pure = false, + .sandboxed = false, .fixed = true, }; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + } + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && !floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) return DerivationType::ContentAddressed { - .pure = true, + .sandboxed = true, .fixed = false, }; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && !deferredIAOutputs.empty() + && impureOutputs.empty()) return DerivationType::InputAddressed { .deferred = true, }; - } else { - throw Error("Can't mix derivation output types"); - } + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && !impureOutputs.empty()) + return DerivationType::Impure { }; + + throw Error("can't mix derivation output types"); } @@ -474,7 +571,7 @@ Sync drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) { { auto hashes = drvHashes.lock(); @@ -509,7 +606,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { auto type = drv.type(); @@ -524,7 +621,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return outputHashes; + return DrvHash { + .hashes = outputHashes, + .kind = DrvHash::Kind::Regular, + }; + } + + if (!type.isPure()) { + std::map outputHashes; + for (const auto & [outputName, _] : drv.outputs) + outputHashes.insert_or_assign(outputName, impureOutputHash); + return DrvHash { + .hashes = outputHashes, + .kind = DrvHash::Kind::Deferred, + }; } auto kind = std::visit(overloaded { @@ -538,67 +648,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; }, + [](const DerivationType::Impure &) -> DrvHash::Kind { + assert(false); + } }, drv.type().raw()); - /* For other derivations, replace the inputs paths with recursive - calls to this function. */ std::map inputs2; for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { // Avoid lambda capture restriction with standard / Clang auto & inputOutputs = inputOutputs0; const auto & res = pathDerivationModulo(store, drvPath); - std::visit(overloaded { - // Regular non-CA derivation, replace derivation - [&](const DrvHash & drvHash) { - kind |= drvHash.kind; - inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs); - }, - // CA derivation's output hashes - [&](const CaOutputHashes & outputHashes) { - std::set justOut = { "out" }; - for (auto & output : inputOutputs) { - /* Put each one in with a single "out" output.. */ - const auto h = outputHashes.at(output); - inputs2.insert_or_assign( - h.to_string(Base16, false), - justOut); - } - }, - }, res.raw()); + if (res.kind == DrvHash::Kind::Deferred) + kind = DrvHash::Kind::Deferred; + for (auto & outputName : inputOutputs) { + const auto h = res.hashes.at(outputName); + inputs2[h.to_string(Base16, false)].insert(outputName); + } } auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); - return DrvHash { .hash = hash, .kind = kind }; -} - - -void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept -{ - switch (other) { - case DrvHash::Kind::Regular: - break; - case DrvHash::Kind::Deferred: - self = other; - break; + std::map outputHashes; + for (const auto & [outputName, _] : drv.outputs) { + outputHashes.insert_or_assign(outputName, hash); } + + return DrvHash { + .hashes = outputHashes, + .kind = kind, + }; } std::map staticOutputHashes(Store & store, const Derivation & drv) { - std::map res; - std::visit(overloaded { - [&](const DrvHash & drvHash) { - for (auto & outputName : drv.outputNames()) { - res.insert({outputName, drvHash.hash}); - } - }, - [&](const CaOutputHashes & outputHashes) { - res = outputHashes; - }, - }, hashDerivationModulo(store, drv, true).raw()); - return res; + return hashDerivationModulo(store, drv, true).hashes; } @@ -625,7 +709,8 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +{ DerivationOutputsAndOptPaths outsAndOptPaths; for (auto output : outputs) outsAndOptPaths.insert(std::make_pair( @@ -636,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s return outsAndOptPaths; } -std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) +{ auto nameWithSuffix = drvPath.name(); constexpr std::string_view extension = ".drv"; assert(hasSuffix(nameWithSuffix, extension)); @@ -698,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << "" << ""; }, + [&](const DerivationOutput::Impure & doi) { + out << "" + << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << "impure"; + }, }, i.second.raw()); } worker_proto::write(store, out, drv.inputSrcs); @@ -723,21 +814,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { - - debug("Rewriting the derivation"); - - for (auto &rewrite: rewrites) { +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +{ + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } drv.builder = rewriteStrings(drv.builder, rewrites); - for (auto & arg: drv.args) { + for (auto & arg : drv.args) { arg = rewriteStrings(arg, rewrites); } StringPairs newEnv; - for (auto & envVar: drv.env) { + for (auto & envVar : drv.env) { auto envName = rewriteStrings(envVar.first, rewrites); auto envValue = rewriteStrings(envVar.second, rewrites); newEnv.emplace(envName, envValue); @@ -747,7 +836,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.raw())) { - auto & h = hashModulo.requireNoFixedNonDeferred(); + auto & h = hashModulo.hashes.at(outputName); auto outPath = store.makeOutputPath(outputName, h, drv.name); drv.env[outputName] = store.printStorePath(outPath); output = DerivationOutput::InputAddressed { @@ -758,55 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -const Hash & DrvHashModulo::requireNoFixedNonDeferred() const { - auto * drvHashOpt = std::get_if(&raw()); - assert(drvHashOpt); - assert(drvHashOpt->kind == DrvHash::Kind::Regular); - return drvHashOpt->hash; -} - -static bool tryResolveInput( - Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, - const StorePath & inputDrv, const StringSet & inputOutputs) +std::optional Derivation::tryResolve(Store & store) const { - auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv); + std::map, StorePath> inputDrvOutputs; - auto getOutput = [&](const std::string & outputName) { - auto & actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) - warn("output %s of input %s missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv) - ); - return actualPathOpt; - }; + for (auto & input : inputDrvs) + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) + if (outputPath) + inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); - for (auto & outputName : inputOutputs) { - auto actualPathOpt = getOutput(outputName); - if (!actualPathOpt) return false; - auto actualPath = *actualPathOpt; - inputRewrites.emplace( - downstreamPlaceholder(store, inputDrv, outputName), - store.printStorePath(actualPath)); - inputSrcs.insert(std::move(actualPath)); - } - - return true; + return tryResolve(store, inputDrvOutputs); } -std::optional Derivation::tryResolve(Store & store) { +std::optional Derivation::tryResolve( + Store & store, + const std::map, StorePath> & inputDrvOutputs) const +{ BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) - if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs)) - return std::nullopt; + for (auto & [inputDrv, inputOutputs] : inputDrvs) { + for (auto & outputName : inputOutputs) { + if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { + inputRewrites.emplace( + downstreamPlaceholder(store, inputDrv, outputName), + store.printStorePath(*actualPath)); + resolved.inputSrcs.insert(*actualPath); + } else { + warn("output '%s' of input '%s' missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv)); + return {}; + } + } + } rewriteDerivation(store, resolved, inputRewrites); return resolved; } +const Hash impureOutputHash = hashString(htSHA256, "impure"); + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8dea90abf..af198a767 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -41,15 +41,26 @@ struct DerivationOutputCAFloating }; /* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known atm + * known yet. */ struct DerivationOutputDeferred {}; +/* Impure output which is moved to a content-addressed location (like + CAFloating) but isn't registered as a realization. + */ +struct DerivationOutputImpure +{ + /* information used for expected hash computation */ + FileIngestionMethod method; + HashType hashType; +}; + typedef std::variant< DerivationOutputInputAddressed, DerivationOutputCAFixed, DerivationOutputCAFloating, - DerivationOutputDeferred + DerivationOutputDeferred, + DerivationOutputImpure > _DerivationOutputRaw; struct DerivationOutput : _DerivationOutputRaw @@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw using CAFixed = DerivationOutputCAFixed; using CAFloating = DerivationOutputCAFloating; using Deferred = DerivationOutputDeferred; + using Impure = DerivationOutputImpure; /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer @@ -90,13 +102,17 @@ struct DerivationType_InputAddressed { }; struct DerivationType_ContentAddressed { - bool pure; + bool sandboxed; bool fixed; }; +struct DerivationType_Impure { +}; + typedef std::variant< DerivationType_InputAddressed, - DerivationType_ContentAddressed + DerivationType_ContentAddressed, + DerivationType_Impure > _DerivationTypeRaw; struct DerivationType : _DerivationTypeRaw { @@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw { using Raw::Raw; using InputAddressed = DerivationType_InputAddressed; using ContentAddressed = DerivationType_ContentAddressed; - + using Impure = DerivationType_Impure; /* Do the outputs of the derivation have paths calculated from their content, or from the derivation itself? */ @@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw { non-CA derivations. */ bool isFixed() const; - /* Is the derivation impure and needs to access non-deterministic resources, or - pure and can be sandboxed? Note that whether or not we actually sandbox the - derivation is controlled separately. Never true for non-CA derivations. */ - bool isImpure() const; + /* Whether the derivation is fully sandboxed. If false, the + sandbox is opened up, e.g. the derivation has access to the + network. Note that whether or not we actually sandbox the + derivation is controlled separately. Always true for non-CA + derivations. */ + bool isSandboxed() const; + + /* Whether the derivation is expected to produce the same result + every time, and therefore it only needs to be built once. This + is only false for derivations that have the attribute '__impure + = true'. */ + bool isPure() const; /* Does the derivation knows its own output paths? Only true when there's no floating-ca derivation involved in the @@ -173,7 +197,14 @@ struct Derivation : BasicDerivation added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ - std::optional tryResolve(Store & store); + std::optional tryResolve(Store & store) const; + + /* Like the above, but instead of querying the Nix database for + realisations, uses a given mapping from input derivation paths + + output names to actual output store paths. */ + std::optional tryResolve( + Store & store, + const std::map, StorePath> & inputDrvOutputs) const; Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } @@ -202,14 +233,16 @@ bool isDerivation(const std::string & fileName); the output name is "out". */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// known CA drv's output hashes, current just for fixed-output derivations -// whose output hashes are always known since they are fixed up-front. -typedef std::map CaOutputHashes; +// The hashes modulo of a derivation. +// +// Each output is given a hash, although in practice only the content-addressed +// derivations (fixed-output or not) will have a different hash for each +// output. struct DrvHash { - Hash hash; + std::map hashes; - enum struct Kind: bool { + enum struct Kind : bool { // Statically determined derivations. // This hash will be directly used to compute the output paths Regular, @@ -222,28 +255,6 @@ struct DrvHash { void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -typedef std::variant< - // Regular normalized derivation hash, and whether it was deferred (because - // an ancestor derivation is a floating content addressed derivation). - DrvHash, - // Fixed-output derivation hashes - CaOutputHashes -> _DrvHashModuloRaw; - -struct DrvHashModulo : _DrvHashModuloRaw { - using Raw = _DrvHashModuloRaw; - using Raw::Raw; - - /* Get hash, throwing if it is per-output CA hashes or a - deferred Drv hash. - */ - const Hash & requireNoFixedNonDeferred() const; - - inline const Raw & raw() const { - return static_cast(*this); - } -}; - /* Returns hashes with the details of fixed-output subderivations expunged. @@ -267,16 +278,18 @@ struct DrvHashModulo : _DrvHashModuloRaw { ATerm, after subderivations have been likewise expunged from that derivation. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Return a map associating each output to a hash that uniquely identifies its derivation (modulo the self-references). + + FIXME: what is the Hash in this map? */ -std::map staticOutputHashes(Store& store, const Derivation& drv); +std::map staticOutputHashes(Store & store, const Derivation & drv); /* Memoisation of hashDerivationModulo(). */ -typedef std::map DrvHashes; +typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. extern Sync drvHashes; @@ -306,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName); dependency which is a CA derivation. */ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); +extern const Hash impureOutputHash; + } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 8ca0882a4..24a0ae773 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -25,6 +25,9 @@ struct DerivedPathOpaque { nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + + bool operator < (const DerivedPathOpaque & b) const + { return path < b.path; } }; /** @@ -46,6 +49,9 @@ struct DerivedPathBuilt { std::string to_string(const Store & store) const; static DerivedPathBuilt parse(const Store & store, std::string_view); nlohmann::json toJSON(ref store) const; + + bool operator < (const DerivedPathBuilt & b) const + { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } }; using _DerivedPathRaw = std::variant< diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 46a547db1..d77fff963 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat // combinations that are currently prohibited. drv.type(); - std::optional h; + std::optional hashesModulo; for (auto & i : drv.outputs) { std::visit(overloaded { [&](const DerivationOutput::InputAddressed & doia) { - if (!h) { + if (!hashesModulo) { // somewhat expensive so we do lazily - auto h0 = hashDerivationModulo(*this, drv, true); - h = h0.requireNoFixedNonDeferred(); + hashesModulo = hashDerivationModulo(*this, drv, true); } - StorePath recomputed = makeOutputPath(i.first, *h, drvName); + StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName); if (doia.path != recomputed) throw Error("derivation '%s' has incorrect output '%s', should be '%s'", printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); @@ -720,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat [&](const DerivationOutput::Deferred &) { /* Nothing to check */ }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, }, i.second.raw()); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1f0bae7fe..2bbd7aa70 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -277,15 +277,15 @@ std::map drvOutputReferences( { std::set inputRealisations; - for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { auto outputHashes = staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto& outputName : outputNames) { + for (const auto & outputName : outputNames) { auto thisRealisation = store.queryRealisation( DrvOutput{outputHashes.at(outputName), outputName}); if (!thisRealisation) throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, + "output '%s' of derivation '%s' isn't built", outputName, store.printStorePath(inputDrv)); inputRealisations.insert(*thisRealisation); } @@ -295,4 +295,5 @@ std::map drvOutputReferences( return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } + } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index e642abcd5..392db225e 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,5 +1,7 @@ #include "store-api.hh" +#include + namespace nix { static void checkName(std::string_view path, std::string_view name) @@ -41,6 +43,13 @@ bool StorePath::isDerivation() const StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); +StorePath StorePath::random(std::string_view name) +{ + Hash hash(htSHA1); + randombytes_buf(hash.hash, hash.hashSize); + return StorePath(hash, name); +} + StorePath Store::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index e65fee622..77fd0f8dc 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -58,6 +58,8 @@ public: } static StorePath dummy; + + static StorePath random(std::string_view name); }; typedef std::set StorePathSet; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b2dfb35b2..02bc5caa5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -9,10 +9,9 @@ namespace nix { const std::string nativeSystem = SYSTEM; -BaseError & BaseError::addTrace(std::optional e, hintformat hint) +void BaseError::addTrace(std::optional e, hintformat hint) { err.traces.push_front(Trace { .pos = e, .hint = hint }); - return *this; } // c++ std::exception descendants must have a 'const char* what()' function. diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 93b789f0b..6a757f9ad 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -175,12 +175,12 @@ public: const ErrorInfo & info() const { calcWhat(); return err; } template - BaseError & addTrace(std::optional e, const std::string & fs, const Args & ... args) + void addTrace(std::optional e, const std::string & fs, const Args & ... args) { - return addTrace(e, hintfmt(fs, args...)); + addTrace(e, hintfmt(fs, args...)); } - BaseError & addTrace(std::optional e, hintformat hint); + void addTrace(std::optional e, hintformat hint); bool hasTrace() const { return !err.traces.empty(); } }; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 01f318fa3..e033a4116 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -7,6 +7,7 @@ namespace nix { std::map stringifiedXpFeatures = { { Xp::CaDerivations, "ca-derivations" }, + { Xp::ImpureDerivations, "impure-derivations" }, { Xp::Flakes, "flakes" }, { Xp::NixCommand, "nix-command" }, { Xp::RecursiveNix, "recursive-nix" }, diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index b5140dcfe..3a254b423 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -16,6 +16,7 @@ namespace nix { enum struct ExperimentalFeature { CaDerivations, + ImpureDerivations, Flakes, NixCommand, RecursiveNix, diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 70eaf4f9c..59e3aad6d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -71,13 +71,11 @@ void clearEnv() unsetenv(name.first.c_str()); } -void replaceEnv(std::map newEnv) +void replaceEnv(const std::map & newEnv) { clearEnv(); - for (auto newEnvVar : newEnv) - { + for (auto & newEnvVar : newEnv) setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); - } } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 7ed558dee..81fb8464a 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -9,7 +9,7 @@ using namespace nix; struct CmdBundle : InstallableCommand { - std::string bundler = "github:matthewbauer/nix-bundle"; + std::string bundler = "github:NixOS/bundlers"; std::optional outLink; CmdBundle() diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d2f9b5a6a..7fc74d34e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,10 +204,10 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore output.second = DerivationOutput::Deferred { }; drv.env[output.first] = ""; } - auto h0 = hashDerivationModulo(*evalStore, drv, true); - const Hash & h = h0.requireNoFixedNonDeferred(); + auto hashesModulo = hashDerivationModulo(*evalStore, drv, true); for (auto & output : drv.outputs) { + Hash h = hashesModulo.hashes.at(output.first); auto outPath = store->makeOutputPath(output.first, h, drv.name); output.second = DerivationOutput::InputAddressed { .path = outPath, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index da990ddc8..f35947ddb 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -62,22 +62,21 @@ struct ProfileElement return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); } - void updateStorePaths(ref evalStore, ref store, Installable & installable) + void updateStorePaths( + ref evalStore, + ref store, + const BuiltPaths & builtPaths) { // FIXME: respect meta.outputsToInstall storePaths.clear(); - for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) { + for (auto & buildable : builtPaths) { std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { storePaths.insert(bo.path); }, [&](const BuiltPath::Built & bfd) { - // TODO: Why are we querying if we know the output - // names already? Is it just to figure out what the - // default one is? - for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + for (auto & output : bfd.outputs) storePaths.insert(output.second); - } }, }, buildable.raw()); } @@ -236,6 +235,16 @@ struct ProfileManifest } }; +static std::map +builtPathsPerInstallable( + const std::vector, BuiltPath>> & builtPaths) +{ + std::map res; + for (auto & [installable, builtPath] : builtPaths) + res[installable.get()].push_back(builtPath); + return res; +} + struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { std::string description() override @@ -254,7 +263,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + auto builtPaths = builtPathsPerInstallable( + Installable::build2( + getEvalStore(), store, Realise::Outputs, installables, bmNormal)); for (auto & installable : installables) { ProfileElement element; @@ -269,7 +280,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile }; } - element.updateStorePaths(getEvalStore(), store, *installable); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); manifest.elements.push_back(std::move(element)); } @@ -457,12 +468,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf warn ("Use 'nix profile list' to see the current profile."); } - auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + auto builtPaths = builtPathsPerInstallable( + Installable::build2( + getEvalStore(), store, Realise::Outputs, installables, bmNormal)); for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, *installable); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); } updateProfile(manifest.build(store)); diff --git a/src/nix/run.cc b/src/nix/run.cc index 033263c36..23e893fbf 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -38,9 +38,12 @@ void runProgramInStore(ref store, unshare(CLONE_NEWUSER) doesn't work in a multithreaded program (which "nix" is), so we exec() a single-threaded helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast(); + auto store2 = store.dynamic_pointer_cast(); - if (store2 && store->storeDir != store2->getRealStoreDir()) { + if (!store2) + throw Error("store '%s' is not a local store so it does not support command execution", store->getUri()); + + if (store->storeDir != store2->getRealStoreDir()) { Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; for (auto & arg : args) helperArgs.push_back(arg); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 0d9655732..fb46b4dbf 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + outputObj.attr("impure", true); + }, }, output.raw()); } } diff --git a/tests/fetchPath.sh b/tests/fetchPath.sh index 8f17638e9..29be38ce2 100644 --- a/tests/fetchPath.sh +++ b/tests/fetchPath.sh @@ -1,6 +1,6 @@ source common.sh -touch foo -t 202211111111 +touch $TEST_ROOT/foo -t 202211111111 # We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. -[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$PWD/foo\").lastModifiedDate")" =~ 2022111.* ]] +[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] diff --git a/tests/impure-derivations.nix b/tests/impure-derivations.nix new file mode 100644 index 000000000..98547e6c1 --- /dev/null +++ b/tests/impure-derivations.nix @@ -0,0 +1,63 @@ +with import ./config.nix; + +rec { + + impure = mkDerivation { + name = "impure"; + outputs = [ "out" "stuff" ]; + buildCommand = + '' + echo impure + x=$(< $TEST_ROOT/counter) + mkdir $out $stuff + echo $x > $out/n + ln -s $out/n $stuff/bla + printf $((x + 1)) > $TEST_ROOT/counter + ''; + __impure = true; + impureEnvVars = [ "TEST_ROOT" ]; + }; + + impureOnImpure = mkDerivation { + name = "impure-on-impure"; + buildCommand = + '' + echo impure-on-impure + x=$(< ${impure}/n) + mkdir $out + printf X$x > $out/n + ln -s ${impure.stuff} $out/symlink + ln -s $out $out/self + ''; + __impure = true; + }; + + # This is not allowed. + inputAddressed = mkDerivation { + name = "input-addressed"; + buildCommand = + '' + cat ${impure} > $out + ''; + }; + + contentAddressed = mkDerivation { + name = "content-addressed"; + buildCommand = + '' + echo content-addressed + x=$(< ${impureOnImpure}/n) + printf ''${x:0:1} > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ="; + }; + + inputAddressedAfterCA = mkDerivation { + name = "input-addressed-after-ca"; + buildCommand = + '' + cat ${contentAddressed} > $out + ''; + }; +} diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh new file mode 100644 index 000000000..35ae3f5d3 --- /dev/null +++ b/tests/impure-derivations.sh @@ -0,0 +1,57 @@ +source common.sh + +requireDaemonNewerThan "2.8pre20220311" + +enableFeatures "ca-derivations ca-references impure-derivations" +restartDaemon + +set -o pipefail + +clearStore + +# Basic test of impure derivations: building one a second time should not use the previous result. +printf 0 > $TEST_ROOT/counter + +json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) +path1=$(echo $json | jq -r .[].outputs.out) +path1_stuff=$(echo $json | jq -r .[].outputs.stuff) +[[ $(< $path1/n) = 0 ]] +[[ $(< $path1_stuff/bla) = 0 ]] + +[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]] + +path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) +[[ $(< $path2/n) = 1 ]] + +# Test impure derivations that depend on impure derivations. +path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out) +[[ $(< $path3/n) = X2 ]] + +path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out) +[[ $(< $path4/n) = X3 ]] + +# Test that (self-)references work. +[[ $(< $path4/symlink/bla) = 3 ]] +[[ $(< $path4/self/n) = X3 ]] + +# Input-addressed derivations cannot depend on impure derivations directly. +(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' + +drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) +[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] + +# Fixed-output derivations *can* depend on impure derivations. +path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) +[[ $(< $path5) = X ]] +[[ $(< $TEST_ROOT/counter) = 5 ]] + +# And they should not be rebuilt. +path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) +[[ $(< $path5) = X ]] +[[ $(< $TEST_ROOT/counter) = 5 ]] + +# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations. +path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out) +[[ $(< $path6) = X ]] +[[ $(< $TEST_ROOT/counter) = 5 ]] diff --git a/tests/local.mk b/tests/local.mk index 592cad68b..56f9edf42 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -98,7 +98,8 @@ nix_tests = \ nix-profile.sh \ suggestions.sh \ store-ping.sh \ - fetchClosure.sh + fetchClosure.sh \ + impure-derivations.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh