Merge pull request #6227 from NixOS/impure-derivations-ng

Impure derivations

doc/manual/src/release-notes/rl-next.md

@@ -14,3 +14,21 @@
 
   This function is only available if you enable the experimental
   feature `fetch-closure`.
+
+* New experimental feature: *impure derivations*. These are
+  derivations that can produce a different result every time they're
+  built. Here is an example:
+
+  ```nix
+  stdenv.mkDerivation {
+    name = "impure";
+    __impure = true; # marks this derivation as impure
+    buildCommand = "date > $out";
+  }
+  ```
+
+  Running `nix build` twice on this expression will build the
+  derivation twice, producing two different content-addressed store
+  paths. Like fixed-output derivations, impure derivations have access
+  to the network. Only fixed-output derivations and impure derivations
+  can depend on an impure derivation.

src/libexpr/eval.cc

@@ -436,6 +436,7 @@ EvalState::EvalState(
     , sBuilder(symbols.create("builder"))
     , sArgs(symbols.create("args"))
     , sContentAddressed(symbols.create("__contentAddressed"))
+    , sImpure(symbols.create("__impure"))
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))

src/libexpr/eval.hh

@@ -78,7 +78,7 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sContentAddressed,
+        sContentAddressed, sImpure,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
         sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,

src/libexpr/primops.cc

@@ -989,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     PathSet context;
 
     bool contentAddressed = false;
+    bool isImpure = false;
     std::optional<std::string> outputHash;
-    std::string outputHashAlgo;
+    std::optional<std::string> outputHashAlgo;
-    auto ingestionMethod = FileIngestionMethod::Flat;
+    std::optional<FileIngestionMethod> ingestionMethod;
 
     StringSet outputs;
     outputs.insert("out");
@@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                     settings.requireExperimentalFeature(Xp::CaDerivations);
             }
 
+            else if (i->name == state.sImpure) {
+                isImpure = state.forceBool(*i->value, pos);
+                if (isImpure)
+                    settings.requireExperimentalFeature(Xp::ImpureDerivations);
+            }
+
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
             else if (i->name == state.sArgs) {
@@ -1183,27 +1190,43 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 .errPos = posDrvName
             });
 
-        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
+        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo.value_or("sha256"));
         Hash h = newHashAllowEmpty(*outputHash, ht);
 
-        auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
+        auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
+        auto outPath = state.store->makeFixedOutputPath(method, h, drvName);
         drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out",
             DerivationOutput::CAFixed {
                 .hash = FixedOutputHash {
-                    .method = ingestionMethod,
+                    .method = method,
                     .hash = std::move(h),
                 },
             });
     }
 
-    else if (contentAddressed) {
+    else if (contentAddressed || isImpure) {
-        HashType ht = parseHashType(outputHashAlgo);
+        if (contentAddressed && isImpure)
+            throw EvalError({
+                .msg = hintfmt("derivation cannot be both content-addressed and impure"),
+                .errPos = posDrvName
+            });
+
+        auto ht = parseHashType(outputHashAlgo.value_or("sha256"));
+        auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
+
         for (auto & i : outputs) {
             drv.env[i] = hashPlaceholder(i);
+            if (isImpure)
+                drv.outputs.insert_or_assign(i,
+                    DerivationOutput::Impure {
+                        .method = method,
+                        .hashType = ht,
+                    });
+            else
                 drv.outputs.insert_or_assign(i,
                     DerivationOutput::CAFloating {
-                    .method = ingestionMethod,
+                        .method = method,
                         .hashType = ht,
                     });
         }

src/libstore/build/derivation-goal.cc

@@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
 {
     trace("have derivation");
 
+    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+
     if (!drv->type().hasKnownOutputPaths())
         settings.requireExperimentalFeature(Xp::CaDerivations);
 
+    if (!drv->type().isPure()) {
+        settings.requireExperimentalFeature(Xp::ImpureDerivations);
+
+        for (auto & [outputName, output] : drv->outputs) {
+            auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
+            assert(!worker.store.isValidPath(randomPath));
+            initialOutputs.insert({
+                outputName,
+                InitialOutput {
+                    .wanted = true,
+                    .outputHash = impureOutputHash,
+                    .known = InitialOutputStatus {
+                        .path = randomPath,
+                        .status = PathStatus::Absent
+                    }
+                }
+            });
+        }
+
+        gaveUpOnSubstitution();
+        return;
+    }
+
     for (auto & i : drv->outputsAndOptPaths(worker.store))
         if (i.second.second)
             worker.store.addTempRoot(*i.second.second);
@@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
         return;
     }
 
-    parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
 {
     trace("all outputs substituted (maybe)");
 
+    assert(drv->type().isPure());
+
     if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
         done(BuildResult::TransientFailure, {},
             Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
 void DerivationGoal::gaveUpOnSubstitution()
 {
     /* The inputs must be built before we can build this goal. */
+    inputDrvOutputs.clear();
     if (useDerivation)
-        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
+        for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+            /* Ensure that pure, non-fixed-output derivations don't
+               depend on impure derivations. */
+            if (drv->type().isPure() && !drv->type().isFixed()) {
+                auto inputDrv = worker.evalStore.readDerivation(i.first);
+                if (!inputDrv.type().isPure())
+                    throw Error("pure derivation '%s' depends on impure derivation '%s'",
+                        worker.store.printStorePath(drvPath),
+                        worker.store.printStorePath(i.first));
+            }
+
             addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+        }
 
     /* Copy the input sources from the eval store to the build
        store. */
@@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
 
 void DerivationGoal::repairClosure()
 {
+    assert(drv->type().isPure());
+
     /* If we're repairing, we now know that our own outputs are valid.
        Now check whether the other paths in the outputs closure are
        good.  If not, then start derivation goals for the derivations
@@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
                        drvs. */
                     : true);
             },
+            [&](const DerivationType::Impure &) {
+                return true;
+            }
         }, drvType.raw());
 
-        if (resolveDrv)
+        if (resolveDrv && !fullDrv.inputDrvs.empty()) {
-        {
             settings.requireExperimentalFeature(Xp::CaDerivations);
 
             /* We are be able to resolve this derivation based on the
-               now-known results of dependencies. If so, we become a stub goal
+               now-known results of dependencies. If so, we become a
-               aliasing that resolved derivation goal */
+               stub goal aliasing that resolved derivation goal. */
-            std::optional attempt = fullDrv.tryResolve(worker.store);
+            std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
             assert(attempt);
             Derivation drvResolved { *std::move(attempt) };
 
             auto pathResolved = writeDerivation(worker.store, drvResolved);
 
-            auto msg = fmt("Resolved derivation: '%s' -> '%s'",
+            auto msg = fmt("resolved derivation: '%s' -> '%s'",
                 worker.store.printStorePath(drvPath),
                 worker.store.printStorePath(pathResolved));
             act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@@ -488,23 +528,15 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            assert(worker.evalStore.isValidPath(drvPath));
+            for (auto & j : wantedDepOutputs)
-            auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
-            for (auto & j : wantedDepOutputs) {
+                    worker.store.computeFSClosure(*outPath, inputPaths);
-                if (outputs.count(j) > 0) {
+                else
-                    auto optRealizedInput = outputs.at(j);
-                    if (!optRealizedInput)
-                        throw Error(
-                            "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
-                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
-                    worker.store.computeFSClosure(*optRealizedInput, inputPaths);
-                } else
                     throw Error(
                         "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
                         worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
         }
     }
-    }
 
     /* Second, the input sources. */
     worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
             st =
                 dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
                 statusOk(status) ? BuildResult::OutputRejected :
-                derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
+                !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure :
                 BuildResult::PermanentFailure;
         }
 
@@ -934,9 +966,15 @@ void DerivationGoal::buildDone()
 
 void DerivationGoal::resolvedFinished()
 {
+    trace("resolved derivation finished");
+
     assert(resolvedDrvGoal);
     auto resolvedDrv = *resolvedDrvGoal->drv;
+    auto & resolvedResult = resolvedDrvGoal->buildResult;
 
+    DrvOutputs builtOutputs;
+
+    if (resolvedResult.success()) {
         auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
 
         StorePathSet outputPaths;
@@ -946,30 +984,22 @@ void DerivationGoal::resolvedFinished()
         if (realWantedOutputs.empty())
             realWantedOutputs = resolvedDrv.outputNames();
 
-    DrvOutputs builtOutputs;
-
         for (auto & wantedOutput : realWantedOutputs) {
             assert(initialOutputs.count(wantedOutput) != 0);
             assert(resolvedHashes.count(wantedOutput) != 0);
-        auto realisation = worker.store.queryRealisation(
+            auto realisation = resolvedResult.builtOutputs.at(
-            DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
+                DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
-        );
+            if (drv->type().isPure()) {
-        // We've just built it, but maybe the build failed, in which case the
+                auto newRealisation = realisation;
-        // realisation won't be there
-        if (realisation) {
-            auto newRealisation = *realisation;
                 newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
                 newRealisation.signatures.clear();
-            newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
+                if (!drv->type().isFixed())
+                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
                 signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
-            outputPaths.insert(realisation->outPath);
-            builtOutputs.emplace(realisation->id, *realisation);
-        } else {
-            // If we don't have a realisation, then it must mean that something
-            // failed when building the resolved drv
-            assert(!buildResult.success());
             }
+            outputPaths.insert(realisation.outPath);
+            builtOutputs.emplace(realisation.id, realisation);
         }
 
         runPostBuildHook(
@@ -978,16 +1008,11 @@ void DerivationGoal::resolvedFinished()
             drvPath,
             outputPaths
         );
-
-    auto status = [&]() {
-        auto & resolvedResult = resolvedDrvGoal->buildResult;
-        switch (resolvedResult.status) {
-            case BuildResult::AlreadyValid:
-                return BuildResult::ResolvesToAlreadyValid;
-            default:
-                return resolvedResult.status;
     }
-    }();
+
+    auto status = resolvedResult.status;
+    if (status == BuildResult::AlreadyValid)
+        status = BuildResult::ResolvesToAlreadyValid;
 
     done(status, std::move(builtOutputs));
 }
@@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine()
 
 std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
 {
+    assert(drv->type().isPure());
     if (!useDerivation || drv->type().hasKnownOutputPaths()) {
         std::map<std::string, std::optional<StorePath>> res;
         for (auto & [name, output] : drv->outputs)
@@ -1248,6 +1274,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
 
 OutputPathMap DerivationGoal::queryDerivationOutputMap()
 {
+    assert(drv->type().isPure());
     if (!useDerivation || drv->type().hasKnownOutputPaths()) {
         OutputPathMap res;
         for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
 
 std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
 {
+    if (!drv->type().isPure()) return { false, {} };
+
     bool checkHash = buildMode == bmRepair;
     auto wantedOutputsLeft = wantedOutputs;
     DrvOutputs validOutputs;
@@ -1304,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
         if (info.wanted && info.known && info.known->isValid())
             validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
     }
+
     // If we requested all the outputs via the empty set, we are always fine.
     // If we requested specific elements, the loop above removes all the valid
     // ones, so any that are left must be invalid.
@@ -1343,7 +1373,6 @@ void DerivationGoal::done(
     if (ex)
         // FIXME: strip: "error: "
         buildResult.errorMsg = ex->what();
-    amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
     if (buildResult.status == BuildResult::TimedOut)
         worker.timedOut = true;
     if (buildResult.status == BuildResult::PermanentFailure)
@@ -1370,7 +1399,21 @@ void DerivationGoal::done(
         fs.open(traceBuiltOutputsFile, std::fstream::out);
         fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
     }
+
+    amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
 }
 
 
+void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
+{
+    Goal::waiteeDone(waitee, result);
+
+    if (waitee->buildResult.success())
+        if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
+            for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
+                inputDrvOutputs.insert_or_assign(
+                    { bfd->drvPath, output.outputName },
+                    realisation.outPath);
+}
+
 }

src/libstore/build/derivation-goal.hh

@@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
        them. */
     StringSet wantedOutputs;
 
+    /* Mapping from input derivations + output names to actual store
+       paths. This is filled in by waiteeDone() as each dependency
+       finishes, before inputsRealised() is reached, */
+    std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
+
     /* Whether additional wanted outputs have been added. */
     bool needRestart = false;
 
@@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
         DrvOutputs builtOutputs = {},
         std::optional<Error> ex = {});
 
+    void waiteeDone(GoalPtr waitee, ExitCode result) override;
+
     StorePathSet exportReferences(const StorePathSet & storePaths);
 };
 

src/libstore/build/local-derivation-goal.cc

@@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = !(derivationType.isImpure()) && !noChroot;
+            useChroot = derivationType.isSandboxed() && !noChroot;
     }
 
     auto & localStore = getLocalStore();
@@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
                 "nogroup:x:65534:\n", sandboxGid()));
 
         /* Create /etc/hosts with localhost entry. */
-        if (!(derivationType.isImpure()))
+        if (derivationType.isSandboxed())
             writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
@@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
            us.
         */
 
-        if (!(derivationType.isImpure()))
+        if (derivationType.isSandboxed())
             privateNetwork = true;
 
         userNamespaceSync.create();
@@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
        to the builder is generally impure, but the output of
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
-    if (derivationType.isImpure()) {
+    if (!derivationType.isSandboxed()) {
         for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
             env[i] = getEnv(i).value_or("");
     }
@@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild()
             /* Fixed-output derivations typically need to access the
                network, so give them access to /etc/resolv.conf and so
                on. */
-            if (derivationType.isImpure()) {
+            if (!derivationType.isSandboxed()) {
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
@@ -1918,7 +1918,7 @@ void LocalDerivationGoal::runChild()
 
                 sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
 
-                if (derivationType.isImpure())
+                if (!derivationType.isSandboxed())
                     sandboxProfile += "(import \"sandbox-network.sb\")\n";
 
                 /* Add the output paths we'll use at build-time to the chroot */
@@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                 assert(false);
             },
 
+            [&](const DerivationOutput::Impure & doi) {
+                return newInfoFromCA(DerivationOutput::CAFloating {
+                    .method = doi.method,
+                    .hashType = doi.hashType,
+                });
+            },
+
         }, output.raw());
 
         /* FIXME: set proper permissions in restorePath() so
@@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
             },
             .outPath = newInfo.path
         };
-        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+        if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
+            && drv->type().isPure())
+        {
             signRealisation(thisRealisation);
             worker.store.registerDrvOutput(thisRealisation);
         }

src/libstore/derivations.cc

@@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
         [](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
             return std::nullopt;
         },
+        [](const DerivationOutput::Impure &) -> std::optional<StorePath> {
+            return std::nullopt;
+        },
     }, raw());
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+{
     return store.makeFixedOutputPath(
         hash.method, hash.hash,
         outputPathName(drvName, outputName));
 }
 
 
-bool DerivationType::isCA() const {
+bool DerivationType::isCA() const
+{
     /* Normally we do the full `std::visit` to make sure we have
        exhaustively handled all variants, but so long as there is a
        variant called `ContentAddressed`, it must be the only one for
        which `isCA` is true for this to make sense!. */
-    return std::holds_alternative<ContentAddressed>(raw());
+    return std::visit(overloaded {
+        [](const InputAddressed & ia) {
+            return false;
+        },
+        [](const ContentAddressed & ca) {
+            return true;
+        },
+        [](const Impure &) {
+            return true;
+        },
+    }, raw());
 }
 
-bool DerivationType::isFixed() const {
+bool DerivationType::isFixed() const
+{
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
             return false;
@@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
         [](const ContentAddressed & ca) {
             return ca.fixed;
         },
+        [](const Impure &) {
+            return false;
+        },
     }, raw());
 }
 
-bool DerivationType::hasKnownOutputPaths() const {
+bool DerivationType::hasKnownOutputPaths() const
+{
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
             return !ia.deferred;
@@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const {
         [](const ContentAddressed & ca) {
             return ca.fixed;
         },
+        [](const Impure &) {
+            return false;
+        },
     }, raw());
 }
 
 
-bool DerivationType::isImpure() const {
+bool DerivationType::isSandboxed() const
+{
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
-            return false;
+            return true;
         },
         [](const ContentAddressed & ca) {
-            return !ca.pure;
+            return ca.sandboxed;
+        },
+        [](const Impure &) {
+            return false;
+        },
+    }, raw());
+}
+
+
+bool DerivationType::isPure() const
+{
+    return std::visit(overloaded {
+        [](const InputAddressed & ia) {
+            return true;
+        },
+        [](const ContentAddressed & ca) {
+            return true;
+        },
+        [](const Impure &) {
+            return false;
         },
     }, raw());
 }
@@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
             hashAlgo = hashAlgo.substr(2);
         }
         const auto hashType = parseHashType(hashAlgo);
-        if (hash != "") {
+        if (hash == "impure") {
+            settings.requireExperimentalFeature(Xp::ImpureDerivations);
+            assert(pathS == "");
+            return DerivationOutput::Impure {
+                .method = std::move(method),
+                .hashType = std::move(hashType),
+            };
+        } else if (hash != "") {
             validatePath(pathS);
             return DerivationOutput::CAFixed {
                 .hash = FixedOutputHash {
@@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
+            },
+            [&](const DerivationOutputImpure & doi) {
+                // FIXME
+                s += ','; printUnquotedString(s, "");
+                s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+                s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw());
         s += ')';
@@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
 
 DerivationType BasicDerivation::type() const
 {
-    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
+    std::set<std::string_view>
+        inputAddressedOutputs,
+        fixedCAOutputs,
+        floatingCAOutputs,
+        deferredIAOutputs,
+        impureOutputs;
     std::optional<HashType> floatingHashType;
+
     for (auto & i : outputs) {
         std::visit(overloaded {
             [&](const DerivationOutput::InputAddressed &) {
@@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const
                     floatingHashType = dof.hashType;
                 } else {
                     if (*floatingHashType != dof.hashType)
-                        throw Error("All floating outputs must use the same hash type");
+                        throw Error("all floating outputs must use the same hash type");
                 }
             },
             [&](const DerivationOutput::Deferred &) {
                 deferredIAOutputs.insert(i.first);
             },
+            [&](const DerivationOutput::Impure &) {
+                impureOutputs.insert(i.first);
+            },
         }, i.second.raw());
     }
 
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+    if (inputAddressedOutputs.empty()
-        throw Error("Must have at least one output");
+        && fixedCAOutputs.empty()
-    } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
+        throw Error("must have at least one output");
+
+    if (!inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
         return DerivationType::InputAddressed {
             .deferred = false,
         };
-    } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+
+    if (inputAddressedOutputs.empty()
+        && !fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
+    {
         if (fixedCAOutputs.size() > 1)
             // FIXME: Experimental feature?
-            throw Error("Only one fixed output is allowed for now");
+            throw Error("only one fixed output is allowed for now");
         if (*fixedCAOutputs.begin() != "out")
-            throw Error("Single fixed output must be named \"out\"");
+            throw Error("single fixed output must be named \"out\"");
         return DerivationType::ContentAddressed {
-            .pure = false,
+            .sandboxed = false,
             .fixed = true,
         };
-    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+    }
+
+    if (inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && !floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
         return DerivationType::ContentAddressed {
-            .pure = true,
+            .sandboxed = true,
             .fixed = false,
         };
-    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
+
+    if (inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && !deferredIAOutputs.empty()
+        && impureOutputs.empty())
         return DerivationType::InputAddressed {
             .deferred = true,
         };
-    } else {
+
-        throw Error("Can't mix derivation output types");
+    if (inputAddressedOutputs.empty()
-    }
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && !impureOutputs.empty())
+        return DerivationType::Impure { };
+
+    throw Error("can't mix derivation output types");
 }
 
 
@@ -530,6 +627,16 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
         };
     }
 
+    if (!type.isPure()) {
+        std::map<std::string, Hash> outputHashes;
+        for (const auto & [outputName, _] : drv.outputs)
+            outputHashes.insert_or_assign(outputName, impureOutputHash);
+        return DrvHash {
+            .hashes = outputHashes,
+            .kind = DrvHash::Kind::Deferred,
+        };
+    }
+
     auto kind = std::visit(overloaded {
         [](const DerivationType::InputAddressed & ia) {
             /* This might be a "pesimistically" deferred output, so we don't
@@ -541,6 +648,9 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
                 ? DrvHash::Kind::Regular
                 : DrvHash::Kind::Deferred;
         },
+        [](const DerivationType::Impure &) -> DrvHash::Kind {
+            assert(false);
+        }
     }, drv.type().raw());
 
     std::map<std::string, StringSet> inputs2;
@@ -599,7 +709,8 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
-DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
+{
     DerivationOutputsAndOptPaths outsAndOptPaths;
     for (auto output : outputs)
         outsAndOptPaths.insert(std::make_pair(
@@ -610,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
     return outsAndOptPaths;
 }
 
-std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
+std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
+{
     auto nameWithSuffix = drvPath.name();
     constexpr std::string_view extension = ".drv";
     assert(hasSuffix(nameWithSuffix, extension));
@@ -672,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
                     << ""
                     << "";
             },
+            [&](const DerivationOutput::Impure & doi) {
+                out << ""
+                    << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
+                    << "impure";
+            },
         }, i.second.raw());
     }
     worker_proto::write(store, out, drv.inputSrcs);
@@ -697,10 +814,8 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
 }
 
 
-static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
-
+{
-    debug("Rewriting the derivation");
-
     for (auto & rewrite : rewrites) {
         debug("rewriting %s as %s", rewrite.first, rewrite.second);
     }
@@ -732,48 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-static bool tryResolveInput(
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
-    Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
-    const StorePath & inputDrv, const StringSet & inputOutputs)
 {
-    auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
+    std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
-    auto getOutput = [&](const std::string & outputName) {
+    for (auto & input : inputDrvs)
-        auto & actualPathOpt = inputDrvOutputs.at(outputName);
+        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
-        if (!actualPathOpt)
+            if (outputPath)
-            warn("output %s of input %s missing, aborting the resolving",
+                inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
-                outputName,
-                store.printStorePath(inputDrv)
-            );
-        return actualPathOpt;
-    };
 
-    for (auto & outputName : inputOutputs) {
+    return tryResolve(store, inputDrvOutputs);
-        auto actualPathOpt = getOutput(outputName);
-        if (!actualPathOpt) return false;
-        auto actualPath = *actualPathOpt;
-        inputRewrites.emplace(
-            downstreamPlaceholder(store, inputDrv, outputName),
-            store.printStorePath(actualPath));
-        inputSrcs.insert(std::move(actualPath));
 }
 
-    return true;
+std::optional<BasicDerivation> Derivation::tryResolve(
-}
+    Store & store,
-
+    const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
-std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
+{
     BasicDerivation resolved { *this };
 
     // Input paths that we'll want to rewrite in the derivation
     StringMap inputRewrites;
 
-    for (auto & [inputDrv, inputOutputs] : inputDrvs)
+    for (auto & [inputDrv, inputOutputs] : inputDrvs) {
-        if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
+        for (auto & outputName : inputOutputs) {
-            return std::nullopt;
+            if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
+                inputRewrites.emplace(
+                    downstreamPlaceholder(store, inputDrv, outputName),
+                    store.printStorePath(*actualPath));
+                resolved.inputSrcs.insert(*actualPath);
+            } else {
+                warn("output '%s' of input '%s' missing, aborting the resolving",
+                    outputName,
+                    store.printStorePath(inputDrv));
+                return {};
+            }
+        }
+    }
 
     rewriteDerivation(store, resolved, inputRewrites);
 
     return resolved;
 }
 
+const Hash impureOutputHash = hashString(htSHA256, "impure");
+
 }

src/libstore/derivations.hh

@@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
 };
 
 /* Input-addressed output which depends on a (CA) derivation whose hash isn't
- * known atm
+ * known yet.
  */
 struct DerivationOutputDeferred {};
 
+/* Impure output which is moved to a content-addressed location (like
+   CAFloating) but isn't registered as a realization.
+ */
+struct DerivationOutputImpure
+{
+    /* information used for expected hash computation */
+    FileIngestionMethod method;
+    HashType hashType;
+};
+
 typedef std::variant<
     DerivationOutputInputAddressed,
     DerivationOutputCAFixed,
     DerivationOutputCAFloating,
-    DerivationOutputDeferred
+    DerivationOutputDeferred,
+    DerivationOutputImpure
 > _DerivationOutputRaw;
 
 struct DerivationOutput : _DerivationOutputRaw
@@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
     using CAFixed = DerivationOutputCAFixed;
     using CAFloating = DerivationOutputCAFloating;
     using Deferred = DerivationOutputDeferred;
+    using Impure = DerivationOutputImpure;
 
     /* Note, when you use this function you should make sure that you're passing
        the right derivation name. When in doubt, you should use the safer
@@ -90,13 +102,17 @@ struct DerivationType_InputAddressed {
 };
 
 struct DerivationType_ContentAddressed {
-    bool pure;
+    bool sandboxed;
     bool fixed;
 };
 
+struct DerivationType_Impure {
+};
+
 typedef std::variant<
     DerivationType_InputAddressed,
-    DerivationType_ContentAddressed
+    DerivationType_ContentAddressed,
+    DerivationType_Impure
 > _DerivationTypeRaw;
 
 struct DerivationType : _DerivationTypeRaw {
@@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
     using Raw::Raw;
     using InputAddressed = DerivationType_InputAddressed;
     using ContentAddressed = DerivationType_ContentAddressed;
-
+    using Impure = DerivationType_Impure;
 
     /* Do the outputs of the derivation have paths calculated from their content,
        or from the derivation itself? */
@@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw {
        non-CA derivations. */
     bool isFixed() const;
 
-    /* Is the derivation impure and needs to access non-deterministic resources, or
+    /* Whether the derivation is fully sandboxed. If false, the
-       pure and can be sandboxed? Note that whether or not we actually sandbox the
+       sandbox is opened up, e.g. the derivation has access to the
-       derivation is controlled separately. Never true for non-CA derivations. */
+       network. Note that whether or not we actually sandbox the
-    bool isImpure() const;
+       derivation is controlled separately. Always true for non-CA
+       derivations. */
+    bool isSandboxed() const;
+
+    /* Whether the derivation is expected to produce the same result
+       every time, and therefore it only needs to be built once. This
+       is only false for derivations that have the attribute '__impure
+       = true'. */
+    bool isPure() const;
 
     /* Does the derivation knows its own output paths?
        Only true when there's no floating-ca derivation involved in the
@@ -173,7 +197,14 @@ struct Derivation : BasicDerivation
           added directly to input sources.
 
        2. Input placeholders are replaced with realized input store paths. */
-    std::optional<BasicDerivation> tryResolve(Store & store);
+    std::optional<BasicDerivation> tryResolve(Store & store) const;
+
+    /* Like the above, but instead of querying the Nix database for
+       realisations, uses a given mapping from input derivation paths
+       + output names to actual output store paths. */
+    std::optional<BasicDerivation> tryResolve(
+        Store & store,
+        const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
 
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
@@ -252,6 +283,8 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
 /*
    Return a map associating each output to a hash that uniquely identifies its
    derivation (modulo the self-references).
+
+   FIXME: what is the Hash in this map?
  */
 std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
 
@@ -286,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName);
    dependency which is a CA derivation. */
 std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
 
+extern const Hash impureOutputHash;
+
 }

src/libstore/local-store.cc

@@ -719,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
             [&](const DerivationOutput::Deferred &) {
                 /* Nothing to check */
             },
+            [&](const DerivationOutput::Impure &) {
+                /* Nothing to check */
+            },
         }, i.second.raw());
     }
 }

src/libstore/misc.cc

@@ -285,7 +285,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
                 DrvOutput{outputHashes.at(outputName), outputName});
             if (!thisRealisation)
                 throw Error(
-                    "output '%s' of derivation '%s' isn’t built", outputName,
+                    "output '%s' of derivation '%s' isn't built", outputName,
                     store.printStorePath(inputDrv));
             inputRealisations.insert(*thisRealisation);
         }
@@ -295,4 +295,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 
     return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
 }
+
 }

src/libstore/path.cc

@@ -1,5 +1,7 @@
 #include "store-api.hh"
 
+#include <sodium.h>
+
 namespace nix {
 
 static void checkName(std::string_view path, std::string_view name)
@@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
 
 StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
 
+StorePath StorePath::random(std::string_view name)
+{
+    Hash hash(htSHA1);
+    randombytes_buf(hash.hash, hash.hashSize);
+    return StorePath(hash, name);
+}
+
 StorePath Store::parseStorePath(std::string_view path) const
 {
     auto p = canonPath(std::string(path));

src/libstore/path.hh

@@ -58,6 +58,8 @@ public:
     }
 
     static StorePath dummy;
+
+    static StorePath random(std::string_view name);
 };
 
 typedef std::set<StorePath> StorePathSet;

src/libutil/experimental-features.cc

@@ -7,6 +7,7 @@ namespace nix {
 
 std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::CaDerivations, "ca-derivations" },
+    { Xp::ImpureDerivations, "impure-derivations" },
     { Xp::Flakes, "flakes" },
     { Xp::NixCommand, "nix-command" },
     { Xp::RecursiveNix, "recursive-nix" },

src/libutil/experimental-features.hh

@@ -16,6 +16,7 @@ namespace nix {
 enum struct ExperimentalFeature
 {
     CaDerivations,
+    ImpureDerivations,
     Flakes,
     NixCommand,
     RecursiveNix,

src/nix/show-derivation.cc

@@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
                             outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                         },
                         [&](const DerivationOutput::Deferred &) {},
+                        [&](const DerivationOutput::Impure & doi) {
+                            outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+                            outputObj.attr("impure", true);
+                        },
                     }, output.raw());
                 }
             }

tests/impure-derivations.nix

@@ -0,0 +1,63 @@
+with import ./config.nix;
+
+rec {
+
+  impure = mkDerivation {
+    name = "impure";
+    outputs = [ "out" "stuff" ];
+    buildCommand =
+      ''
+        echo impure
+        x=$(< $TEST_ROOT/counter)
+        mkdir $out $stuff
+        echo $x > $out/n
+        ln -s $out/n $stuff/bla
+        printf $((x + 1)) > $TEST_ROOT/counter
+      '';
+    __impure = true;
+    impureEnvVars = [ "TEST_ROOT" ];
+  };
+
+  impureOnImpure = mkDerivation {
+    name = "impure-on-impure";
+    buildCommand =
+      ''
+        echo impure-on-impure
+        x=$(< ${impure}/n)
+        mkdir $out
+        printf X$x > $out/n
+        ln -s ${impure.stuff} $out/symlink
+        ln -s $out $out/self
+      '';
+    __impure = true;
+  };
+
+  # This is not allowed.
+  inputAddressed = mkDerivation {
+    name = "input-addressed";
+    buildCommand =
+      ''
+        cat ${impure} > $out
+      '';
+  };
+
+  contentAddressed = mkDerivation {
+    name = "content-addressed";
+    buildCommand =
+      ''
+        echo content-addressed
+        x=$(< ${impureOnImpure}/n)
+        printf ''${x:0:1} > $out
+      '';
+    outputHashMode = "recursive";
+    outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ=";
+  };
+
+  inputAddressedAfterCA = mkDerivation {
+    name = "input-addressed-after-ca";
+    buildCommand =
+      ''
+        cat ${contentAddressed} > $out
+      '';
+  };
+}

tests/impure-derivations.sh

@@ -0,0 +1,57 @@
+source common.sh
+
+requireDaemonNewerThan "2.8pre20220311"
+
+enableFeatures "ca-derivations ca-references impure-derivations"
+restartDaemon
+
+set -o pipefail
+
+clearStore
+
+# Basic test of impure derivations: building one a second time should not use the previous result.
+printf 0 > $TEST_ROOT/counter
+
+json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
+path1=$(echo $json | jq -r .[].outputs.out)
+path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
+[[ $(< $path1/n) = 0 ]]
+[[ $(< $path1_stuff/bla) = 0 ]]
+
+[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
+
+path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
+[[ $(< $path2/n) = 1 ]]
+
+# Test impure derivations that depend on impure derivations.
+path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
+[[ $(< $path3/n) = X2 ]]
+
+path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out)
+[[ $(< $path4/n) = X3 ]]
+
+# Test that (self-)references work.
+[[ $(< $path4/symlink/bla) = 3 ]]
+[[ $(< $path4/self/n) = X3 ]]
+
+# Input-addressed derivations cannot depend on impure derivations directly.
+(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation'
+
+drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
+[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
+[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]
+
+# Fixed-output derivations *can* depend on impure derivations.
+path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
+[[ $(< $path5) = X ]]
+[[ $(< $TEST_ROOT/counter) = 5 ]]
+
+# And they should not be rebuilt.
+path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out)
+[[ $(< $path5) = X ]]
+[[ $(< $TEST_ROOT/counter) = 5 ]]
+
+# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations.
+path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out)
+[[ $(< $path6) = X ]]
+[[ $(< $TEST_ROOT/counter) = 5 ]]

tests/local.mk

@@ -97,7 +97,8 @@ nix_tests = \
   nix-profile.sh \
   suggestions.sh \
   store-ping.sh \
-  fetchClosure.sh
+  fetchClosure.sh \
+  impure-derivations.sh
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	nix_tests += compute-levels.sh