Merge branch 'master' of github.com:NixOS/nix into make-narHash-not-optional

src/libexpr/eval.cc

@@ -345,6 +345,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sStructuredAttrs(symbols.create("__structuredAttrs"))
     , sBuilder(symbols.create("builder"))
     , sArgs(symbols.create("args"))
+    , sContentAddressed(symbols.create("__contentAddressed"))
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
@@ -1259,7 +1260,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
             addErrorTrace(e, lambda.pos, "while evaluating %s",
               (lambda.name.set()
                   ? "'" + (string) lambda.name + "'"
-                  : "anonymous lambdaction"));
+                  : "anonymous lambda"));
             addErrorTrace(e, pos, "from call site%s", "");
             throw;
         }

src/libexpr/eval.hh

@@ -74,6 +74,7 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
+        sContentAddressed,
         sOutputHash, sOutputHashAlgo, sOutputHashMode,
         sRecurseForDerivations,
         sDescription, sSelf, sEpsilon;

src/libexpr/primops.cc

@@ -583,6 +583,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
 
     PathSet context;
 
+    bool contentAddressed = false;
     std::optional<std::string> outputHash;
     std::string outputHashAlgo;
     auto ingestionMethod = FileIngestionMethod::Flat;
@@ -639,9 +640,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (i->value->type == tNull) continue;
             }
 
+            if (i->name == state.sContentAddressed) {
+                settings.requireExperimentalFeature("ca-derivations");
+                contentAddressed = state.forceBool(*i->value, pos);
+            }
+
             /* The `args' attribute is special: it supplies the
                command-line arguments to the builder. */
-            if (i->name == state.sArgs) {
+            else if (i->name == state.sArgs) {
                 state.forceList(*i->value, pos);
                 for (unsigned int n = 0; n < i->value->listSize(); ++n) {
                     string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true);
@@ -761,7 +767,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         });
 
     if (outputHash) {
-        /* Handle fixed-output derivations. */
+        /* Handle fixed-output derivations.
+
+           Ignore `__contentAddressed` because fixed output derivations are
+           already content addressed. */
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error({
                 .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
@@ -774,7 +783,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
-                .output = DerivationOutputFixed {
+                .output = DerivationOutputCAFixed {
                     .hash = FixedOutputHash {
                         .method = ingestionMethod,
                         .hash = std::move(h),
@@ -783,6 +792,19 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         });
     }
 
+    else if (contentAddressed) {
+        HashType ht = parseHashType(outputHashAlgo);
+        for (auto & i : outputs) {
+            if (!jsonObject) drv.env[i] = hashPlaceholder(i);
+            drv.outputs.insert_or_assign(i, DerivationOutput {
+                .output = DerivationOutputCAFloating {
+                    .method = ingestionMethod,
+                    .hashType = std::move(ht),
+                },
+            });
+        }
+    }
+
     else {
         /* Compute a hash over the "masked" store derivation, which is
            the final one except that in the list of outputs, the
@@ -800,7 +822,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 });
         }
 
-        Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
+        // Regular, non-CA derivation should always return a single hash and not
+        // hash per output.
+        Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
 
         for (auto & i : outputs) {
             auto outPath = state.store->makeOutputPath(i, h, drvName);

src/libfetchers/git.cc

@@ -121,7 +121,7 @@ struct GitInputScheme : InputScheme
             args.push_back(*ref);
         }
 
-        if (input.getRev()) throw Error("cloning a specific revision is not implemented");
+        if (input.getRev()) throw UnimplementedError("cloning a specific revision is not implemented");
 
         args.push_back(destDir);
 

src/libstore/build.cc

@@ -806,8 +806,8 @@ private:
     /* RAII object to delete the chroot directory. */
     std::shared_ptr<AutoDelete> autoDelChroot;
 
-    /* Whether this is a fixed-output derivation. */
+    /* The sort of derivation we are building. */
-    bool fixedOutput;
+    DerivationType derivationType;
 
     /* Whether to run the build in a private network namespace. */
     bool privateNetwork = false;
@@ -1195,9 +1195,9 @@ void DerivationGoal::haveDerivation()
 
     parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
 
-    if (parsedDrv->contentAddressed()) {
+    if (drv->type() == DerivationType::CAFloating) {
         settings.requireExperimentalFeature("ca-derivations");
-        throw Error("ca-derivations isn't implemented yet");
+        throw UnimplementedError("ca-derivations isn't implemented yet");
     }
 
 
@@ -1392,12 +1392,12 @@ void DerivationGoal::inputsRealised()
 
     debug("added input paths %s", worker.store.showPaths(inputPaths));
 
-    /* Is this a fixed-output derivation? */
+    /* What type of derivation are we building? */
-    fixedOutput = drv->isFixedOutput();
+    derivationType = drv->type();
 
     /* Don't repeat fixed-output derivations since they're already
        verified by their output hash.*/
-    nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1;
+    nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
 
     /* Okay, try to build.  Note that here we don't wait for a build
        slot to become available, since we don't need one if there is a
@@ -1783,7 +1783,7 @@ void DerivationGoal::buildDone()
             st =
                 dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
                 statusOk(status) ? BuildResult::OutputRejected :
-                fixedOutput || diskFull ? BuildResult::TransientFailure :
+                derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
                 BuildResult::PermanentFailure;
         }
 
@@ -1996,7 +1996,7 @@ void DerivationGoal::startBuilder()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
-            useChroot = !fixedOutput && !noChroot;
+            useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
     }
 
     if (worker.store.storeDir != worker.store.realStoreDir) {
@@ -2165,7 +2165,7 @@ void DerivationGoal::startBuilder()
                 "nogroup:x:65534:\n") % sandboxGid).str());
 
         /* Create /etc/hosts with localhost entry. */
-        if (!fixedOutput)
+        if (!(derivationIsImpure(derivationType)))
             writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
@@ -2373,7 +2373,7 @@ void DerivationGoal::startBuilder()
            us.
         */
 
-        if (!fixedOutput)
+        if (!(derivationIsImpure(derivationType)))
             privateNetwork = true;
 
         userNamespaceSync.create();
@@ -2574,7 +2574,7 @@ void DerivationGoal::initEnv()
        derivation, tell the builder, so that for instance `fetchurl'
        can skip checking the output.  On older Nixes, this environment
        variable won't be set, so `fetchurl' will do the check. */
-    if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
+    if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
 
     /* *Only* if this is a fixed-output derivation, propagate the
        values of the environment variables specified in the
@@ -2585,7 +2585,7 @@ void DerivationGoal::initEnv()
        to the builder is generally impure, but the output of
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
-    if (fixedOutput) {
+    if (derivationIsImpure(derivationType)) {
         for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
             env[i] = getEnv(i).value_or("");
     }
@@ -3195,7 +3195,7 @@ void DerivationGoal::runChild()
             /* Fixed-output derivations typically need to access the
                network, so give them access to /etc/resolv.conf and so
                on. */
-            if (fixedOutput) {
+            if (derivationIsImpure(derivationType)) {
                 ss.push_back("/etc/resolv.conf");
 
                 // Only use nss functions to resolve hosts and
@@ -3436,7 +3436,7 @@ void DerivationGoal::runChild()
 
                 sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
 
-                if (fixedOutput)
+                if (derivationIsImpure(derivationType))
                     sandboxProfile += "(import \"sandbox-network.sb\")\n";
 
                 /* Our rwx outputs */
@@ -3721,9 +3721,22 @@ void DerivationGoal::registerOutputs()
            hash). */
         std::optional<ContentAddress> ca;
 
-        if (fixedOutput) {
+        if (! std::holds_alternative<DerivationOutputInputAddressed>(i.second.output)) {
-
+            DerivationOutputCAFloating outputHash;
-            FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash;
+            std::visit(overloaded {
+                [&](DerivationOutputInputAddressed doi) {
+                    assert(false); // Enclosing `if` handles this case in other branch
+                },
+                [&](DerivationOutputCAFixed dof) {
+                    outputHash = DerivationOutputCAFloating {
+                        .method = dof.hash.method,
+                        .hashType = dof.hash.hash.type,
+                    };
+                },
+                [&](DerivationOutputCAFloating dof) {
+                    outputHash = dof;
+                },
+            }, i.second.output);
 
             if (outputHash.method == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
@@ -3737,12 +3750,17 @@ void DerivationGoal::registerOutputs()
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
             Hash h2 = outputHash.method == FileIngestionMethod::Recursive
-                ? hashPath(outputHash.hash.type, actualPath).first
+                ? hashPath(outputHash.hashType, actualPath).first
-                : hashFile(outputHash.hash.type, actualPath);
+                : hashFile(outputHash.hashType, actualPath);
 
             auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
 
-            if (outputHash.hash != h2) {
+            // true if either floating CA, or incorrect fixed hash.
+            bool needsMove = true;
+
+            if (auto p = std::get_if<DerivationOutputCAFixed>(& i.second.output)) {
+              Hash & h = p->hash.hash;
+              if (h != h2) {
 
                 /* Throw an error after registering the path as
                    valid. */
@@ -3750,9 +3768,15 @@ void DerivationGoal::registerOutputs()
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
                         worker.store.printStorePath(dest),
-                        outputHash.hash.to_string(SRI, true),
+                        h.to_string(SRI, true),
                         h2.to_string(SRI, true)));
+              } else {
+                  // matched the fixed hash, so no move needed.
+                  needsMove = false;
+              }
+            }
 
+            if (needsMove) {
                 Path actualDest = worker.store.Store::toRealPath(dest);
 
                 if (worker.store.isValidPath(dest))

src/libstore/content-address.cc

@@ -26,10 +26,6 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
         + hash.to_string(Base32, true);
 }
 
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
 std::string renderContentAddress(ContentAddress ca) {
     return std::visit(overloaded {
         [](TextHash th) {
@@ -48,14 +44,14 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
     {
         auto optPrefix = splitPrefixTo(rest, ':');
         if (!optPrefix)
-            throw UsageError("not a content address because it is not in the form \"<prefix>:<rest>\": %s", rawCa);
+            throw UsageError("not a content address because it is not in the form '<prefix>:<rest>': %s", rawCa);
         prefix = *optPrefix;
     }
 
     auto parseHashType_ = [&](){
         auto hashTypeRaw = splitPrefixTo(rest, ':');
         if (!hashTypeRaw)
-            throw UsageError("content address hash must be in form \"<algo>:<hash>\", but found: %s", rawCa);
+            throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", rawCa);
         HashType hashType = parseHashType(*hashTypeRaw);
         return std::move(hashType);
     };
@@ -81,7 +77,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
             .hash = Hash::parseNonSRIUnprefixed(rest, std::move(hashType)),
         };
     } else
-        throw UsageError("content address prefix \"%s\" is unrecognized. Recogonized prefixes are \"text\" or \"fixed\"", prefix);
+        throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix);
 };
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {

src/libstore/derivations.cc

@@ -7,23 +7,54 @@
 
 namespace nix {
 
-// FIXME Put this somewhere?
+std::optional<StorePath> DerivationOutput::pathOpt(const Store & store, std::string_view drvName) const
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
-StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
 {
     return std::visit(overloaded {
-        [](DerivationOutputInputAddressed doi) {
+        [](DerivationOutputInputAddressed doi) -> std::optional<StorePath> {
-            return doi.path;
+            return { doi.path };
+        },
+        [&](DerivationOutputCAFixed dof) -> std::optional<StorePath> {
+            return {
+                store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName)
+            };
+        },
+        [](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
+            return std::nullopt;
         },
-        [&](DerivationOutputFixed dof) {
-            return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
-        }
     }, output);
 }
 
 
+bool derivationIsCA(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return false;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return true;
+    };
+    // Since enums can have non-variant values, but making a `default:` would
+    // disable exhaustiveness warnings.
+    assert(false);
+}
+
+bool derivationIsFixed(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return false;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return false;
+    };
+    assert(false);
+}
+
+bool derivationIsImpure(DerivationType dt) {
+    switch (dt) {
+    case DerivationType::InputAddressed: return false;
+    case DerivationType::CAFixed: return true;
+    case DerivationType::CAFloating: return false;
+    };
+    assert(false);
+}
+
+
 bool BasicDerivation::isBuiltin() const
 {
     return string(builder, 0, 8) == "builtin:";
@@ -123,14 +154,22 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings
         }
         const HashType hashType = parseHashType(hashAlgo);
 
-        return DerivationOutput {
+        return hash != ""
-            .output = DerivationOutputFixed {
+            ? DerivationOutput {
+                  .output = DerivationOutputCAFixed {
                       .hash = FixedOutputHash {
                           .method = std::move(method),
                           .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
                       },
                   }
-        };
+               }
+            : (settings.requireExperimentalFeature("ca-derivations"),
+              DerivationOutput {
+                  .output =  DerivationOutputCAFloating {
+                      .method = std::move(method),
+                      .hashType = std::move(hashType),
+                  },
+              });
     } else
         return DerivationOutput {
             .output = DerivationOutputInputAddressed {
@@ -278,13 +317,20 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
         s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
-        if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
+        std::visit(overloaded {
-            s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
+            [&](DerivationOutputInputAddressed doi) {
-            s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
-        } else {
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
-        }
+            },
+            [&](DerivationOutputCAFixed dof) {
+                s += ','; printUnquotedString(s, dof.hash.printMethodAlgo());
+                s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false));
+            },
+            [&](DerivationOutputCAFloating dof) {
+                s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+                s += ','; printUnquotedString(s, "");
+            },
+        }, i.second.output);
         s += ')';
     }
 
@@ -336,60 +382,134 @@ bool isDerivation(const string & fileName)
 }
 
 
-bool BasicDerivation::isFixedOutput() const
+DerivationType BasicDerivation::type() const
 {
-    return outputs.size() == 1 &&
+    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
-        outputs.begin()->first == "out" &&
+    std::optional<HashType> floatingHashType;
-        std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output);
+    for (auto & i : outputs) {
+        std::visit(overloaded {
+            [&](DerivationOutputInputAddressed _) {
+               inputAddressedOutputs.insert(i.first);
+            },
+            [&](DerivationOutputCAFixed _) {
+                fixedCAOutputs.insert(i.first);
+            },
+            [&](DerivationOutputCAFloating dof) {
+                floatingCAOutputs.insert(i.first);
+                if (!floatingHashType) {
+                    floatingHashType = dof.hashType;
+                } else {
+                    if (*floatingHashType != dof.hashType)
+                        throw Error("All floating outputs must use the same hash type");
+                }
+            },
+        }, i.second.output);
+    }
+
+    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+        throw Error("Must have at least one output");
+    } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+        return DerivationType::InputAddressed;
+    } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+        if (fixedCAOutputs.size() > 1)
+            // FIXME: Experimental feature?
+            throw Error("Only one fixed output is allowed for now");
+        if (*fixedCAOutputs.begin() != "out")
+            throw Error("Single fixed output must be named \"out\"");
+        return DerivationType::CAFixed;
+    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
+        return DerivationType::CAFloating;
+    } else {
+        throw Error("Can't mix derivation output types");
+    }
 }
 
 
 DrvHashes drvHashes;
 
+/* pathDerivationModulo and hashDerivationModulo are mutually recursive
+ */
 
-/* Returns the hash of a derivation modulo fixed-output
+/* Look up the derivation by value and memoize the
-   subderivations.  A fixed-output derivation is a derivation with one
+   `hashDerivationModulo` call.
-   output (`out') for which an expected hash and hash algorithm are
+ */
-   specified (using the `outputHash' and `outputHashAlgo'
+static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
-   attributes).  We don't want changes to such derivations to
+{
-   propagate upwards through the dependency graph, changing output
+    auto h = drvHashes.find(drvPath);
-   paths everywhere.
+    if (h == drvHashes.end()) {
+        assert(store.isValidPath(drvPath));
+        // Cache it
+        h = drvHashes.insert_or_assign(
+            drvPath,
+            hashDerivationModulo(
+                store,
+                store.readDerivation(drvPath),
+                false)).first;
+    }
+    return h->second;
+}
 
-   For instance, if we change the url in a call to the `fetchurl'
+/* See the header for interface details. These are the implementation details.
-   function, we do not want to rebuild everything depending on it
-   (after all, (the hash of) the file being downloaded is unchanged).
-   So the *output paths* should not change.  On the other hand, the
-   *derivation paths* should change to reflect the new dependency
-   graph.
 
-   That's what this function does: it returns a hash which is just the
+   For fixed-output derivations, each hash in the map is not the
-   hash of the derivation ATerm, except that any input derivation
+   corresponding output's content hash, but a hash of that hash along
-   paths have been replaced by the result of a recursive call to this
+   with other constant data. The key point is that the value is a pure
-   function, and that for fixed-output derivations we return a hash of
+   function of the output's contents, and there are no preimage attacks
-   its output path. */
+   either spoofing an output's contents for a derivation, or
-Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
+   spoofing a derivation for an output's contents.
+
+   For regular derivations, it looks up each subderivation from its hash
+   and recurs. If the subderivation is also regular, it simply
+   substitutes the derivation path with its hash. If the subderivation
+   is fixed-output, however, it takes each output hash and pretends it
+   is a derivation hash producing a single "out" output. This is so we
+   don't leak the provenance of fixed outputs, reducing pointless cache
+   misses as the build itself won't know this.
+ */
+DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
 {
     /* Return a fixed hash for fixed-output derivations. */
-    if (drv.isFixedOutput()) {
+    switch (drv.type()) {
-        DerivationOutputs::const_iterator i = drv.outputs.begin();
+    case DerivationType::CAFloating:
-        auto hash = std::get<DerivationOutputFixed>(i->second.output);
+        throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
-        return hashString(htSHA256, "fixed:out:"
+    case DerivationType::CAFixed: {
-            + hash.hash.printMethodAlgo() + ":"
+        std::map<std::string, Hash> outputHashes;
-            + hash.hash.hash.to_string(Base16, false) + ":"
+        for (const auto & i : drv.outputs) {
-            + store.printStorePath(i->second.path(store, drv.name)));
+            auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
+            auto hash = hashString(htSHA256, "fixed:out:"
+                + dof.hash.printMethodAlgo() + ":"
+                + dof.hash.hash.to_string(Base16, false) + ":"
+                + store.printStorePath(i.second.path(store, drv.name)));
+            outputHashes.insert_or_assign(i.first, std::move(hash));
+        }
+        return outputHashes;
+    }
+    case DerivationType::InputAddressed:
+        break;
     }
 
     /* For other derivations, replace the inputs paths with recursive
-       calls to this function.*/
+       calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & i : drv.inputDrvs) {
-        auto h = drvHashes.find(i.first);
+        const auto & res = pathDerivationModulo(store, i.first);
-        if (h == drvHashes.end()) {
+        std::visit(overloaded {
-            assert(store.isValidPath(i.first));
+            // Regular non-CA derivation, replace derivation
-            h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store,
+            [&](Hash drvHash) {
-                store.readDerivation(i.first), false)).first;
+                inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
+            },
+            // CA derivation's output hashes
+            [&](CaOutputHashes outputHashes) {
+                std::set<std::string> justOut = { "out" };
+                for (auto & output : i.second) {
+                    /* Put each one in with a single "out" output.. */
+                    const auto h = outputHashes.at(output);
+                    inputs2.insert_or_assign(
+                        h.to_string(Base16, false),
+                        justOut);
                 }
-        inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
+            },
+        }, res);
     }
 
     return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@@ -431,14 +551,22 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
             hashAlgo = string(hashAlgo, 2);
         }
         auto hashType = parseHashType(hashAlgo);
-        return DerivationOutput {
+        return hash != ""
-            .output = DerivationOutputFixed {
+            ? DerivationOutput {
+                  .output = DerivationOutputCAFixed {
                       .hash = FixedOutputHash {
                           .method = std::move(method),
                           .hash = Hash::parseNonSRIUnprefixed(hash, hashType),
                       },
                   }
-        };
+               }
+            : (settings.requireExperimentalFeature("ca-derivations"),
+              DerivationOutput {
+                  .output = DerivationOutputCAFloating {
+                      .method = std::move(method),
+                      .hashType = std::move(hashType),
+                  },
+              });
     } else
         return DerivationOutput {
             .output = DerivationOutputInputAddressed {
@@ -498,12 +626,19 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
     for (auto & i : drv.outputs) {
         out << i.first
             << store.printStorePath(i.second.path(store, drv.name));
-        if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
+        std::visit(overloaded {
-            out << hash->hash.printMethodAlgo()
+            [&](DerivationOutputInputAddressed doi) {
-                << hash->hash.hash.to_string(Base16, false);
-        } else {
                 out << "" << "";
-        }
+            },
+            [&](DerivationOutputCAFixed dof) {
+                out << dof.hash.printMethodAlgo()
+                    << dof.hash.hash.to_string(Base16, false);
+            },
+            [&](DerivationOutputCAFloating dof) {
+                out << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
+                    << "";
+            },
+        }, i.second.output);
     }
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;

src/libstore/derivations.hh

@@ -6,6 +6,7 @@
 #include "content-address.hh"
 
 #include <map>
+#include <variant>
 
 
 namespace nix {
@@ -13,20 +14,46 @@ namespace nix {
 
 /* Abstract syntax of derivations. */
 
+/* The traditional non-fixed-output derivation type. */
 struct DerivationOutputInputAddressed
 {
+    /* Will need to become `std::optional<StorePath>` once input-addressed
+       derivations are allowed to depend on cont-addressed derivations */
     StorePath path;
 };
 
-struct DerivationOutputFixed
+/* Fixed-output derivations, whose output paths are content addressed
+   according to that fixed output. */
+struct DerivationOutputCAFixed
 {
     FixedOutputHash hash; /* hash used for expected hash computation */
 };
 
+/* Floating-output derivations, whose output paths are content addressed, but
+   not fixed, and so are dynamically calculated from whatever the output ends
+   up being. */
+struct DerivationOutputCAFloating
+{
+    /* information used for expected hash computation */
+    FileIngestionMethod method;
+    HashType hashType;
+};
+
 struct DerivationOutput
 {
-    std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output;
+    std::variant<
-    StorePath path(const Store & store, std::string_view drvName) const;
+        DerivationOutputInputAddressed,
+        DerivationOutputCAFixed,
+        DerivationOutputCAFloating
+    > output;
+    std::optional<HashType> hashAlgoOpt(const Store & store) const;
+    std::optional<StorePath> pathOpt(const Store & store, std::string_view drvName) const;
+    /* DEPRECATED: Remove after CA drvs are fully implemented */
+    StorePath path(const Store & store, std::string_view drvName) const {
+        auto p = pathOpt(store, drvName);
+        if (!p) throw UnimplementedError("floating content-addressed derivations are not yet implemented");
+        return *p;
+    }
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -37,6 +64,25 @@ typedef std::map<StorePath, StringSet> DerivationInputs;
 
 typedef std::map<string, string> StringPairs;
 
+enum struct DerivationType : uint8_t {
+    InputAddressed,
+    CAFixed,
+    CAFloating,
+};
+
+/* Do the outputs of the derivation have paths calculated from their content,
+   or from the derivation itself? */
+bool derivationIsCA(DerivationType);
+
+/* Is the content of the outputs fixed a-priori via a hash? Never true for
+   non-CA derivations. */
+bool derivationIsFixed(DerivationType);
+
+/* Is the derivation impure and needs to access non-deterministic resources, or
+   pure and can be sandboxed? Note that whether or not we actually sandbox the
+   derivation is controlled separately. Never true for non-CA derivations. */
+bool derivationIsImpure(DerivationType);
+
 struct BasicDerivation
 {
     DerivationOutputs outputs; /* keyed on symbolic IDs */
@@ -53,7 +99,7 @@ struct BasicDerivation
     bool isBuiltin() const;
 
     /* Return true iff this is a fixed-output derivation. */
-    bool isFixedOutput() const;
+    DerivationType type() const;
 
     /* Return the output paths of a derivation. */
     StorePathSet outputPaths(const Store & store) const;
@@ -90,10 +136,42 @@ Derivation readDerivation(const Store & store, const Path & drvPath, std::string
 // FIXME: remove
 bool isDerivation(const string & fileName);
 
-Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
+// known CA drv's output hashes, current just for fixed-output derivations
+// whose output hashes are always known since they are fixed up-front.
+typedef std::map<std::string, Hash> CaOutputHashes;
+
+typedef std::variant<
+    Hash, // regular DRV normalized hash
+    CaOutputHashes
+> DrvHashModulo;
+
+/* Returns hashes with the details of fixed-output subderivations
+   expunged.
+
+   A fixed-output derivation is a derivation whose outputs have a
+   specified content hash and hash algorithm. (Currently they must have
+   exactly one output (`out'), which is specified using the `outputHash'
+   and `outputHashAlgo' attributes, but the algorithm doesn't assume
+   this.) We don't want changes to such derivations to propagate upwards
+   through the dependency graph, changing output paths everywhere.
+
+   For instance, if we change the url in a call to the `fetchurl'
+   function, we do not want to rebuild everything depending on it---after
+   all, (the hash of) the file being downloaded is unchanged.  So the
+   *output paths* should not change. On the other hand, the *derivation
+   paths* should change to reflect the new dependency graph.
+
+   For fixed-output derivations, this returns a map from the name of
+   each output to its hash, unique up to the output's contents.
+
+   For regular derivations, it returns a single hash of the derivation
+   ATerm, after subderivations have been likewise expunged from that
+   derivation.
+ */
+DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
 
 /* Memoisation of hashDerivationModulo(). */
-typedef std::map<StorePath, Hash> DrvHashes;
+typedef std::map<StorePath, DrvHashModulo> DrvHashes;
 
 extern DrvHashes drvHashes; // FIXME: global, not thread-safe
 

src/libstore/local-store.cc

@@ -544,11 +544,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     std::string drvName(drvPath.name());
     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 
-    auto check = [&](const StorePath & expected, const StorePath & actual, const std::string & varName)
+    auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
     {
-        if (actual != expected)
-            throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                printStorePath(drvPath), printStorePath(actual), printStorePath(expected));
         auto j = drv.env.find(varName);
         if (j == drv.env.end() || parseStorePath(j->second) != actual)
             throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
@@ -556,16 +553,34 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
     };
 
 
-    if (drv.isFixedOutput()) {
+    // Don't need the answer, but do this anyways to assert is proper
-        DerivationOutputs::const_iterator out = drv.outputs.find("out");
+    // combination. The code below is more general and naturally allows
-        if (out == drv.outputs.end())
+    // combinations that are currently prohibited.
-            throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
+    drv.type();
-    }
 
-    else {
+    std::optional<Hash> h;
-        Hash h = hashDerivationModulo(*this, drv, true);
+    for (auto & i : drv.outputs) {
-        for (auto & i : drv.outputs)
+        std::visit(overloaded {
-            check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
+            [&](DerivationOutputInputAddressed doia) {
+                if (!h) {
+                    // somewhat expensive so we do lazily
+                    auto temp = hashDerivationModulo(*this, drv, true);
+                    h = std::get<Hash>(temp);
+                }
+                StorePath recomputed = makeOutputPath(i.first, *h, drvName);
+                if (doia.path != recomputed)
+                    throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                        printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
+                envHasRightPath(doia.path, i.first);
+            },
+            [&](DerivationOutputCAFixed dof) {
+                StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
+                envHasRightPath(path, i.first);
+            },
+            [&](DerivationOutputCAFloating _) {
+                throw UnimplementedError("floating CA output derivations are not yet implemented");
+            },
+        }, i.second.output);
     }
 }
 

src/libstore/misc.cc

@@ -4,6 +4,7 @@
 #include "local-store.hh"
 #include "store-api.hh"
 #include "thread-pool.hh"
+#include "topo-sort.hh"
 
 
 namespace nix {
@@ -112,7 +113,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
 {
     auto out = drv.outputs.find("out");
     if (out != drv.outputs.end()) {
-        if (auto v = std::get_if<DerivationOutputFixed>(&out->second.output))
+        if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output))
             return v->hash;
     }
     return std::nullopt;
@@ -256,41 +257,21 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
 
 StorePaths Store::topoSortPaths(const StorePathSet & paths)
 {
-    StorePaths sorted;
+    return topoSort(paths,
-    StorePathSet visited, parents;
+        {[&](const StorePath & path) {
-
-    std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit;
-
-    dfsVisit = [&](const StorePath & path, const StorePath * parent) {
-        if (parents.count(path))
-            throw BuildError("cycle detected in the references of '%s' from '%s'",
-                printStorePath(path), printStorePath(*parent));
-
-        if (!visited.insert(path).second) return;
-        parents.insert(path);
-
             StorePathSet references;
             try {
                 references = queryPathInfo(path)->references;
             } catch (InvalidPath &) {
             }
-
+            return references;
-        for (auto & i : references)
+        }},
-            /* Don't traverse into paths that don't exist.  That can
+        {[&](const StorePath & path, const StorePath & parent) {
-               happen due to substitutes for non-existent paths. */
+            return BuildError(
-            if (i != path && paths.count(i))
+                "cycle detected in the references of '%s' from '%s'",
-                dfsVisit(i, &path);
+                printStorePath(path),
-
+                printStorePath(parent));
-        sorted.push_back(path);
+        }});
-        parents.erase(path);
-    };
-
-    for (auto & i : paths)
-        dfsVisit(i, nullptr);
-
-    std::reverse(sorted.begin(), sorted.end());
-
-    return sorted;
 }
 
 

src/libstore/parsed-derivations.cc

@@ -117,9 +117,4 @@ bool ParsedDerivation::substitutesAllowed() const
     return getBoolAttr("allowSubstitutes", true);
 }
 
-bool ParsedDerivation::contentAddressed() const
-{
-    return getBoolAttr("__contentAddressed", false);
-}
-
 }

src/libstore/parsed-derivations.hh

@@ -34,8 +34,6 @@ public:
     bool willBuildLocally() const;
 
     bool substitutesAllowed() const;
-
-    bool contentAddressed() const;
 };
 
 }

src/libstore/store-api.cc

@@ -193,10 +193,6 @@ StorePath Store::makeFixedOutputPath(
     }
 }
 
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
 StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
     const StorePathSet & references, bool hasSelfReference) const
 {

src/libutil/error.hh

@@ -192,6 +192,7 @@ public:
 
 MakeError(Error, BaseError);
 MakeError(UsageError, Error);
+MakeError(UnimplementedError, Error);
 
 class SysError : public Error
 {

src/libutil/topo-sort.hh

@@ -0,0 +1,40 @@
+#include "error.hh"
+
+namespace nix {
+
+template<typename T>
+std::vector<T> topoSort(std::set<T> items,
+        std::function<std::set<T>(const T &)> getChildren,
+        std::function<Error(const T &, const T &)> makeCycleError)
+{
+    std::vector<T> sorted;
+    std::set<T> visited, parents;
+
+    std::function<void(const T & path, const T * parent)> dfsVisit;
+
+    dfsVisit = [&](const T & path, const T * parent) {
+        if (parents.count(path)) throw makeCycleError(path, *parent);
+
+        if (!visited.insert(path).second) return;
+        parents.insert(path);
+
+        std::set<T> references = getChildren(path);
+
+        for (auto & i : references)
+            /* Don't traverse into items that don't exist in our starting set. */
+            if (i != path && items.count(i))
+                dfsVisit(i, &path);
+
+        sorted.push_back(path);
+        parents.erase(path);
+    };
+
+    for (auto & i : items)
+        dfsVisit(i, nullptr);
+
+    std::reverse(sorted.begin(), sorted.end());
+
+    return sorted;
+}
+
+}

src/libutil/util.hh

@@ -601,4 +601,9 @@ constexpr auto enumerate(T && iterable)
 }
 
 
+// C++17 std::visit boilerplate
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+
 }

src/nix/develop.cc

@@ -135,7 +135,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.env["_outputs_saved"] = drv.env["outputs"];
     drv.env["outputs"] = "out";
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    Hash h = hashDerivationModulo(*store, drv, true);
+    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
     drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
                 .path = shellOutPath

src/nix/show-derivation.cc

@@ -70,10 +70,18 @@ struct CmdShowDerivation : InstallablesCommand
                 for (auto & output : drv.outputs) {
                     auto outputObj(outputsObj.object(output.first));
                     outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
-                    if (auto hash = std::get_if<DerivationOutputFixed>(&output.second.output)) {
+
-                        outputObj.attr("hashAlgo", hash->hash.printMethodAlgo());
+                    std::visit(overloaded {
-                        outputObj.attr("hash", hash->hash.hash.to_string(Base16, false));
+                        [&](DerivationOutputInputAddressed doi) {
-                    }
+                        },
+                        [&](DerivationOutputCAFixed dof) {
+                            outputObj.attr("hashAlgo", dof.hash.printMethodAlgo());
+                            outputObj.attr("hash", dof.hash.hash.to_string(Base16, false));
+                        },
+                        [&](DerivationOutputCAFloating dof) {
+                            outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
+                        },
+                    }, output.second.output);
                 }
             }