Merge remote-tracking branch 'upstream/master' into multi-output-hashDerivationModulo

doc/manual/advanced-topics/diff-hook.xml

@@ -70,7 +70,7 @@ path just built.</para>
 
   <screen>
 $ nix-build ./deterministic.nix -A stable
-these derivations will be built:
+this derivation will be built:
   /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv
 building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable
@@ -85,7 +85,7 @@ checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'...
 
   <screen>
 $ nix-build ./deterministic.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv
 building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'...
 /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable
@@ -193,7 +193,7 @@ repeat = 1
     An example output of this configuration:
     <screen>
 $ nix-build ./test.nix -A unstable
-these derivations will be built:
+this derivation will be built:
   /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)...
 building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)...

doc/manual/advanced-topics/post-build-hook.xml

@@ -122,7 +122,7 @@ post-build-hook = /etc/nix/upload-to-cache.sh
 
   <screen>
 $ nix-build -E '(import &lt;nixpkgs&gt; {}).writeText "example" (builtins.toString builtins.currentTime)'
-these derivations will be built:
+this derivation will be built:
   /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv
 building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'...
 running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'...

doc/manual/command-ref/nix-env.xml

@@ -516,7 +516,7 @@ source:
 $ nix-env -f '<nixpkgs>' -iA hello --dry-run
 (dry run; not doing anything)
 installing ‘hello-2.10’
-these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked):
+this path will be fetched (0.04 MiB download, 0.19 MiB unpacked):
   /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10
   <replaceable>...</replaceable></screen>
 

doc/manual/installation/env-variables.xml

@@ -39,7 +39,7 @@ bundle.</para>
   <step><para>Set the environment variable and install Nix</para>
     <screen>
 $ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt
-$ sh &lt;(curl https://nixos.org/nix/install)
+$ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen></step>
 
   <step><para>In the shell profile and rc files (for example,

doc/manual/installation/installing-binary.xml

@@ -12,7 +12,7 @@
 </para>
 
 <screen>
-  $ sh &lt;(curl https://nixos.org/nix/install)
+  $ sh &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
 <para>
@@ -39,7 +39,7 @@
     To explicitly select a single-user installation on your system:
 
     <screen>
-  sh &lt;(curl https://nixos.org/nix/install) --no-daemon
+  sh &lt;(curl -L https://nixos.org/nix/install) --no-daemon
 </screen>
   </para>
 

doc/manual/introduction/quick-start.xml

@@ -15,7 +15,7 @@ to subsequent chapters.</para>
 <step><para>Install single-user Nix by running the following:
 
 <screen>
-$ bash &lt;(curl https://nixos.org/nix/install)
+$ bash &lt;(curl -L https://nixos.org/nix/install)
 </screen>
 
 This will install Nix in <filename>/nix</filename>. The install script

shell.nix

@@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; };
 (if useClang then clangStdenv else stdenv).mkDerivation {
   name = "nix";
 
-  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps ++ [ pkgs.rustfmt ];
+  buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
 
   inherit configureFlags;
 

src/libexpr/attr-path.cc

@@ -6,11 +6,11 @@
 namespace nix {
 
 
-static Strings parseAttrPath(const string & s)
+static Strings parseAttrPath(std::string_view s)
 {
     Strings res;
     string cur;
-    string::const_iterator i = s.begin();
+    auto i = s.begin();
     while (i != s.end()) {
         if (*i == '.') {
             res.push_back(cur);
@@ -32,6 +32,15 @@ static Strings parseAttrPath(const string & s)
 }
 
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s)
+{
+    std::vector<Symbol> res;
+    for (auto & a : parseAttrPath(s))
+        res.push_back(state.symbols.create(a));
+    return res;
+}
+
+
 std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath,
     Bindings & autoArgs, Value & vIn)
 {

src/libexpr/attr-path.hh

@@ -16,4 +16,6 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
 /* Heuristic to find the filename and lineno or a nix value. */
 Pos findDerivationFilename(EvalState & state, Value & v, std::string what);
 
+std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s);
+
 }

src/libexpr/eval.cc

@@ -161,12 +161,12 @@ const Value *getPrimOp(const Value &v) {
 }
 
 
-string showType(const Value & v)
+string showType(ValueType type)
 {
-    switch (v.type) {
+    switch (type) {
         case tInt: return "an integer";
-        case tBool: return "a boolean";
+        case tBool: return "a Boolean";
-        case tString: return v.string.context ? "a string with context" : "a string";
+        case tString: return "a string";
         case tPath: return "a path";
         case tNull: return "null";
         case tAttrs: return "a set";
@@ -175,14 +175,27 @@ string showType(const Value & v)
         case tApp: return "a function application";
         case tLambda: return "a function";
         case tBlackhole: return "a black hole";
+        case tPrimOp: return "a built-in function";
+        case tPrimOpApp: return "a partially applied built-in function";
+        case tExternal: return "an external value";
+        case tFloat: return "a float";
+    }
+    abort();
+}
+
+
+string showType(const Value & v)
+{
+    switch (v.type) {
+        case tString: return v.string.context ? "a string with context" : "a string";
         case tPrimOp:
             return fmt("the built-in function '%s'", string(v.primOp->name));
         case tPrimOpApp:
             return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
         case tExternal: return v.external->showType();
-        case tFloat: return "a float";
+    default:
+        return showType(v.type);
     }
-    abort();
 }
 
 
@@ -323,6 +336,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sOutputHash(symbols.create("outputHash"))
     , sOutputHashAlgo(symbols.create("outputHashAlgo"))
     , sOutputHashMode(symbols.create("outputHashMode"))
+    , sRecurseForDerivations(symbols.create("recurseForDerivations"))
     , repair(NoRepair)
     , store(store)
     , baseEnv(allocEnv(128))
@@ -471,14 +485,21 @@ Value * EvalState::addConstant(const string & name, Value & v)
 Value * EvalState::addPrimOp(const string & name,
     size_t arity, PrimOpFun primOp)
 {
+    auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
+    Symbol sym = symbols.create(name2);
+
+    /* Hack to make constants lazy: turn them into a application of
+       the primop to a dummy value. */
     if (arity == 0) {
+        auto vPrimOp = allocValue();
+        vPrimOp->type = tPrimOp;
+        vPrimOp->primOp = new PrimOp(primOp, 1, sym);
         Value v;
-        primOp(*this, noPos, nullptr, v);
+        mkApp(v, *vPrimOp, *vPrimOp);
         return addConstant(name, v);
     }
+
     Value * v = allocValue();
-    string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name;
-    Symbol sym = symbols.create(name2);
     v->type = tPrimOp;
     v->primOp = new PrimOp(primOp, arity, sym);
     staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;

src/libexpr/eval.hh

@@ -18,7 +18,7 @@ namespace nix {
 
 class Store;
 class EvalState;
-struct StorePath;
+class StorePath;
 enum RepairFlag : bool;
 
 
@@ -74,7 +74,8 @@ public:
         sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
         sFile, sLine, sColumn, sFunctor, sToString,
         sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
-        sOutputHash, sOutputHashAlgo, sOutputHashMode;
+        sOutputHash, sOutputHashAlgo, sOutputHashMode,
+        sRecurseForDerivations;
     Symbol sDerivationNix;
 
     /* If set, force copying files to the Nix store even if they
@@ -324,6 +325,7 @@ private:
 
 
 /* Return a string representing the type of the value `v'. */
+string showType(ValueType type);
 string showType(const Value & v);
 
 /* Decode a context string ‘!<name>!<path>’ into a pair <path,

src/libexpr/get-drvs.cc

@@ -348,7 +348,7 @@ static void getDerivations(EvalState & state, Value & vIn,
                    should we recurse into it?  => Only if it has a
                    `recurseForDerivations = true' attribute. */
                 if (i->value->type == tAttrs) {
-                    Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations"));
+                    Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
                     if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
                         getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
                 }

src/libexpr/primops.cc

@@ -50,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context)
     std::vector<StorePathWithOutputs> drvs;
 
     for (auto & i : context) {
-        std::pair<string, string> decoded = decodeContext(i);
+        auto [ctxS, outputName] = decodeContext(i);
-        auto ctx = store->parseStorePath(decoded.first);
+        auto ctx = store->parseStorePath(ctxS);
         if (!store->isValidPath(ctx))
             throw InvalidPathError(store->printStorePath(ctx));
-        if (!decoded.second.empty() && ctx.isDerivation()) {
+        if (!outputName.empty() && ctx.isDerivation()) {
-            drvs.push_back(StorePathWithOutputs{ctx, {decoded.second}});
+            drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
 
             /* Add the output of this derivation to the allowed
                paths. */
             if (allowedPaths) {
-                auto drv = store->derivationFromPath(store->parseStorePath(decoded.first));
+                auto drv = store->derivationFromPath(ctx);
-                DerivationOutputs::iterator i = drv.outputs.find(decoded.second);
+                DerivationOutputs::iterator i = drv.outputs.find(outputName);
                 if (i == drv.outputs.end())
-                    throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second);
+                    throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
                 allowedPaths->insert(store->printStorePath(i->second.path));
             }
         }
@@ -79,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context)
     StorePathSet willBuild, willSubstitute, unknown;
     unsigned long long downloadSize, narSize;
     store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+
     store->buildPaths(drvs);
 }
 
@@ -768,17 +769,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 .nixCode = NixCode { .errPos = posDrvName }
             });
 
-        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
+        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
-
         Hash h = newHashAllowEmpty(*outputHash, ht);
 
         auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
-            std::move(outPath),
+            .path = std::move(outPath),
-            (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
+            .hash = DerivationOutputHash {
-                + printHashType(h.type),
+                .method = ingestionMethod,
-            h.to_string(Base16, false),
+                .hash = std::move(h),
+            },
         });
     }
 
@@ -792,7 +793,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput { StorePath::dummy, "", "" });
+                DerivationOutput {
+                    .path = StorePath::dummy,
+                    .hash = std::optional<DerivationOutputHash> {},
+                });
         }
 
         // Regular, non-CA derivation should always return a single hash and not
@@ -803,7 +807,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             auto outPath = state.store->makeOutputPath(i, h, drvName);
             if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
             drv.outputs.insert_or_assign(i,
-                DerivationOutput { std::move(outPath), "", "" });
+                DerivationOutput {
+                    .path = std::move(outPath),
+                    .hash = std::optional<DerivationOutputHash>(),
+                });
         }
     }
 
@@ -1000,8 +1007,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
+    std::optional<HashType> ht = parseHashType(type);
-    if (ht == htUnknown)
+    if (!ht)
       throw Error({
           .hint = hintfmt("unknown hash type '%1%'", type),
           .nixCode = NixCode { .errPos = pos }
@@ -1010,7 +1017,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
 
-    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+    mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
 }
 
 /* Read a directory (without . or ..) */
@@ -1937,8 +1944,8 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
+    std::optional<HashType> ht = parseHashType(type);
-    if (ht == htUnknown)
+    if (!ht)
         throw Error({
             .hint = hintfmt("unknown hash type '%1%'", type),
             .nixCode = NixCode { .errPos = pos }
@@ -1947,7 +1954,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, hashString(ht, s).to_string(Base16, false), context);
+    mkString(v, hashString(*ht, s).to_string(Base16, false), context);
 }
 
 
@@ -2203,10 +2210,11 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args
 RegisterPrimOp::PrimOps * RegisterPrimOp::primOps;
 
 
-RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
+RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun,
+    std::optional<std::string> requiredFeature)
 {
     if (!primOps) primOps = new PrimOps;
-    primOps->emplace_back(name, arity, fun);
+    primOps->push_back({name, arity, fun, requiredFeature});
 }
 
 
@@ -2398,7 +2406,8 @@ void EvalState::createBaseEnv()
 
     if (RegisterPrimOp::primOps)
         for (auto & primOp : *RegisterPrimOp::primOps)
-            addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp));
+            if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature))
+                addPrimOp(primOp.name, primOp.arity, primOp.primOp);
 
     /* Now that we've added all primops, sort the `builtins' set,
        because attribute lookups expect it to be sorted. */

src/libexpr/primops.hh

@@ -7,12 +7,25 @@ namespace nix {
 
 struct RegisterPrimOp
 {
-    typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps;
+    struct Info
+    {
+        std::string name;
+        size_t arity;
+        PrimOpFun primOp;
+        std::optional<std::string> requiredFeature;
+    };
+
+    typedef std::vector<Info> PrimOps;
     static PrimOps * primOps;
+
     /* You can register a constant by passing an arity of 0. fun
        will get called during EvalState initialization, so there
        may be primops not yet added and builtins is not yet sorted. */
-    RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun);
+    RegisterPrimOp(
+        std::string name,
+        size_t arity,
+        PrimOpFun fun,
+        std::optional<std::string> requiredFeature = {});
 };
 
 /* These primops are disabled without enableNativeCode, but plugins

src/libfetchers/fetchers.cc

@@ -36,7 +36,7 @@ std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
         if (res) {
             if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
                 // FIXME: require SRI hash.
-                res->narHash = newHashAllowEmpty(*narHash, htUnknown);
+                res->narHash = newHashAllowEmpty(*narHash, {});
             return res;
         }
     }

src/libfetchers/tarball.cc

@@ -264,7 +264,7 @@ struct TarballInputScheme : InputScheme
 
         auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
         if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            input->hash = newHashAllowEmpty(*hash, htUnknown);
+            input->hash = newHashAllowEmpty(*hash, {});
 
         return input;
     }

src/libmain/shared.cc

@@ -48,7 +48,10 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
 {
     if (!willBuild.empty()) {
-        printMsg(lvl, "these derivations will be built:");
+        if (willBuild.size() == 1)
+            printMsg(lvl, fmt("this derivation will be built:"));
+        else
+            printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size()));
         auto sorted = store->topoSortPaths(willBuild);
         reverse(sorted.begin(), sorted.end());
         for (auto & i : sorted)
@@ -56,9 +59,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild,
     }
 
     if (!willSubstitute.empty()) {
-        printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+        const float downloadSizeMiB = downloadSize / (1024.f * 1024.f);
-                downloadSize / (1024.0 * 1024.0),
+        const float narSizeMiB = narSize / (1024.f * 1024.f);
-                narSize / (1024.0 * 1024.0)));
+        if (willSubstitute.size() == 1) {
+            printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                downloadSizeMiB,
+                narSizeMiB));
+        } else {
+            printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
+                willSubstitute.size(),
+                downloadSizeMiB,
+                narSizeMiB));
+        }
         for (auto & i : willSubstitute)
             printMsg(lvl, fmt("  %s", store->printStorePath(i)));
     }

src/libstore/binary-cache-store.cc

@@ -388,8 +388,6 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe
 
     narInfo->sigs.insert(sigs.begin(), sigs.end());
 
-    auto narInfoFile = narInfoFileFor(narInfo->path);
-
     writeNarInfo(narInfo);
 }
 

src/libstore/build.cc

@@ -86,7 +86,7 @@ struct HookInstance;
 
 
 /* A pointer to a goal. */
-class Goal;
+struct Goal;
 class DerivationGoal;
 typedef std::shared_ptr<Goal> GoalPtr;
 typedef std::weak_ptr<Goal> WeakGoalPtr;
@@ -1195,6 +1195,12 @@ void DerivationGoal::haveDerivation()
 
     parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
 
+    if (parsedDrv->contentAddressed()) {
+        settings.requireExperimentalFeature("ca-derivations");
+        throw Error("ca-derivations isn't implemented yet");
+    }
+
+
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
@@ -3712,10 +3718,7 @@ void DerivationGoal::registerOutputs()
 
         if (fixedOutput) {
 
-            FileIngestionMethod outputHashMode; Hash h;
+            if (i.second.hash->method == FileIngestionMethod::Flat) {
-            i.second.parseHashInfo(outputHashMode, h);
-
-            if (outputHashMode == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
@@ -3726,20 +3729,22 @@ void DerivationGoal::registerOutputs()
 
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
-            Hash h2 = outputHashMode == FileIngestionMethod::Recursive
+            Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
-                ? hashPath(h.type, actualPath).first
+                ? hashPath(*i.second.hash->hash.type, actualPath).first
-                : hashFile(h.type, actualPath);
+                : hashFile(*i.second.hash->hash.type, actualPath);
 
-            auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
+            auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
 
-            if (h != h2) {
+            if (i.second.hash->hash != h2) {
 
                 /* Throw an error after registering the path as
                    valid. */
                 worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
+                        worker.store.printStorePath(dest),
+                        i.second.hash->hash.to_string(SRI, true),
+                        h2.to_string(SRI, true)));
 
                 Path actualDest = worker.store.Store::toRealPath(dest);
 
@@ -3759,7 +3764,7 @@ void DerivationGoal::registerOutputs()
             else
                 assert(worker.store.parseStorePath(path) == dest);
 
-            ca = makeFixedOutputCA(outputHashMode, h2);
+            ca = makeFixedOutputCA(i.second.hash->method, h2);
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -4992,7 +4997,7 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
+        HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
         Hash nullHash(htSHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }

src/libstore/builtins/fetchurl.cc

@@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashType(getAttr("outputHashAlgo"));
+                auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
                 auto h = Hash(getAttr("outputHash"), ht);
-                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+                fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());

src/libstore/derivations.cc

@@ -8,22 +8,8 @@
 
 namespace nix {
 
-
+std::string DerivationOutputHash::printMethodAlgo() const {
-void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
+    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
-{
-    recursive = FileIngestionMethod::Flat;
-    string algo = hashAlgo;
-
-    if (string(algo, 0, 2) == "r:") {
-        recursive = FileIngestionMethod::Recursive;
-        algo = string(algo, 2);
-    }
-
-    HashType hashType = parseHashType(algo);
-    if (hashType == htUnknown)
-        throw Error("unknown hash algorithm '%s'", algo);
-
-    hash = Hash(this->hash, hashType);
 }
 
 
@@ -120,6 +106,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
+static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
+{
+    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+    expect(str, ","); auto hashAlgo = parseString(str);
+    expect(str, ","); const auto hash = parseString(str);
+    expect(str, ")");
+
+    std::optional<DerivationOutputHash> fsh;
+    if (hashAlgo != "") {
+        auto method = FileIngestionMethod::Flat;
+        if (string(hashAlgo, 0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = string(hashAlgo, 2);
+        }
+        const HashType hashType = parseHashType(hashAlgo);
+        fsh = DerivationOutputHash {
+            .method = std::move(method),
+            .hash = Hash(hash, hashType),
+        };
+    }
+
+    return DerivationOutput {
+        .path = std::move(path),
+        .hash = std::move(fsh),
+    };
+}
+
+
 static Derivation parseDerivation(const Store & store, const string & s)
 {
     Derivation drv;
@@ -129,15 +143,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
     /* Parse the list of outputs. */
     while (!endOfList(str)) {
         expect(str, "("); std::string id = parseString(str);
-        expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+        auto output = parseDerivationOutput(store, str);
-        expect(str, ","); auto hashAlgo = parseString(str);
+        drv.outputs.emplace(std::move(id), std::move(output));
-        expect(str, ","); auto hash = parseString(str);
-        expect(str, ")");
-        drv.outputs.emplace(id, DerivationOutput {
-            .path = std::move(path),
-            .hashAlgo = std::move(hashAlgo),
-            .hash = std::move(hash)
-        });
     }
 
     /* Parse the list of input derivations. */
@@ -263,8 +270,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
         s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printUnquotedString(s, i.second.hashAlgo);
+        s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
-        s += ','; printUnquotedString(s, i.second.hash);
+        s += ','; printUnquotedString(s,
+            i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
         s += ')';
     }
 
@@ -320,7 +328,7 @@ bool BasicDerivation::isFixedOutput() const
 {
     return outputs.size() == 1 &&
         outputs.begin()->first == "out" &&
-        outputs.begin()->second.hash != "";
+        outputs.begin()->second.hash;
 }
 
 
@@ -332,7 +340,7 @@ DrvHashes drvHashes;
 /* Look up the derivation by value and memoize the
    `hashDerivationModulo` call.
  */
-static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
+static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
 {
     auto h = drvHashes.find(drvPath);
     if (h == drvHashes.end()) {
@@ -348,6 +356,10 @@ static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drv
     return h->second;
 }
 
+// FIXME: Boilerplatflate for `std::visit`, put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
 /* See the header for interface details. These are the implementation details.
 
    For fixed-output derivations, each hash in the map is not the
@@ -372,8 +384,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
         std::map<std::string, Hash> outputHashes;
         for (const auto & i : drv.outputs) {
             const Hash h = hashString(htSHA256, "fixed:out:"
-                + i.second.hashAlgo + ":"
+                + i.second.hash->printMethodAlgo() + ":"
-                + i.second.hash + ":"
+                + i.second.hash->hash.to_string(Base16, false) + ":"
                 + store.printStorePath(i.second.path));
             outputHashes.insert_or_assign(std::string(i.first), std::move(h));
         }
@@ -384,21 +396,24 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
        calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & i : drv.inputDrvs) {
-        const auto res = pathDerivationModulo(store, i.first);
+        const auto & res = pathDerivationModulo(store, i.first);
-        if (const Hash *pval = std::get_if<0>(&res)) {
+        std::visit(overloaded {
-            // regular non-CA derivation, replace derivation
+            // Regular non-CA derivation, replace derivation
-            inputs2.insert_or_assign(pval->to_string(Base16, false), i.second);
+            [&](Hash drvHash) {
-        } else if (const std::map<std::string, Hash> *pval = std::get_if<1>(&res)) {
+                inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
+            },
             // CA derivation's output hashes
+            [&](CaOutputHashes outputHashes) {
                 std::set justOut = { std::string("out") };
                 for (auto & output : i.second) {
                     /* Put each one in with a single "out" output.. */
-                const auto h = pval->at(output);
+                    const auto h = outputHashes.at(output);
                     inputs2.insert_or_assign(
                         h.to_string(Base16, false),
                         justOut);
                 }
-        }
+            },
+        }, res);
     }
 
     return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
@@ -427,6 +442,31 @@ StorePathSet BasicDerivation::outputPaths() const
     return paths;
 }
 
+static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+{
+    auto path = store.parseStorePath(readString(in));
+    auto hashAlgo = readString(in);
+    const auto hash = readString(in);
+
+    std::optional<DerivationOutputHash> fsh;
+    if (hashAlgo != "") {
+        auto method = FileIngestionMethod::Flat;
+        if (string(hashAlgo, 0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = string(hashAlgo, 2);
+        }
+        const HashType hashType = parseHashType(hashAlgo);
+        fsh = DerivationOutputHash {
+            .method = std::move(method),
+            .hash = Hash(hash, hashType),
+        };
+    }
+
+    return DerivationOutput {
+        .path = std::move(path),
+        .hash = std::move(fsh),
+    };
+}
 
 StringSet BasicDerivation::outputNames() const
 {
@@ -443,14 +483,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
     auto nr = readNum<size_t>(in);
     for (size_t n = 0; n < nr; n++) {
         auto name = readString(in);
-        auto path = store.parseStorePath(readString(in));
+        auto output = readDerivationOutput(in, store);
-        auto hashAlgo = readString(in);
+        drv.outputs.emplace(std::move(name), std::move(output));
-        auto hash = readString(in);
-        drv.outputs.emplace(name, DerivationOutput {
-            .path = std::move(path),
-            .hashAlgo = std::move(hashAlgo),
-            .hash = std::move(hash)
-        });
     }
 
     drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@@ -472,7 +506,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs)
-        out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
+        out << i.first
+            << store.printStorePath(i.second.path)
+            << i.second.hash->printMethodAlgo()
+            << i.second.hash->hash.to_string(Base16, false);
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();

src/libstore/derivations.hh

@@ -13,11 +13,17 @@ namespace nix {
 
 /* Abstract syntax of derivations. */
 
+/// Pair of a hash, and how the file system was ingested
+struct DerivationOutputHash {
+    FileIngestionMethod method;
+    Hash hash;
+    std::string printMethodAlgo() const;
+};
+
 struct DerivationOutput
 {
     StorePath path;
-    std::string hashAlgo; /* hash used for expected hash computation */
+    std::optional<DerivationOutputHash> hash; /* hash used for expected hash computation */
-    std::string hash; /* expected hash, may be null */
     void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
 };
 
@@ -82,9 +88,13 @@ Derivation readDerivation(const Store & store, const Path & drvPath);
 // FIXME: remove
 bool isDerivation(const string & fileName);
 
+// known CA drv's output hashes, current just for fixed-output derivations
+// whose output hashes are always known since they are fixed up-front.
+typedef std::map<std::string, Hash> CaOutputHashes;
+
 typedef std::variant<
     Hash, // regular DRV normalized hash
-    std::map<std::string, Hash> // known CA drv's output hashes
+    CaOutputHashes
 > DrvHashModulo;
 
 /* Returns hashes with the details of fixed-output subderivations

src/libstore/export-import.cc

@@ -55,7 +55,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashAndWriteSink.currentHash();
-    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
+    if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 

src/libstore/filetransfer.cc

@@ -72,6 +72,7 @@ struct curlFileTransfer : public FileTransfer
 
         curl_off_t writtenToSink = 0;
 
+        inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
         /* Get the HTTP status code, or 0 for other protocols. */
         long getHTTPStatus()
         {
@@ -98,7 +99,7 @@ struct curlFileTransfer : public FileTransfer
 
                     /* Only write data to the sink if this is a
                        successful response. */
-                    if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+                    if (successfulStatuses.count(httpStatus)) {
                         writtenToSink += len;
                         this->request.dataCallback((char *) data, len);
                     }
@@ -352,8 +353,7 @@ struct curlFileTransfer : public FileTransfer
             if (writeException)
                 failEx(writeException);
 
-            else if (code == CURLE_OK &&
+            else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
-                (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */))
             {
                 result.cached = httpStatus == 304;
                 act.progress(result.bodySize, result.bodySize);

src/libstore/local-store.cc

@@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        FileIngestionMethod method; Hash h;
+        check(
-        out->second.parseHashInfo(method, h);
+            makeFixedOutputPath(
-
+                out->second.hash->method,
-        check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
+                out->second.hash->hash,
+                drvName),
+            out->second.path, "out");
     }
 
     else {
@@ -1255,9 +1257,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
                 std::unique_ptr<AbstractHashSink> hashSink;
                 if (info->ca == "" || !info->references.count(info->path))
-                    hashSink = std::make_unique<HashSink>(info->narHash.type);
+                    hashSink = std::make_unique<HashSink>(*info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
+                    hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();

src/libstore/parsed-derivations.cc

@@ -117,4 +117,9 @@ bool ParsedDerivation::substitutesAllowed() const
     return getBoolAttr("allowSubstitutes", true);
 }
 
+bool ParsedDerivation::contentAddressed() const
+{
+    return getBoolAttr("__contentAddressed", false);
+}
+
 }

src/libstore/parsed-derivations.hh

@@ -34,6 +34,8 @@ public:
     bool willBuildLocally() const;
 
     bool substitutesAllowed() const;
+
+    bool contentAddressed() const;
 };
 
 }

src/libstore/remote-store.cc

@@ -228,7 +228,7 @@ struct ConnectionHandle
 
     ~ConnectionHandle()
     {
-        if (!daemonException && std::uncaught_exception()) {
+        if (!daemonException && std::uncaught_exceptions()) {
             handle.markBad();
             debug("closing daemon connection because of an exception");
         }

src/libstore/store-api.cc

@@ -61,7 +61,7 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
 
 /* Store paths have the following form:
 
-   <store>/<h>-<name>
+   <realized-path> = <store>/<h>-<name>
 
    where
 
@@ -85,11 +85,14 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
    <type> = one of:
      "text:<r1>:<r2>:...<rN>"
        for plain text files written to the store using
-       addTextToStore(); <r1> ... <rN> are the references of the
+       addTextToStore(); <r1> ... <rN> are the store paths referenced
-       path.
+       by this path, in the form described by <realized-path>
-     "source"
+     "source:<r1>:<r2>:...:<rN>:self"
        for paths copied to the store using addToStore() when recursive
-       = true and hashAlgo = "sha256"
+       = true and hashAlgo = "sha256". Just like in the text case, we
+       can have the store paths referenced by the path.
+       Additionally, we can have an optional :self label to denote self
+       reference.
      "output:<id>"
        for either the outputs created by derivations, OR paths copied
        to the store using addToStore() with recursive != true or
@@ -117,6 +120,12 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p
              the contents of the path (or expected contents of the
              path for fixed-output derivations)
 
+   Note that since an output derivation has always type output, while
+   something added by addToStore can have type output or source depending
+   on the hash, this means that the same input can be hashed differently
+   if added to the store via addToStore or via a derivation, in the sha256
+   recursive case.
+
    It would have been nicer to handle fixed-output derivations under
    "source", e.g. have something like "source:<rec><algo>", but we're
    stuck with this for now...
@@ -164,20 +173,20 @@ static std::string makeType(
 
 
 StorePath Store::makeFixedOutputPath(
-    FileIngestionMethod recursive,
+    FileIngestionMethod method,
     const Hash & hash,
     std::string_view name,
     const StorePathSet & references,
     bool hasSelfReference) const
 {
-    if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
+    if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
         return makeStorePath("output:out",
             hashString(htSHA256,
                 "fixed:out:"
-                + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+                + makeFileIngestionPrefix(method)
                 + hash.to_string(Base16, true) + ":"),
             name);
     }
@@ -813,10 +822,21 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
-std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
+std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+    switch (m) {
+    case FileIngestionMethod::Flat:
+        return "";
+    case FileIngestionMethod::Recursive:
+        return "r:";
+    default:
+        throw Error("impossible, caught both cases");
+    }
+}
+
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
 {
     return "fixed:"
-        + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
+        + makeFileIngestionPrefix(method)
         + hash.to_string(Base32, true);
 }
 

src/libstore/store-api.hh

@@ -189,9 +189,10 @@ struct ValidPathInfo
 
     Strings shortRefs() const;
 
-    ValidPathInfo(const StorePath & path) : path(path) { }
+    ValidPathInfo(const ValidPathInfo & other) = default;
 
-    ValidPathInfo(StorePath && path) : path(std::move(path)) { }
+    ValidPathInfo(StorePath && path) : path(std::move(path)) { };
+    ValidPathInfo(const StorePath & path) : path(path) { };
 
     virtual ~ValidPathInfo() { }
 };
@@ -838,6 +839,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
     std::istream & str,
     bool hashGiven = false);
 
+/* Compute the prefix to the hash algorithm which indicates how the files were
+   ingested. */
+std::string makeFileIngestionPrefix(const FileIngestionMethod m);
 
 /* Compute the content-addressability assertion (ValidPathInfo::ca)
    for paths created by makeFixedOutputPath() / addToStore(). */

src/libutil/ansicolor.hh

@@ -11,5 +11,7 @@ namespace nix {
 #define ANSI_GREEN "\e[32;1m"
 #define ANSI_YELLOW "\e[33;1m"
 #define ANSI_BLUE "\e[34;1m"
+#define ANSI_MAGENTA "\e[35m;1m"
+#define ANSI_CYAN "\e[36m;1m"
 
 }

src/libutil/args.cc

@@ -162,8 +162,18 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
         .labels = {"hash-algo"},
         .handler = {[ht](std::string s) {
             *ht = parseHashType(s);
-            if (*ht == htUnknown)
+        }}
-                throw UsageError("unknown hash type '%1%'", s);
+    };
+}
+
+Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht)
+{
+    return Flag {
+        .longName = std::move(longName),
+        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
+        .labels = {"hash-algo"},
+        .handler = {[oht](std::string s) {
+            *oht = std::optional<HashType> { parseHashType(s) };
         }}
     };
 }

src/libutil/args.hh

@@ -85,6 +85,7 @@ protected:
         Handler handler;
 
         static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
+        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
     };
 
     std::map<std::string, Flag::ptr> longFlags;

src/libutil/hash.cc

@@ -4,6 +4,7 @@
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
+#include "args.hh"
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
@@ -18,11 +19,13 @@ namespace nix {
 
 void Hash::init()
 {
-    if (type == htMD5) hashSize = md5HashSize;
+    if (!type) abort();
-    else if (type == htSHA1) hashSize = sha1HashSize;
+    switch (*type) {
-    else if (type == htSHA256) hashSize = sha256HashSize;
+    case htMD5: hashSize = md5HashSize; break;
-    else if (type == htSHA512) hashSize = sha512HashSize;
+    case htSHA1: hashSize = sha1HashSize; break;
-    else abort();
+    case htSHA256: hashSize = sha256HashSize; break;
+    case htSHA512: hashSize = sha512HashSize; break;
+    }
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -102,11 +105,18 @@ string printHash16or32(const Hash & hash)
 }
 
 
+HashType assertInitHashType(const Hash & h) {
+    if (h.type)
+        return *h.type;
+    else
+        abort();
+}
+
 std::string Hash::to_string(Base base, bool includeType) const
 {
     std::string s;
     if (base == SRI || includeType) {
-        s += printHashType(type);
+        s += printHashType(assertInitHashType(*this));
         s += base == SRI ? '-' : ':';
     }
     switch (base) {
@@ -124,8 +134,10 @@ std::string Hash::to_string(Base base, bool includeType) const
     return s;
 }
 
+Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
+Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { }
 
-Hash::Hash(std::string_view s, HashType type)
+Hash::Hash(std::string_view s, std::optional<HashType> type)
     : type(type)
 {
     size_t pos = 0;
@@ -136,17 +148,17 @@ Hash::Hash(std::string_view s, HashType type)
         sep = s.find('-');
         if (sep != string::npos) {
             isSRI = true;
-        } else if (type == htUnknown)
+        } else if (! type)
             throw BadHash("hash '%s' does not include a type", s);
     }
 
     if (sep != string::npos) {
         string hts = string(s, 0, sep);
         this->type = parseHashType(hts);
-        if (this->type == htUnknown)
+        if (!this->type)
             throw BadHash("unknown hash type '%s'", hts);
-        if (type != htUnknown && type != this->type)
+        if (type && type != this->type)
-            throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
+            throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type));
         pos = sep + 1;
     }
 
@@ -202,14 +214,16 @@ Hash::Hash(std::string_view s, HashType type)
     }
 
     else
-        throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type));
+        throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type));
 }
 
-Hash newHashAllowEmpty(std::string hashStr, HashType ht)
+Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
 {
     if (hashStr.empty()) {
-        Hash h(ht);
+        if (!ht)
-        warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
+            throw BadHash("empty hash requires explicit hash type");
+        Hash h(*ht);
+        warn("found empty hash, assuming '%s'", h.to_string(Base::SRI, true));
         return h;
     } else
         return Hash(hashStr, ht);
@@ -328,24 +342,35 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
 }
 
 
-HashType parseHashType(const string & s)
+std::optional<HashType> parseHashTypeOpt(const string & s)
 {
     if (s == "md5") return htMD5;
     else if (s == "sha1") return htSHA1;
     else if (s == "sha256") return htSHA256;
     else if (s == "sha512") return htSHA512;
-    else return htUnknown;
+    else return std::optional<HashType> {};
 }
 
+HashType parseHashType(const string & s)
+{
+    auto opt_h = parseHashTypeOpt(s);
+    if (opt_h)
+        return *opt_h;
+    else
+        throw UsageError("unknown hash algorithm '%1%'", s);
+}
 
 string printHashType(HashType ht)
 {
-    if (ht == htMD5) return "md5";
+    switch (ht) {
-    else if (ht == htSHA1) return "sha1";
+    case htMD5: return "md5"; break;
-    else if (ht == htSHA256) return "sha256";
+    case htSHA1: return "sha1"; break;
-    else if (ht == htSHA512) return "sha512";
+    case htSHA256: return "sha256"; break;
-    else abort();
+    case htSHA512: return "sha512"; break;
+    }
+    // illegal hash type enum value internally, as opposed to external input
+    // which should be validated with nice error message.
+    abort();
 }
 
-
 }

src/libutil/hash.hh

@@ -10,7 +10,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
+enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 };
 
 
 const int md5HashSize = 16;
@@ -29,7 +29,7 @@ struct Hash
     unsigned int hashSize = 0;
     unsigned char hash[maxHashSize] = {};
 
-    HashType type = htUnknown;
+    std::optional<HashType> type = {};
 
     /* Create an unset hash object. */
     Hash() { };
@@ -40,14 +40,18 @@ struct Hash
     /* Initialize the hash from a string representation, in the format
        "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
        Subresource Integrity hash expression). If the 'type' argument
-       is htUnknown, then the hash type must be specified in the
+       is not present, then the hash type must be specified in the
        string. */
-    Hash(std::string_view s, HashType type = htUnknown);
+    Hash(std::string_view s, std::optional<HashType> type);
+    // type must be provided
+    Hash(std::string_view s, HashType type);
+    // hash type must be part of string
+    Hash(std::string_view s);
 
     void init();
 
     /* Check whether a hash is set. */
-    operator bool () const { return type != htUnknown; }
+    operator bool () const { return (bool) type; }
 
     /* Check whether two hash are equal. */
     bool operator == (const Hash & h2) const;
@@ -95,7 +99,7 @@ struct Hash
 };
 
 /* Helper that defaults empty hashes to the 0 hash. */
-Hash newHashAllowEmpty(std::string hashStr, HashType ht);
+Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht);
 
 /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
 string printHash16or32(const Hash & hash);
@@ -118,6 +122,8 @@ Hash compressHash(const Hash & hash, unsigned int newSize);
 
 /* Parse a string representing a hash type. */
 HashType parseHashType(const string & s);
+/* Will return nothing on parse error */
+std::optional<HashType> parseHashTypeOpt(const string & s);
 
 /* And the reverse. */
 string printHashType(HashType ht);

src/libutil/json.cc

@@ -173,7 +173,7 @@ JSONObject JSONPlaceholder::object()
 
 JSONPlaceholder::~JSONPlaceholder()
 {
-    assert(!first || std::uncaught_exception());
+    assert(!first || std::uncaught_exceptions());
 }
 
 }

src/libutil/tests/compression.cc

@@ -0,0 +1,78 @@
+#include "compression.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * compress / decompress
+     * --------------------------------------------------------------------------*/
+
+    TEST(compress, compressWithUnknownMethod) {
+        ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod);
+    }
+
+    TEST(compress, noneMethodDoesNothingToTheInput) {
+        ref<std::string> o = compress("none", "this-is-a-test");
+
+        ASSERT_EQ(*o, "this-is-a-test");
+    }
+
+    TEST(decompress, decompressXzCompressed) {
+        auto method = "xz";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, *compress(method, str));
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressBzip2Compressed) {
+        auto method = "bzip2";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, *compress(method, str));
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressBrCompressed) {
+        auto method = "br";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, *compress(method, str));
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressInvalidInputThrowsCompressionError) {
+        auto method = "bzip2";
+        auto str = "this is a string that does not qualify as valid bzip2 data";
+
+        ASSERT_THROW(decompress(method, str), CompressionError);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * compression sinks
+     * --------------------------------------------------------------------------*/
+
+    TEST(makeCompressionSink, noneSinkDoesNothingToInput) {
+        StringSink strSink;
+        auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        auto sink = makeCompressionSink("none", strSink);
+        (*sink)(inputString);
+        sink->finish();
+
+        ASSERT_STREQ((*strSink.s).c_str(), inputString);
+    }
+
+    TEST(makeCompressionSink, compressAndDecompress) {
+        StringSink strSink;
+        auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        auto decompressionSink = makeDecompressionSink("bzip2", strSink);
+        auto sink = makeCompressionSink("bzip2", *decompressionSink);
+
+        (*sink)(inputString);
+        sink->finish();
+        decompressionSink->finish();
+
+        ASSERT_STREQ((*strSink.s).c_str(), inputString);
+    }
+
+}

src/libutil/tests/hash.cc

@@ -72,9 +72,4 @@ namespace nix {
                 "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"
                 "c7d329eeb6dd26545e96e55b874be909");
     }
-
-    TEST(hashString, hashingWithUnknownAlgoExits) {
-        auto s = "unknown";
-        ASSERT_DEATH(hashString(HashType::htUnknown, s), "");
-    }
 }

src/libutil/util.cc

@@ -35,7 +35,7 @@
 #endif
 
 
-extern char * * environ;
+extern char * * environ __attribute__((weak));
 
 
 namespace nix {
@@ -1199,7 +1199,7 @@ void _interrupted()
     /* Block user interrupts while an exception is being handled.
        Throwing an exception while another exception is being handled
        kills the program! */
-    if (!interruptThrown && !std::uncaught_exception()) {
+    if (!interruptThrown && !std::uncaught_exceptions()) {
         interruptThrown = true;
         throw Interrupted("interrupted by the user");
     }

src/nix-build/nix-build.cc

@@ -21,7 +21,7 @@
 using namespace nix;
 using namespace std::string_literals;
 
-extern char * * environ;
+extern char * * environ __attribute__((weak));
 
 /* Recreate the effect of the perl shellwords function, breaking up a
  * string into arguments like a shell word, including escapes

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -72,8 +72,6 @@ static int _main(int argc, char * * argv)
             else if (*arg == "--type") {
                 string s = getArg(*arg, arg, end);
                 ht = parseHashType(s);
-                if (ht == htUnknown)
-                    throw UsageError("unknown hash type '%1%'", s);
             }
             else if (*arg == "--print-path")
                 printPath = true;

src/nix-store/nix-store.cc

@@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = store->queryPathInfo(path);
-        HashSink sink(info->narHash.type);
+        HashSink sink(*info->narHash.type);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {

src/nix/command.cc

@@ -4,7 +4,7 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 
-extern char * * environ;
+extern char * * environ __attribute__((weak));
 
 namespace nix {
 

src/nix/develop.cc

@@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
-    drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" });
+    drv.outputs.insert_or_assign("out", DerivationOutput {
+        .path = shellOutPath,
+        .hash = DerivationOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = Hash { },
+        },
+    });
     drv.env["out"] = store->printStorePath(shellOutPath);
     auto shellDrvPath2 = writeDerivation(store, drv, drvName);
 

src/nix/hash.cc

@@ -79,12 +79,12 @@ static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngest
 struct CmdToBase : Command
 {
     Base base;
-    HashType ht = htUnknown;
+    std::optional<HashType> ht;
     std::vector<std::string> args;
 
     CmdToBase(Base base) : base(base)
     {
-        addFlag(Flag::mkHashTypeFlag("type", &ht));
+        addFlag(Flag::mkHashTypeOptFlag("type", &ht));
         expectArgs("strings", &args);
     }
 
@@ -132,8 +132,6 @@ static int compatNixHash(int argc, char * * argv)
         else if (*arg == "--type") {
             string s = getArg(*arg, arg, end);
             ht = parseHashType(s);
-            if (ht == htUnknown)
-                throw UsageError("unknown hash type '%1%'", s);
         }
         else if (*arg == "--to-base16") op = opTo16;
         else if (*arg == "--to-base32") op = opTo32;

src/nix/repl.cc

@@ -19,6 +19,7 @@ extern "C" {
 }
 #endif
 
+#include "ansicolor.hh"
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
@@ -37,14 +38,6 @@ extern "C" {
 
 namespace nix {
 
-#define ESC_RED "\033[31m"
-#define ESC_GRE "\033[32m"
-#define ESC_YEL "\033[33m"
-#define ESC_BLU "\033[34;1m"
-#define ESC_MAG "\033[35m"
-#define ESC_CYA "\033[36m"
-#define ESC_END "\033[0m"
-
 struct NixRepl : gc
 {
     string curDir;
@@ -645,25 +638,25 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     switch (v.type) {
 
     case tInt:
-        str << ESC_CYA << v.integer << ESC_END;
+        str << ANSI_CYAN << v.integer << ANSI_NORMAL;
         break;
 
     case tBool:
-        str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END;
+        str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
         break;
 
     case tString:
-        str << ESC_YEL;
+        str << ANSI_YELLOW;
         printStringValue(str, v.string.s);
-        str << ESC_END;
+        str << ANSI_NORMAL;
         break;
 
     case tPath:
-        str << ESC_GRE << v.path << ESC_END; // !!! escaping?
+        str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping?
         break;
 
     case tNull:
-        str << ESC_CYA "null" ESC_END;
+        str << ANSI_CYAN "null" ANSI_NORMAL;
         break;
 
     case tAttrs: {
@@ -699,7 +692,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                     try {
                         printValue(str, *i.second, maxDepth - 1, seen);
                     } catch (AssertionError & e) {
-                        str << ESC_RED "«error: " << e.msg() << "»" ESC_END;
+                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
                     }
                 str << "; ";
             }
@@ -725,7 +718,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
                     try {
                         printValue(str, *v.listElems()[n], maxDepth - 1, seen);
                     } catch (AssertionError & e) {
-                        str << ESC_RED "«error: " << e.msg() << "»" ESC_END;
+                        str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL;
                     }
                 str << " ";
             }
@@ -737,16 +730,16 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
     case tLambda: {
         std::ostringstream s;
         s << v.lambda.fun->pos;
-        str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END;
+        str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
         break;
     }
 
     case tPrimOp:
-        str << ESC_MAG "«primop»" ESC_END;
+        str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
         break;
 
     case tPrimOpApp:
-        str << ESC_BLU "«primop-app»" ESC_END;
+        str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
         break;
 
     case tFloat:
@@ -754,7 +747,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
         break;
 
     default:
-        str << ESC_RED "«unknown»" ESC_END;
+        str << ANSI_RED "«unknown»" ANSI_NORMAL;
         break;
     }
 

src/nix/show-derivation.cc

@@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand
                 for (auto & output : drv.outputs) {
                     auto outputObj(outputsObj.object(output.first));
                     outputObj.attr("path", store->printStorePath(output.second.path));
-                    if (output.second.hash != "") {
+                    if (output.second.hash) {
-                        outputObj.attr("hashAlgo", output.second.hashAlgo);
+                        outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
-                        outputObj.attr("hash", output.second.hash);
+                        outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
                     }
                 }
             }

src/nix/verify.cc

@@ -88,9 +88,9 @@ struct CmdVerify : StorePathsCommand
 
                     std::unique_ptr<AbstractHashSink> hashSink;
                     if (info->ca == "")
-                        hashSink = std::make_unique<HashSink>(info->narHash.type);
+                        hashSink = std::make_unique<HashSink>(*info->narHash.type);
                     else
-                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
+                        hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
 
                     store->narFromPath(info->path, *hashSink);