Merge remote-tracking branch 'upstream/master' into add-trace

src/libexpr/get-drvs.cc

@@ -1,7 +1,7 @@
 #include "get-drvs.hh"
 #include "util.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 #include <cstring>
 #include <regex>

src/libexpr/primops.cc

@@ -771,17 +771,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 .nixCode = NixCode { .errPos = posDrvName }
             });
 
-        HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
+        std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
-
         Hash h = newHashAllowEmpty(*outputHash, ht);
 
         auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
-            std::move(outPath),
+            .path = std::move(outPath),
-            (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
+            .hash = FixedOutputHash {
-                + printHashType(h.type),
+                .method = ingestionMethod,
-            h.to_string(Base16, false),
+                .hash = std::move(h),
+            },
         });
     }
 
@@ -795,7 +795,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         for (auto & i : outputs) {
             if (!jsonObject) drv.env[i] = "";
             drv.outputs.insert_or_assign(i,
-                DerivationOutput { StorePath::dummy, "", "" });
+                DerivationOutput {
+                    .path = StorePath::dummy,
+                    .hash = std::optional<FixedOutputHash> {},
+                });
         }
 
         Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);
@@ -804,7 +807,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             auto outPath = state.store->makeOutputPath(i, h, drvName);
             if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
             drv.outputs.insert_or_assign(i,
-                DerivationOutput { std::move(outPath), "", "" });
+                DerivationOutput {
+                    .path = std::move(outPath),
+                    .hash = std::optional<FixedOutputHash>(),
+                });
         }
     }
 
@@ -1001,8 +1007,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
 static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
+    std::optional<HashType> ht = parseHashType(type);
-    if (ht == htUnknown)
+    if (!ht)
       throw Error({
           .hint = hintfmt("unknown hash type '%1%'", type),
           .nixCode = NixCode { .errPos = pos }
@@ -1011,7 +1017,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     PathSet context; // discarded
     Path p = state.coerceToPath(pos, *args[1], context);
 
-    mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
+    mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context);
 }
 
 /* Read a directory (without . or ..) */
@@ -1938,8 +1944,8 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args
 static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v)
 {
     string type = state.forceStringNoCtx(*args[0], pos);
-    HashType ht = parseHashType(type);
+    std::optional<HashType> ht = parseHashType(type);
-    if (ht == htUnknown)
+    if (!ht)
         throw Error({
             .hint = hintfmt("unknown hash type '%1%'", type),
             .nixCode = NixCode { .errPos = pos }
@@ -1948,7 +1954,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     PathSet context; // discarded
     string s = state.forceString(*args[1], context, pos);
 
-    mkString(v, hashString(ht, s).to_string(Base16, false), context);
+    mkString(v, hashString(*ht, s).to_string(Base16, false), context);
 }
 
 

src/libexpr/primops/context.cc

@@ -1,6 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
-#include "derivations.hh"
+#include "store-api.hh"
 
 namespace nix {
 

src/libfetchers/fetchers.cc

@@ -36,7 +36,7 @@ std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
         if (res) {
             if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
                 // FIXME: require SRI hash.
-                res->narHash = newHashAllowEmpty(*narHash, htUnknown);
+                res->narHash = newHashAllowEmpty(*narHash, {});
             return res;
         }
     }

src/libfetchers/tarball.cc

@@ -70,7 +70,10 @@ DownloadFileResult downloadFile(
         ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
         info.narHash = hashString(htSHA256, *sink.s);
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = hash,
+        };
         auto source = StringSource { *sink.s };
         store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
@@ -264,7 +267,7 @@ struct TarballInputScheme : InputScheme
 
         auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
         if (auto hash = maybeGetStrAttr(attrs, "hash"))
-            input->hash = newHashAllowEmpty(*hash, htUnknown);
+            input->hash = newHashAllowEmpty(*hash, {});
 
         return input;
     }

src/libstore/build.cc

@@ -3714,14 +3714,11 @@ void DerivationGoal::registerOutputs()
         /* Check that fixed-output derivations produced the right
            outputs (i.e., the content hash should match the specified
            hash). */
-        std::string ca;
+        std::optional<ContentAddress> ca;
 
         if (fixedOutput) {
 
-            FileIngestionMethod outputHashMode; Hash h;
+            if (i.second.hash->method == FileIngestionMethod::Flat) {
-            i.second.parseHashInfo(outputHashMode, h);
-
-            if (outputHashMode == FileIngestionMethod::Flat) {
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
@@ -3732,20 +3729,22 @@ void DerivationGoal::registerOutputs()
 
             /* Check the hash. In hash mode, move the path produced by
                the derivation to its content-addressed location. */
-            Hash h2 = outputHashMode == FileIngestionMethod::Recursive
+            Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
-                ? hashPath(h.type, actualPath).first
+                ? hashPath(*i.second.hash->hash.type, actualPath).first
-                : hashFile(h.type, actualPath);
+                : hashFile(*i.second.hash->hash.type, actualPath);
 
-            auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name());
+            auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
 
-            if (h != h2) {
+            if (i.second.hash->hash != h2) {
 
                 /* Throw an error after registering the path as
                    valid. */
                 worker.hashMismatch = true;
                 delayedException = std::make_exception_ptr(
                     BuildError("hash mismatch in fixed-output derivation '%s':\n  wanted: %s\n  got:    %s",
-                        worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true)));
+                        worker.store.printStorePath(dest),
+                        i.second.hash->hash.to_string(SRI, true),
+                        h2.to_string(SRI, true)));
 
                 Path actualDest = worker.store.Store::toRealPath(dest);
 
@@ -3765,7 +3764,10 @@ void DerivationGoal::registerOutputs()
             else
                 assert(worker.store.parseStorePath(path) == dest);
 
-            ca = makeFixedOutputCA(outputHashMode, h2);
+            ca = FixedOutputHash {
+                .method = i.second.hash->method,
+                .hash = h2,
+            };
         }
 
         /* Get rid of all weird permissions.  This also checks that
@@ -3838,7 +3840,10 @@ void DerivationGoal::registerOutputs()
         info.ca = ca;
         worker.store.signPathInfo(info);
 
-        if (!info.references.empty()) info.ca.clear();
+        if (!info.references.empty()) {
+            // FIXME don't we have an experimental feature for fixed output with references?
+            info.ca = {};
+        }
 
         infos.emplace(i.first, std::move(info));
     }
@@ -4998,7 +5003,7 @@ bool Worker::pathContentsGood(const StorePath & path)
     if (!pathExists(store.printStorePath(path)))
         res = false;
     else {
-        HashResult current = hashPath(info->narHash.type, store.printStorePath(path));
+        HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
         Hash nullHash(htSHA256);
         res = info->narHash == nullHash || info->narHash == current.first;
     }

src/libstore/builtins/fetchurl.cc

@@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         for (auto hashedMirror : settings.hashedMirrors.get())
             try {
                 if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashType(getAttr("outputHashAlgo"));
+                auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
                 auto h = Hash(getAttr("outputHash"), ht);
-                fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
+                fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
                 return;
             } catch (Error & e) {
                 debug(e.what());

src/libstore/content-address.cc

@@ -0,0 +1,85 @@
+#include "content-address.hh"
+
+namespace nix {
+
+std::string FixedOutputHash::printMethodAlgo() const {
+    return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+}
+
+std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
+    switch (m) {
+    case FileIngestionMethod::Flat:
+        return "";
+    case FileIngestionMethod::Recursive:
+        return "r:";
+    default:
+        throw Error("impossible, caught both cases");
+    }
+}
+
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
+{
+    return "fixed:"
+        + makeFileIngestionPrefix(method)
+        + hash.to_string(Base32, true);
+}
+
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+std::string renderContentAddress(ContentAddress ca) {
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return "text:" + th.hash.to_string(Base32, true);
+        },
+        [](FixedOutputHash fsh) {
+            return makeFixedOutputCA(fsh.method, fsh.hash);
+        }
+    }, ca);
+}
+
+ContentAddress parseContentAddress(std::string_view rawCa) {
+    auto prefixSeparator = rawCa.find(':');
+    if (prefixSeparator != string::npos) {
+        auto prefix = string(rawCa, 0, prefixSeparator);
+        if (prefix == "text") {
+            auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+            Hash hash = Hash(string(hashTypeAndHash));
+            if (*hash.type != htSHA256) {
+                throw Error("parseContentAddress: the text hash should have type SHA256");
+            }
+            return TextHash { hash };
+        } else if (prefix == "fixed") {
+            // This has to be an inverse of makeFixedOutputCA
+            auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
+            if (methodAndHash.substr(0,2) == "r:") {
+                std::string_view hashRaw = methodAndHash.substr(2,string::npos);
+                return FixedOutputHash {
+                    .method = FileIngestionMethod::Recursive,
+                    .hash = Hash(string(hashRaw)),
+                };
+            } else {
+                std::string_view hashRaw = methodAndHash;
+                return FixedOutputHash {
+                    .method = FileIngestionMethod::Flat,
+                    .hash = Hash(string(hashRaw)),
+                };
+            }
+        } else {
+            throw Error("parseContentAddress: format not recognized; has to be text or fixed");
+        }
+    } else {
+        throw Error("Not a content address because it lacks an appropriate prefix");
+    }
+};
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
+    return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
+};
+
+std::string renderContentAddress(std::optional<ContentAddress> ca) {
+    return ca ? renderContentAddress(*ca) : "";
+}
+
+}

src/libstore/content-address.hh

@@ -0,0 +1,56 @@
+#pragma once
+
+#include <variant>
+#include "hash.hh"
+
+namespace nix {
+
+enum struct FileIngestionMethod : uint8_t {
+    Flat = false,
+    Recursive = true
+};
+
+struct TextHash {
+    Hash hash;
+};
+
+/// Pair of a hash, and how the file system was ingested
+struct FixedOutputHash {
+    FileIngestionMethod method;
+    Hash hash;
+    std::string printMethodAlgo() const;
+};
+
+/*
+  We've accumulated several types of content-addressed paths over the years;
+  fixed-output derivations support multiple hash algorithms and serialisation
+  methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
+
+  * ‘text:sha256:<sha256 hash of file contents>’: For paths
+    computed by makeTextPath() / addTextToStore().
+
+  * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
+    makeFixedOutputPath() / addToStore().
+*/
+typedef std::variant<
+    TextHash, // for paths computed by makeTextPath() / addTextToStore
+    FixedOutputHash // for path computed by makeFixedOutputPath
+> ContentAddress;
+
+/* Compute the prefix to the hash algorithm which indicates how the files were
+   ingested. */
+std::string makeFileIngestionPrefix(const FileIngestionMethod m);
+
+/* Compute the content-addressability assertion (ValidPathInfo::ca)
+   for paths created by makeFixedOutputPath() / addToStore(). */
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
+
+std::string renderContentAddress(ContentAddress ca);
+
+std::string renderContentAddress(std::optional<ContentAddress> ca);
+
+ContentAddress parseContentAddress(std::string_view rawCa);
+
+std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+
+}

src/libstore/daemon.cc

@@ -652,7 +652,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
             if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
                 to << info->ultimate
                    << info->sigs
-                   << info->ca;
+                   << renderContentAddress(info->ca);
             }
         } else {
             assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -710,7 +710,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         info.references = readStorePaths<StorePathSet>(*store, from);
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
-        from >> info.ca >> repair >> dontCheckSigs;
+        info.ca = parseContentAddressOpt(readString(from));
+        from >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
         if (!trusted)

src/libstore/derivations.cc

@@ -8,25 +8,6 @@
 
 namespace nix {
 
-
-void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
-{
-    recursive = FileIngestionMethod::Flat;
-    string algo = hashAlgo;
-
-    if (string(algo, 0, 2) == "r:") {
-        recursive = FileIngestionMethod::Recursive;
-        algo = string(algo, 2);
-    }
-
-    HashType hashType = parseHashType(algo);
-    if (hashType == htUnknown)
-        throw Error("unknown hash algorithm '%s'", algo);
-
-    hash = Hash(this->hash, hashType);
-}
-
-
 const StorePath & BasicDerivation::findOutput(const string & id) const
 {
     auto i = outputs.find(id);
@@ -120,6 +101,34 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
 }
 
 
+static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
+{
+    expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+    expect(str, ","); auto hashAlgo = parseString(str);
+    expect(str, ","); const auto hash = parseString(str);
+    expect(str, ")");
+
+    std::optional<FixedOutputHash> fsh;
+    if (hashAlgo != "") {
+        auto method = FileIngestionMethod::Flat;
+        if (string(hashAlgo, 0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = string(hashAlgo, 2);
+        }
+        const HashType hashType = parseHashType(hashAlgo);
+        fsh = FixedOutputHash {
+            .method = std::move(method),
+            .hash = Hash(hash, hashType),
+        };
+    }
+
+    return DerivationOutput {
+        .path = std::move(path),
+        .hash = std::move(fsh),
+    };
+}
+
+
 static Derivation parseDerivation(const Store & store, const string & s)
 {
     Derivation drv;
@@ -129,15 +138,8 @@ static Derivation parseDerivation(const Store & store, const string & s)
     /* Parse the list of outputs. */
     while (!endOfList(str)) {
         expect(str, "("); std::string id = parseString(str);
-        expect(str, ","); auto path = store.parseStorePath(parsePath(str));
+        auto output = parseDerivationOutput(store, str);
-        expect(str, ","); auto hashAlgo = parseString(str);
+        drv.outputs.emplace(std::move(id), std::move(output));
-        expect(str, ","); auto hash = parseString(str);
-        expect(str, ")");
-        drv.outputs.emplace(id, DerivationOutput {
-            .path = std::move(path),
-            .hashAlgo = std::move(hashAlgo),
-            .hash = std::move(hash)
-        });
     }
 
     /* Parse the list of input derivations. */
@@ -263,8 +265,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
         if (first) first = false; else s += ',';
         s += '('; printUnquotedString(s, i.first);
         s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
-        s += ','; printUnquotedString(s, i.second.hashAlgo);
+        s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
-        s += ','; printUnquotedString(s, i.second.hash);
+        s += ','; printUnquotedString(s,
+            i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
         s += ')';
     }
 
@@ -320,7 +323,7 @@ bool BasicDerivation::isFixedOutput() const
 {
     return outputs.size() == 1 &&
         outputs.begin()->first == "out" &&
-        outputs.begin()->second.hash != "";
+        outputs.begin()->second.hash;
 }
 
 
@@ -353,8 +356,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
     if (drv.isFixedOutput()) {
         DerivationOutputs::const_iterator i = drv.outputs.begin();
         return hashString(htSHA256, "fixed:out:"
-            + i->second.hashAlgo + ":"
+            + i->second.hash->printMethodAlgo() + ":"
-            + i->second.hash + ":"
+            + i->second.hash->hash.to_string(Base16, false) + ":"
             + store.printStorePath(i->second.path));
     }
 
@@ -397,6 +400,31 @@ StorePathSet BasicDerivation::outputPaths() const
     return paths;
 }
 
+static DerivationOutput readDerivationOutput(Source & in, const Store & store)
+{
+    auto path = store.parseStorePath(readString(in));
+    auto hashAlgo = readString(in);
+    const auto hash = readString(in);
+
+    std::optional<FixedOutputHash> fsh;
+    if (hashAlgo != "") {
+        auto method = FileIngestionMethod::Flat;
+        if (string(hashAlgo, 0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = string(hashAlgo, 2);
+        }
+        const HashType hashType = parseHashType(hashAlgo);
+        fsh = FixedOutputHash {
+            .method = std::move(method),
+            .hash = Hash(hash, hashType),
+        };
+    }
+
+    return DerivationOutput {
+        .path = std::move(path),
+        .hash = std::move(fsh),
+    };
+}
 
 StringSet BasicDerivation::outputNames() const
 {
@@ -413,14 +441,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
     auto nr = readNum<size_t>(in);
     for (size_t n = 0; n < nr; n++) {
         auto name = readString(in);
-        auto path = store.parseStorePath(readString(in));
+        auto output = readDerivationOutput(in, store);
-        auto hashAlgo = readString(in);
+        drv.outputs.emplace(std::move(name), std::move(output));
-        auto hash = readString(in);
-        drv.outputs.emplace(name, DerivationOutput {
-            .path = std::move(path),
-            .hashAlgo = std::move(hashAlgo),
-            .hash = std::move(hash)
-        });
     }
 
     drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
@@ -442,7 +464,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
 {
     out << drv.outputs.size();
     for (auto & i : drv.outputs)
-        out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash;
+        out << i.first
+            << store.printStorePath(i.second.path)
+            << i.second.hash->printMethodAlgo()
+            << i.second.hash->hash.to_string(Base16, false);
     writeStorePaths(store, out, drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();

src/libstore/derivations.hh

@@ -1,8 +1,9 @@
 #pragma once
 
+#include "path.hh"
 #include "types.hh"
 #include "hash.hh"
-#include "store-api.hh"
+#include "content-address.hh"
 
 #include <map>
 
@@ -15,9 +16,7 @@ namespace nix {
 struct DerivationOutput
 {
     StorePath path;
-    std::string hashAlgo; /* hash used for expected hash computation */
+    std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
-    std::string hash; /* expected hash, may be null */
-    void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -70,6 +69,7 @@ struct Derivation : BasicDerivation
 
 class Store;
 
+enum RepairFlag : bool { NoRepair = false, Repair = true };
 
 /* Write a derivation to the Nix store, and return its path. */
 StorePath writeDerivation(ref<Store> store,

src/libstore/export-import.cc

@@ -55,7 +55,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
        filesystem corruption from spreading to other machines.
        Don't complain if the stored hash is zero (unknown). */
     Hash hash = hashAndWriteSink.currentHash();
-    if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
+    if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
         throw Error("hash of path '%s' has changed from '%s' to '%s'!",
             printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
 

src/libstore/legacy-ssh-store.cc

@@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
             if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
                 auto s = readString(conn->from);
                 info->narHash = s.empty() ? Hash() : Hash(s);
-                conn->from >> info->ca;
+                info->ca = parseContentAddressOpt(readString(conn->from));
                 info->sigs = readStrings<StringSet>(conn->from);
             }
 
@@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
                 << info.narSize
                 << info.ultimate
                 << info.sigs
-                << info.ca;
+                << renderContentAddress(info.ca);
             try {
                 copyNAR(source, conn->to);
             } catch (...) {

src/libstore/local-store.cc

@@ -561,10 +561,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
         if (out == drv.outputs.end())
             throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
 
-        FileIngestionMethod method; Hash h;
+        check(
-        out->second.parseHashInfo(method, h);
+            makeFixedOutputPath(
-
+                out->second.hash->method,
-        check(makeFixedOutputPath(method, h, drvName), out->second.path, "out");
+                out->second.hash->hash,
+                drvName),
+            out->second.path, "out");
     }
 
     else {
@@ -578,7 +580,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
 uint64_t LocalStore::addValidPath(State & state,
     const ValidPathInfo & info, bool checkOutputs)
 {
-    if (info.ca != "" && !info.isContentAddressed(*this))
+    if (info.ca.has_value() && !info.isContentAddressed(*this))
         throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
             printStorePath(info.path));
 
@@ -590,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state,
         (info.narSize, info.narSize != 0)
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
-        (info.ca, !info.ca.empty())
+        (renderContentAddress(info.ca), (bool) info.ca)
         .exec();
     uint64_t id = sqlite3_last_insert_rowid(state.db);
 
@@ -664,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
             if (s) info->sigs = tokenizeString<StringSet>(s, " ");
 
             s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
-            if (s) info->ca = s;
+            if (s) info->ca = parseContentAddressOpt(s);
 
             /* Get the references. */
             auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -687,7 +689,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
         (info.narHash.to_string(Base16, true))
         (info.ultimate ? 1 : 0, info.ultimate)
         (concatStringsSep(" ", info.sigs), !info.sigs.empty())
-        (info.ca, !info.ca.empty())
+        (renderContentAddress(info.ca), (bool) info.ca)
         (printStorePath(info.path))
         .exec();
 }
@@ -983,15 +985,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
             deletePath(realPath);
 
-            if (info.ca != "" &&
+            // text hashing has long been allowed to have non-self-references because it is used for drv files.
-                !((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
+            bool refersToSelf = info.references.count(info.path) > 0;
-                    || info.references.empty()))
+            if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
                 settings.requireExperimentalFeature("ca-references");
 
             /* While restoring the path from the NAR, compute the hash
                of the NAR. */
             std::unique_ptr<AbstractHashSink> hashSink;
-            if (info.ca == "" || !info.references.count(info.path))
+            if (!info.ca.has_value() || !info.references.count(info.path))
                 hashSink = std::make_unique<HashSink>(htSHA256);
             else
                 hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart()));
@@ -1077,7 +1079,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
             ValidPathInfo info(dstPath);
             info.narHash = hash.first;
             info.narSize = hash.second;
-            info.ca = makeFixedOutputCA(method, h);
+            info.ca = FixedOutputHash { .method = method, .hash = h };
             registerValidPath(info);
         }
 
@@ -1141,7 +1143,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
             info.narHash = narHash;
             info.narSize = sink.s->size();
             info.references = references;
-            info.ca = "text:" + hash.to_string(Base32, true);
+            info.ca = TextHash { .hash = hash };
             registerValidPath(info);
         }
 
@@ -1252,10 +1254,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                 printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i));
 
                 std::unique_ptr<AbstractHashSink> hashSink;
-                if (info->ca == "" || !info->references.count(info->path))
+                if (!info->ca || !info->references.count(info->path))
-                    hashSink = std::make_unique<HashSink>(info->narHash.type);
+                    hashSink = std::make_unique<HashSink>(*info->narHash.type);
                 else
-                    hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
+                    hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
 
                 dumpPath(Store::toRealPath(i), *hashSink);
                 auto current = hashSink->finish();

src/libstore/nar-info-disk-cache.cc

@@ -203,7 +203,7 @@ public:
                 narInfo->deriver = StorePath(queryNAR.getStr(9));
             for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
                 narInfo->sigs.insert(sig);
-            narInfo->ca = queryNAR.getStr(11);
+            narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11));
 
             return {oValid, narInfo};
         });
@@ -237,7 +237,7 @@ public:
                     (concatStringsSep(" ", info->shortRefs()))
                     (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
                     (concatStringsSep(" ", info->sigs))
-                    (info->ca)
+                    (renderContentAddress(info->ca))
                     (time(0)).exec();
 
             } else {

src/libstore/nar-info.cc

@@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
         else if (name == "Sig")
             sigs.insert(value);
         else if (name == "CA") {
-            if (!ca.empty()) corrupt();
+            if (ca) corrupt();
-            ca = value;
+            // FIXME: allow blank ca or require skipping field?
+            ca = parseContentAddressOpt(value);
         }
 
         pos = eol + 1;
@@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const
     for (auto sig : sigs)
         res += "Sig: " + sig + "\n";
 
-    if (!ca.empty())
+    if (ca)
-        res += "CA: " + ca + "\n";
+        res += "CA: " + renderContentAddress(*ca) + "\n";
 
     return res;
 }

src/libstore/parsed-derivations.hh

@@ -1,4 +1,4 @@
-#include "derivations.hh"
+#include "store-api.hh"
 
 #include <nlohmann/json_fwd.hpp>
 

src/libstore/path.hh

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "content-address.hh"
 #include "types.hh"
 
 namespace nix {
@@ -65,11 +66,6 @@ typedef std::vector<StorePath> StorePaths;
 /* Extension of derivations in the Nix store. */
 const std::string drvExtension = ".drv";
 
-enum struct FileIngestionMethod : uint8_t {
-    Flat = false,
-    Recursive = true
-};
-
 struct StorePathWithOutputs
 {
     StorePath path;

src/libstore/remote-store.cc

@@ -381,7 +381,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
             if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
                 conn->from >> info->ultimate;
                 info->sigs = readStrings<StringSet>(conn->from);
-                conn->from >> info->ca;
+                info->ca = parseContentAddressOpt(readString(conn->from));
             }
         }
         callback(std::move(info));
@@ -465,7 +465,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << info.narHash.to_string(Base16, false);
         writeStorePaths(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
-                 << info.ultimate << info.sigs << info.ca
+                 << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
         bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
         if (!tunnel) copyNAR(source, conn->to);

src/libstore/store-api.cc

@@ -173,20 +173,20 @@ static std::string makeType(
 
 
 StorePath Store::makeFixedOutputPath(
-    FileIngestionMethod recursive,
+    FileIngestionMethod method,
     const Hash & hash,
     std::string_view name,
     const StorePathSet & references,
     bool hasSelfReference) const
 {
-    if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
+    if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
         return makeStorePath("output:out",
             hashString(htSHA256,
                 "fixed:out:"
-                + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+                + makeFileIngestionPrefix(method)
                 + hash.to_string(Base16, true) + ":"),
             name);
     }
@@ -471,8 +471,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
                     jsonRefs.elem(printStorePath(ref));
             }
 
-            if (info->ca != "")
+            if (info->ca)
-                jsonPath.attr("ca", info->ca);
+                jsonPath.attr("ca", renderContentAddress(info->ca));
 
             std::pair<uint64_t, uint64_t> closureSizes;
 
@@ -757,41 +757,35 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
     sigs.insert(secretKey.signDetached(fingerprint(store)));
 }
 
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 
 bool ValidPathInfo::isContentAddressed(const Store & store) const
 {
-    auto warn = [&]() {
+    if (! ca) return false;
-        logWarning(
-            ErrorInfo{
-                .name = "Path not content-addressed",
-                .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path))
-            });
-    };
 
-    if (hasPrefix(ca, "text:")) {
+    auto caPath = std::visit(overloaded {
-        Hash hash(ca.substr(5));
+        [&](TextHash th) {
-        if (store.makeTextPath(path.name(), hash, references) == path)
+            return store.makeTextPath(path.name(), th.hash, references);
-            return true;
+        },
-        else
+        [&](FixedOutputHash fsh) {
-            warn();
+            auto refs = references;
-    }
+            bool hasSelfReference = false;
-
+            if (refs.count(path)) {
-    else if (hasPrefix(ca, "fixed:")) {
+                hasSelfReference = true;
-        FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
+                refs.erase(path);
-        Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6));
+            }
-        auto refs = references;
+            return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference);
-        bool hasSelfReference = false;
-        if (refs.count(path)) {
-            hasSelfReference = true;
-            refs.erase(path);
         }
-        if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
+    }, *ca);
-            return true;
-        else
-            warn();
-    }
 
-    return false;
+    bool res = caPath == path;
+
+    if (!res)
+        printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path));
+
+    return res;
 }
 
 
@@ -822,14 +816,6 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
-std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
-{
-    return "fixed:"
-        + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
-        + hash.to_string(Base32, true);
-}
-
-
 }
 
 

src/libstore/store-api.hh

@@ -2,12 +2,14 @@
 
 #include "path.hh"
 #include "hash.hh"
+#include "content-address.hh"
 #include "serialise.hh"
 #include "crypto.hh"
 #include "lru-cache.hh"
 #include "sync.hh"
 #include "globals.hh"
 #include "config.hh"
+#include "derivations.hh"
 
 #include <atomic>
 #include <limits>
@@ -17,6 +19,7 @@
 #include <memory>
 #include <string>
 #include <chrono>
+#include <variant>
 
 
 namespace nix {
@@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error);
 MakeError(NotInStore, Error);
 
 
-struct BasicDerivation;
-struct Derivation;
 class FSAccessor;
 class NarInfoDiskCache;
 class Store;
 class JSONPlaceholder;
 
 
-enum RepairFlag : bool { NoRepair = false, Repair = true };
 enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
 enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
 enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
@@ -111,7 +111,6 @@ struct SubstitutablePathInfo
 
 typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
 
-
 struct ValidPathInfo
 {
     StorePath path;
@@ -140,21 +139,11 @@ struct ValidPathInfo
        that a particular output path was produced by a derivation; the
        path then implies the contents.)
 
-       Ideally, the content-addressability assertion would just be a
+       Ideally, the content-addressability assertion would just be a Boolean,
-       Boolean, and the store path would be computed from
+       and the store path would be computed from the name component, ‘narHash’
-       the name component, ‘narHash’ and ‘references’. However,
+       and ‘references’. However, we support many types of content addresses.
-       1) we've accumulated several types of content-addressed paths
-       over the years; and 2) fixed-output derivations support
-       multiple hash algorithms and serialisation methods (flat file
-       vs NAR). Thus, ‘ca’ has one of the following forms:
-
-       * ‘text:sha256:<sha256 hash of file contents>’: For paths
-         computed by makeTextPath() / addTextToStore().
-
-       * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
-         makeFixedOutputPath() / addToStore().
     */
-    std::string ca;
+    std::optional<ContentAddress> ca;
 
     bool operator == (const ValidPathInfo & i) const
     {
@@ -189,9 +178,10 @@ struct ValidPathInfo
 
     Strings shortRefs() const;
 
-    ValidPathInfo(const StorePath & path) : path(path) { }
+    ValidPathInfo(const ValidPathInfo & other) = default;
 
-    ValidPathInfo(StorePath && path) : path(std::move(path)) { }
+    ValidPathInfo(StorePath && path) : path(std::move(path)) { };
+    ValidPathInfo(const StorePath & path) : path(path) { };
 
     virtual ~ValidPathInfo() { }
 };
@@ -838,12 +828,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
     std::istream & str,
     bool hashGiven = false);
 
-
-/* Compute the content-addressability assertion (ValidPathInfo::ca)
-   for paths created by makeFixedOutputPath() / addToStore(). */
-std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
-
-
 /* Split URI into protocol+hierarchy part and its parameter set. */
 std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
 

src/libutil/args.cc

@@ -162,8 +162,18 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
         .labels = {"hash-algo"},
         .handler = {[ht](std::string s) {
             *ht = parseHashType(s);
-            if (*ht == htUnknown)
+        }}
-                throw UsageError("unknown hash type '%1%'", s);
+    };
+}
+
+Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht)
+{
+    return Flag {
+        .longName = std::move(longName),
+        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
+        .labels = {"hash-algo"},
+        .handler = {[oht](std::string s) {
+            *oht = std::optional<HashType> { parseHashType(s) };
         }}
     };
 }

src/libutil/args.hh

@@ -85,6 +85,7 @@ protected:
         Handler handler;
 
         static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
+        static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
     };
 
     std::map<std::string, Flag::ptr> longFlags;

src/libutil/fmt.hh

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <boost/format.hpp>
+#include <boost/algorithm/string/replace.hpp>
 #include <string>
 #include "ansicolor.hh"
 
@@ -103,7 +104,9 @@ class hintformat
 public:
     hintformat(const string &format) :fmt(format)
     {
-        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+        fmt.exceptions(boost::io::all_error_bits ^ 
+                       boost::io::too_many_args_bit ^
+                       boost::io::too_few_args_bit);
     }
 
     hintformat(const hintformat &hf)
@@ -117,6 +120,13 @@ public:
         return *this;
     }
 
+    template<class T>
+    hintformat& operator%(const normaltxt<T> &value)
+    {
+        fmt % value.value;
+        return *this;
+    }
+
     std::string str() const
     {
         return fmt.str();
@@ -136,4 +146,9 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args)
     return f;
 }
 
+inline hintformat hintfmt(std::string plain_string)
+{
+    // we won't be receiving any args in this case, so just print the original string
+    return hintfmt("%s", normaltxt(plain_string));
+}
 }

src/libutil/hash.cc

@@ -4,6 +4,7 @@
 #include <openssl/md5.h>
 #include <openssl/sha.h>
 
+#include "args.hh"
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
@@ -18,11 +19,13 @@ namespace nix {
 
 void Hash::init()
 {
-    if (type == htMD5) hashSize = md5HashSize;
+    if (!type) abort();
-    else if (type == htSHA1) hashSize = sha1HashSize;
+    switch (*type) {
-    else if (type == htSHA256) hashSize = sha256HashSize;
+    case htMD5: hashSize = md5HashSize; break;
-    else if (type == htSHA512) hashSize = sha512HashSize;
+    case htSHA1: hashSize = sha1HashSize; break;
-    else abort();
+    case htSHA256: hashSize = sha256HashSize; break;
+    case htSHA512: hashSize = sha512HashSize; break;
+    }
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -102,11 +105,18 @@ string printHash16or32(const Hash & hash)
 }
 
 
+HashType assertInitHashType(const Hash & h) {
+    if (h.type)
+        return *h.type;
+    else
+        abort();
+}
+
 std::string Hash::to_string(Base base, bool includeType) const
 {
     std::string s;
     if (base == SRI || includeType) {
-        s += printHashType(type);
+        s += printHashType(assertInitHashType(*this));
         s += base == SRI ? '-' : ':';
     }
     switch (base) {
@@ -124,8 +134,10 @@ std::string Hash::to_string(Base base, bool includeType) const
     return s;
 }
 
+Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
+Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { }
 
-Hash::Hash(std::string_view s, HashType type)
+Hash::Hash(std::string_view s, std::optional<HashType> type)
     : type(type)
 {
     size_t pos = 0;
@@ -136,17 +148,17 @@ Hash::Hash(std::string_view s, HashType type)
         sep = s.find('-');
         if (sep != string::npos) {
             isSRI = true;
-        } else if (type == htUnknown)
+        } else if (! type)
             throw BadHash("hash '%s' does not include a type", s);
     }
 
     if (sep != string::npos) {
         string hts = string(s, 0, sep);
         this->type = parseHashType(hts);
-        if (this->type == htUnknown)
+        if (!this->type)
             throw BadHash("unknown hash type '%s'", hts);
-        if (type != htUnknown && type != this->type)
+        if (type && type != this->type)
-            throw BadHash("hash '%s' should have type '%s'", s, printHashType(type));
+            throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type));
         pos = sep + 1;
     }
 
@@ -202,13 +214,15 @@ Hash::Hash(std::string_view s, HashType type)
     }
 
     else
-        throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type));
+        throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type));
 }
 
-Hash newHashAllowEmpty(std::string hashStr, HashType ht)
+Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
 {
     if (hashStr.empty()) {
-        Hash h(ht);
+        if (!ht)
+            throw BadHash("empty hash requires explicit hash type");
+        Hash h(*ht);
         warn("found empty hash, assuming '%s'", h.to_string(SRI, true));
         return h;
     } else
@@ -328,24 +342,35 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
 }
 
 
-HashType parseHashType(const string & s)
+std::optional<HashType> parseHashTypeOpt(const string & s)
 {
     if (s == "md5") return htMD5;
     else if (s == "sha1") return htSHA1;
     else if (s == "sha256") return htSHA256;
     else if (s == "sha512") return htSHA512;
-    else return htUnknown;
+    else return std::optional<HashType> {};
 }
 
+HashType parseHashType(const string & s)
+{
+    auto opt_h = parseHashTypeOpt(s);
+    if (opt_h)
+        return *opt_h;
+    else
+        throw UsageError("unknown hash algorithm '%1%'", s);
+}
 
 string printHashType(HashType ht)
 {
-    if (ht == htMD5) return "md5";
+    switch (ht) {
-    else if (ht == htSHA1) return "sha1";
+    case htMD5: return "md5"; break;
-    else if (ht == htSHA256) return "sha256";
+    case htSHA1: return "sha1"; break;
-    else if (ht == htSHA512) return "sha512";
+    case htSHA256: return "sha256"; break;
-    else abort();
+    case htSHA512: return "sha512"; break;
+    }
+    // illegal hash type enum value internally, as opposed to external input
+    // which should be validated with nice error message.
+    abort();
 }
 
-
 }

src/libutil/hash.hh

@@ -10,7 +10,7 @@ namespace nix {
 MakeError(BadHash, Error);
 
 
-enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
+enum HashType : char { htMD5, htSHA1, htSHA256, htSHA512 };
 
 
 const int md5HashSize = 16;
@@ -29,7 +29,7 @@ struct Hash
     unsigned int hashSize = 0;
     unsigned char hash[maxHashSize] = {};
 
-    HashType type = htUnknown;
+    std::optional<HashType> type = {};
 
     /* Create an unset hash object. */
     Hash() { };
@@ -40,14 +40,18 @@ struct Hash
     /* Initialize the hash from a string representation, in the format
        "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a
        Subresource Integrity hash expression). If the 'type' argument
-       is htUnknown, then the hash type must be specified in the
+       is not present, then the hash type must be specified in the
        string. */
-    Hash(std::string_view s, HashType type = htUnknown);
+    Hash(std::string_view s, std::optional<HashType> type);
+    // type must be provided
+    Hash(std::string_view s, HashType type);
+    // hash type must be part of string
+    Hash(std::string_view s);
 
     void init();
 
     /* Check whether a hash is set. */
-    operator bool () const { return type != htUnknown; }
+    operator bool () const { return (bool) type; }
 
     /* Check whether two hash are equal. */
     bool operator == (const Hash & h2) const;
@@ -95,7 +99,7 @@ struct Hash
 };
 
 /* Helper that defaults empty hashes to the 0 hash. */
-Hash newHashAllowEmpty(std::string hashStr, HashType ht);
+Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht);
 
 /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */
 string printHash16or32(const Hash & hash);
@@ -118,6 +122,8 @@ Hash compressHash(const Hash & hash, unsigned int newSize);
 
 /* Parse a string representing a hash type. */
 HashType parseHashType(const string & s);
+/* Will return nothing on parse error */
+std::optional<HashType> parseHashTypeOpt(const string & s);
 
 /* And the reverse. */
 string printHashType(HashType ht);

src/libutil/tests/compression.cc

@@ -0,0 +1,78 @@
+#include "compression.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    /* ----------------------------------------------------------------------------
+     * compress / decompress
+     * --------------------------------------------------------------------------*/
+
+    TEST(compress, compressWithUnknownMethod) {
+        ASSERT_THROW(compress("invalid-method", "something-to-compress"), UnknownCompressionMethod);
+    }
+
+    TEST(compress, noneMethodDoesNothingToTheInput) {
+        ref<std::string> o = compress("none", "this-is-a-test");
+
+        ASSERT_EQ(*o, "this-is-a-test");
+    }
+
+    TEST(decompress, decompressXzCompressed) {
+        auto method = "xz";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, *compress(method, str));
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressBzip2Compressed) {
+        auto method = "bzip2";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, *compress(method, str));
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressBrCompressed) {
+        auto method = "br";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, *compress(method, str));
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressInvalidInputThrowsCompressionError) {
+        auto method = "bzip2";
+        auto str = "this is a string that does not qualify as valid bzip2 data";
+
+        ASSERT_THROW(decompress(method, str), CompressionError);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * compression sinks
+     * --------------------------------------------------------------------------*/
+
+    TEST(makeCompressionSink, noneSinkDoesNothingToInput) {
+        StringSink strSink;
+        auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        auto sink = makeCompressionSink("none", strSink);
+        (*sink)(inputString);
+        sink->finish();
+
+        ASSERT_STREQ((*strSink.s).c_str(), inputString);
+    }
+
+    TEST(makeCompressionSink, compressAndDecompress) {
+        StringSink strSink;
+        auto inputString = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        auto decompressionSink = makeDecompressionSink("bzip2", strSink);
+        auto sink = makeCompressionSink("bzip2", *decompressionSink);
+
+        (*sink)(inputString);
+        sink->finish();
+        decompressionSink->finish();
+
+        ASSERT_STREQ((*strSink.s).c_str(), inputString);
+    }
+
+}

src/libutil/tests/hash.cc

@@ -72,9 +72,4 @@ namespace nix {
                 "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a"
                 "c7d329eeb6dd26545e96e55b874be909");
     }
-
-    TEST(hashString, hashingWithUnknownAlgoExits) {
-        auto s = "unknown";
-        ASSERT_DEATH(hashString(HashType::htUnknown, s), "");
-    }
 }

src/libutil/tests/logging.cc

@@ -1,6 +1,7 @@
 #include "logging.hh"
 #include "nixexpr.hh"
 #include "util.hh"
+#include <fstream>
 
 #include <gtest/gtest.h>
 
@@ -42,7 +43,7 @@ namespace nix {
             logger->logEI(ei);
             auto str = testing::internal::GetCapturedStderr();
 
-            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n");
+            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\ninitial error; subsequent error message.\n");
         }
 
     }
@@ -60,8 +61,7 @@ namespace nix {
             logError(e.info());
             auto str = testing::internal::GetCapturedStderr();
 
-            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n");
+            ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nstatting file: \x1B[33;1mBad file descriptor\x1B[0m\n");
-
         }
     }
 
@@ -69,9 +69,9 @@ namespace nix {
         testing::internal::CaptureStderr();
 
         logger->logEI({ .level = lvlInfo,
-                .name = "Info name",
+                        .name = "Info name",
-                .description = "Info description",
+                        .description = "Info description",
-                });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name --- error-unit-test\x1B[0m\nInfo description\n");
@@ -85,7 +85,7 @@ namespace nix {
         logger->logEI({ .level = lvlTalkative,
                         .name = "Talkative name",
                         .description = "Talkative description",
-                        });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name --- error-unit-test\x1B[0m\nTalkative description\n");
@@ -99,7 +99,7 @@ namespace nix {
         logger->logEI({ .level = lvlChatty,
                         .name = "Chatty name",
                         .description = "Talkative description",
-                        });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name --- error-unit-test\x1B[0m\nTalkative description\n");
@@ -113,7 +113,7 @@ namespace nix {
         logger->logEI({ .level = lvlDebug,
                         .name = "Debug name",
                         .description = "Debug description",
-                        });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name --- error-unit-test\x1B[0m\nDebug description\n");
@@ -127,7 +127,7 @@ namespace nix {
         logger->logEI({ .level = lvlVomit,
                         .name = "Vomit name",
                         .description = "Vomit description",
-                        });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name --- error-unit-test\x1B[0m\nVomit description\n");
@@ -144,7 +144,7 @@ namespace nix {
         logError({
                 .name = "name",
                 .description = "error description",
-                });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n");
@@ -160,13 +160,13 @@ namespace nix {
                 .name = "error name",
                 .description = "error with code lines",
                 .hint = hintfmt("this hint has %1% templated %2%!!",
-                        "yellow",
+                    "yellow",
-                        "values"),
+                    "values"),
                 .nixCode = NixCode {
-                .errPos = Pos(foFile, problem_file, 40, 13),
+                    .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = "previous line of code",
+                    .prevLineOfCode = "previous line of code",
-                .errLineOfCode = "this is the problem line of code",
+                    .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = "next line of code",
+                    .nextLineOfCode = "next line of code",
                 }});
 
 
@@ -183,10 +183,10 @@ namespace nix {
                 .name = "error name",
                 .description = "error without any code lines.",
                 .hint = hintfmt("this hint has %1% templated %2%!!",
-                        "yellow",
+                    "yellow",
-                        "values"),
+                    "values"),
                 .nixCode = NixCode {
-                .errPos = Pos(foFile, problem_file, 40, 13)
+                    .errPos = Pos(problem_file, 40, 13)
                 }});
 
         auto str = testing::internal::GetCapturedStderr();
@@ -202,7 +202,7 @@ namespace nix {
                 .name = "error name",
                 .hint = hintfmt("hint %1%", "only"),
                 .nixCode = NixCode {
-                .errPos = Pos(foFile, problem_file, 40, 13)
+                    .errPos = Pos(problem_file, 40, 13)
                 }});
 
         auto str = testing::internal::GetCapturedStderr();
@@ -218,10 +218,10 @@ namespace nix {
         testing::internal::CaptureStderr();
 
         logWarning({
-            .name = "name",
+                .name = "name",
-            .description = "error description",
+                .description = "error description",
-            .hint = hintfmt("there was a %1%", "warning"),
+                .hint = hintfmt("there was a %1%", "warning"),
-        });
+            });
 
         auto str = testing::internal::GetCapturedStderr();
         ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
@@ -238,13 +238,13 @@ namespace nix {
                 .name = "warning name",
                 .description = "warning description",
                 .hint = hintfmt("this hint has %1% templated %2%!!",
-                        "yellow",
+                    "yellow",
-                        "values"),
+                    "values"),
                 .nixCode = NixCode {
-                .errPos = Pos(foFile, problem_file, 40, 13),
+                    .errPos = Pos(problem_file, 40, 13),
-                .prevLineOfCode = std::nullopt,
+                    .prevLineOfCode = std::nullopt,
-                .errLineOfCode = "this is the problem line of code",
+                    .errLineOfCode = "this is the problem line of code",
-                .nextLineOfCode = std::nullopt
+                    .nextLineOfCode = std::nullopt
                 }});
 
 
@@ -252,4 +252,41 @@ namespace nix {
         ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
+    /* ----------------------------------------------------------------------------
+     * hintfmt
+     * --------------------------------------------------------------------------*/
+
+    TEST(hintfmt, percentStringWithoutArgs) {
+
+        const char *teststr = "this is 100%s correct!";
+
+        ASSERT_STREQ(
+            hintfmt(teststr).str().c_str(),
+            teststr);
+
+    }
+
+    TEST(hintfmt, fmtToHintfmt) {
+
+        ASSERT_STREQ(
+            hintfmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(),
+            "the color of this this text is not yellow");
+
+    }
+
+    TEST(hintfmt, tooFewArguments) {
+
+        ASSERT_STREQ(
+            hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(),
+            "only one arg " ANSI_YELLOW "fulfilled" ANSI_NORMAL " ");
+
+    }
+
+    TEST(hintfmt, tooManyArguments) {
+
+        ASSERT_STREQ(
+            hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(),
+            "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
+
+    }
 }

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -72,8 +72,6 @@ static int _main(int argc, char * * argv)
             else if (*arg == "--type") {
                 string s = getArg(*arg, arg, end);
                 ht = parseHashType(s);
-                if (ht == htUnknown)
-                    throw UsageError("unknown hash type '%1%'", s);
             }
             else if (*arg == "--print-path")
                 printPath = true;

src/nix-store/nix-store.cc

@@ -725,7 +725,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
         auto path = store->followLinksToStorePath(i);
         printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
         auto info = store->queryPathInfo(path);
-        HashSink sink(info->narHash.type);
+        HashSink sink(*info->narHash.type);
         store->narFromPath(path, sink);
         auto current = sink.finish();
         if (current.first != info->narHash) {
@@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         out << info->narSize // downloadSize
                             << info->narSize;
                         if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
-                            out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs;
+                            out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs;
                     } catch (InvalidPath &) {
                     }
                 }
@@ -952,7 +952,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 info.references = readStorePaths<StorePathSet>(*store, in);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
-                in >> info.ca;
+                info.ca = parseContentAddressOpt(readString(in));
 
                 if (info.narSize == 0)
                     throw Error("narInfo is too old and missing the narSize field");

src/nix/add-to-store.cc

@@ -48,7 +48,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
         info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
+        info.ca = std::optional { FixedOutputHash {
+            .method = FileIngestionMethod::Recursive,
+            .hash = info.narHash,
+        } };
 
         if (!dryRun) {
             auto source = StringSource { *sink.s };

src/nix/develop.cc

@@ -135,7 +135,13 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = hashDerivationModulo(*store, drv, true);
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
-    drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" });
+    drv.outputs.insert_or_assign("out", DerivationOutput {
+        .path = shellOutPath,
+        .hash = FixedOutputHash {
+            .method = FileIngestionMethod::Flat,
+            .hash = Hash { },
+        },
+    });
     drv.env["out"] = store->printStorePath(shellOutPath);
     auto shellDrvPath2 = writeDerivation(store, drv, drvName);
 

src/nix/hash.cc

@@ -1,5 +1,6 @@
 #include "command.hh"
 #include "hash.hh"
+#include "content-address.hh"
 #include "legacy.hh"
 #include "shared.hh"
 #include "references.hh"
@@ -79,12 +80,12 @@ static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngest
 struct CmdToBase : Command
 {
     Base base;
-    HashType ht = htUnknown;
+    std::optional<HashType> ht;
     std::vector<std::string> args;
 
     CmdToBase(Base base) : base(base)
     {
-        addFlag(Flag::mkHashTypeFlag("type", &ht));
+        addFlag(Flag::mkHashTypeOptFlag("type", &ht));
         expectArgs("strings", &args);
     }
 
@@ -132,8 +133,6 @@ static int compatNixHash(int argc, char * * argv)
         else if (*arg == "--type") {
             string s = getArg(*arg, arg, end);
             ht = parseHashType(s);
-            if (ht == htUnknown)
-                throw UsageError("unknown hash type '%1%'", s);
         }
         else if (*arg == "--to-base16") op = opTo16;
         else if (*arg == "--to-base32") op = opTo32;

src/nix/make-content-addressable.cc

@@ -82,7 +82,10 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             if (hasSelfReference) info.references.insert(info.path);
             info.narHash = narHash;
             info.narSize = sink.s->size();
-            info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
+            info.ca = FixedOutputHash {
+                .method = FileIngestionMethod::Recursive,
+                .hash = info.narHash,
+            };
 
             if (!json)
                 printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));

src/nix/path-info.cc

@@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
                     std::cout << '\t';
                     Strings ss;
                     if (info->ultimate) ss.push_back("ultimate");
-                    if (info->ca != "") ss.push_back("ca:" + info->ca);
+                    if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca));
                     for (auto & sig : info->sigs) ss.push_back(sig);
                     std::cout << concatStringsSep(" ", ss);
                 }

src/nix/show-derivation.cc

@@ -70,9 +70,9 @@ struct CmdShowDerivation : InstallablesCommand
                 for (auto & output : drv.outputs) {
                     auto outputObj(outputsObj.object(output.first));
                     outputObj.attr("path", store->printStorePath(output.second.path));
-                    if (output.second.hash != "") {
+                    if (output.second.hash) {
-                        outputObj.attr("hashAlgo", output.second.hashAlgo);
+                        outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
-                        outputObj.attr("hash", output.second.hash);
+                        outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
                     }
                 }
             }

src/nix/verify.cc

@@ -87,10 +87,10 @@ struct CmdVerify : StorePathsCommand
                 if (!noContents) {
 
                     std::unique_ptr<AbstractHashSink> hashSink;
-                    if (info->ca == "")
+                    if (!info->ca)
-                        hashSink = std::make_unique<HashSink>(info->narHash.type);
+                        hashSink = std::make_unique<HashSink>(*info->narHash.type);
                     else
-                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
+                        hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
 
                     store->narFromPath(info->path, *hashSink);