From 6ee03b8444e1838b9985a9fd4f0f46947e958b3b Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 3 Jun 2020 12:38:23 +0200 Subject: [PATCH] libutils/hash: remove default encoding This will make it easier to reason about the hash encoding and switch to SRI everywhere where possible. --- perl/lib/Nix/Store.xs | 4 ++-- src/libexpr/primops/fetchTree.cc | 4 ++-- src/libfetchers/fetchers.cc | 4 ++-- src/libfetchers/tarball.cc | 6 +++--- src/libstore/build.cc | 2 +- src/libstore/export-import.cc | 2 +- src/libstore/local-store.cc | 10 +++++----- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- src/libstore/store-api.cc | 12 ++++++------ src/libutil/hash.hh | 2 +- src/libutil/tests/hash.cc | 16 ++++++++-------- src/nix-store/nix-store.cc | 6 +++--- src/nix/verify.cc | 2 +- 15 files changed, 40 insertions(+), 40 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 890310b3e..945ed49c7 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -80,7 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16); + auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c5a0d9886..b90408798 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -23,7 +23,7 @@ void emitTreeAttrs( assert(tree.info.narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI)); + tree.info.narHash.to_string(SRI, true)); if (input->getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); @@ -140,7 +140,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(), hash.to_string()); + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 94ac30e38..ca555114d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -47,7 +47,7 @@ Attrs Input::toAttrs() const { auto attrs = toAttrsInternal(); if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("narHash", narHash->to_string(SRI, true)); attrs.emplace("type", type()); return attrs; } @@ -67,7 +67,7 @@ std::pair> Input::fetchTree(ref store) if (narHash && narHash != input->narHash) throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); return {std::move(tree), input}; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b6e57379b..1b0eba18a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -196,9 +196,9 @@ struct TarballInput : Input // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); + url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); return url2; } @@ -207,7 +207,7 @@ struct TarballInput : Input Attrs attrs; attrs.emplace("url", url.to_string()); if (hash) - attrs.emplace("hash", hash->to_string(SRI)); + attrs.emplace("hash", hash->to_string(SRI, true)); return attrs; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..9582a9007 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3718,7 +3718,7 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index f0d01a240..54471d4a3 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -57,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(), hash.to_string()); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..1c3795eb1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -580,7 +580,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -680,7 +680,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -1022,7 +1022,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1155,7 +1155,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = cloneStorePathSet(references); - info.ca = "text:" + hash.to_string(); + info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } @@ -1273,7 +1273,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 442541330..e8cf1d177 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..6b16be08a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..3f4b72b9c 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -150,7 +150,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..0645fca84 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -142,7 +142,7 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); return StorePath::make(h.hash, name); } @@ -186,7 +186,7 @@ StorePath Store::makeFixedOutputPath( hashString(htSHA256, "fixed:out:" + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") - + hash.to_string(Base16) + ":"), + + hash.to_string(Base16, true) + ":"), name); } } @@ -461,7 +461,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase)) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -504,7 +504,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -760,7 +760,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -836,7 +836,7 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { return "fixed:" + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + + hash.to_string(Base32, true); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ea9fca3e7..4ff13f6b2 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -79,7 +79,7 @@ struct Hash /* Return a string representation of the hash, in base-16, base-32 or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ - std::string to_string(Base base = Base32, bool includeType = true) const; + std::string to_string(Base base, bool includeType) const; std::string gitRev() const { diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 7cb439817..5334b046e 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -11,28 +11,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -40,7 +40,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -48,7 +48,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -56,7 +56,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -67,7 +67,7 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad8..9491a0c26 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -373,7 +373,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32)); + cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -728,7 +728,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError( "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), info->narHash.to_string(), current.first.to_string()); + store->printStorePath(path), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); status = 1; } } @@ -856,7 +856,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index cf1fa6a99..287dad101 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -101,7 +101,7 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError( "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string()); + store->printStorePath(info->path), info->narHash.to_string(Base32, true), hash.first.to_string(Base32, true)); } }