From 34f25124ba2ab32b8a95d6b37cd68d7bb85ff2d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Apr 2018 23:46:20 +0200 Subject: [PATCH 1/9] Make LocalStore::addToStore(srcPath) run in constant memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reduces memory consumption of nix-instantiate \ -E 'with import {}; runCommand "foo" { src = ./blender; } "echo foo"' \ --option nar-buffer-size 10000 (where ./blender is a 1.1 GiB tree) from 1716 to 36 MiB, while still ensuring that we don't do any write I/O for small source paths (up to 'nar-buffer-size' bytes). The downside is that large paths are now always written to a temporary location in the store, even if they produce an already valid store path. Thus, adding large paths might be slower and run out of disk space. ¯\_(ツ)_/¯ Of course, you can always restore the old behaviour by setting 'nar-buffer-size' to a very high value. --- src/libstore/globals.hh | 3 + src/libstore/local-store.cc | 121 +++++++++++++++++++++++++++++++++--- 2 files changed, 115 insertions(+), 9 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 58cf08763..4d5eec7bf 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -365,6 +365,9 @@ public: Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + + Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", + "Maximum size of NARs before spilling them to disk."}; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eed225349..b9176ec38 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1098,16 +1098,119 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (method == FileIngestionMethod::Recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = make_ref(readFile(srcPath)); + if (method != FileIngestionMethod::Recursive) + return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); + /* For computing the NAR hash. */ + auto sha256Sink = std::make_unique(htSHA256); + + /* For computing the store path. In recursive SHA-256 mode, this + is the same as the NAR hash, so no need to do it again. */ + std::unique_ptr hashSink = + hashAlgo == htSHA256 + ? nullptr + : std::make_unique(hashAlgo); + + /* Read the source path into memory, but only if it's up to + narBufferSize bytes. If it's larger, write it to a temporary + location in the Nix store. If the subsequently computed + destination store path is already valid, we just delete the + temporary path. Otherwise, we move it to the destination store + path. */ + bool inMemory = true; + std::string nar; + + auto source = sinkToSource([&](Sink & sink) { + + LambdaSink sink2([&](const unsigned char * buf, size_t len) { + (*sha256Sink)(buf, len); + if (hashSink) (*hashSink)(buf, len); + + if (inMemory) { + if (nar.size() + len > settings.narBufferSize) { + inMemory = false; + sink << 1; + sink((const unsigned char *) nar.data(), nar.size()); + nar.clear(); + } else { + nar.append((const char *) buf, len); + } + } + + if (!inMemory) sink(buf, len); + }); + + dumpPath(srcPath, sink2, filter); + }); + + std::unique_ptr delTempDir; + Path tempPath; + + try { + /* Wait for the source coroutine to give us some dummy + data. This is so that we don't create the temporary + directory if the NAR fits in memory. */ + readInt(*source); + + auto tempDir = createTempDir(realStoreDir, "add"); + delTempDir = std::make_unique(tempDir); + tempPath = tempDir + "/x"; + + restorePath(tempPath, *source); + + } catch (EndOfFile &) { + if (!inMemory) throw; + /* The NAR fits in memory, so we didn't do restorePath(). */ + } + + auto sha256 = sha256Sink->finish(); + + Hash hash = hashSink ? hashSink->finish().first : sha256.first; + + Path dstPath = makeFixedOutputPath(method, hash, name); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if (repair || !isValidPath(dstPath)) { + + deletePath(realPath); + + autoGC(); + + if (inMemory) { + /* Restore from the NAR in memory. */ + StringSource source(nar); + restorePath(realPath, source); + } else { + /* Move the temporary path we restored above. */ + if (rename(tempPath.c_str(), realPath.c_str())) + throw Error("renaming '%s' to '%s'", tempPath, realPath); + } + + canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath + + optimisePath(realPath); + + ValidPathInfo info(dstPath); + info.narHash = sha256.first; + info.narSize = sha256.second; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; } From b981e5aacf3848424264f4a84826f8f9ca33da14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jul 2020 22:07:10 +0200 Subject: [PATCH 2/9] Cleanup --- src/libstore/local-store.cc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b9176ec38..fa66c1c90 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -976,7 +976,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); + auto realPath = Store::toRealPath(info.path); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a @@ -1047,8 +1047,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1167,7 +1166,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, Hash hash = hashSink ? hashSink->finish().first : sha256.first; - Path dstPath = makeFixedOutputPath(method, hash, name); + auto dstPath = makeFixedOutputPath(method, hash, name); addTempRoot(dstPath); @@ -1176,7 +1175,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); @@ -1224,8 +1223,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/"; - realPath += dstPath.to_string(); + auto realPath = Store::toRealPath(dstPath); PathLocks outputLock({realPath}); From cfe6ea746c3b88887eafda3eaccedc654f61651f Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 8 Jul 2020 20:10:22 -0500 Subject: [PATCH 3/9] add temp CI job to test syspolicy impact Starting in Catalina, macOS runs a syspolicyd "assessment" that hits the network for each binary/script executable. It does cache these results, but Nix tends to introduce many "new" executables per build. (You can read more about this at https://github.com/NixOS/nix/issues/3789). This PR adds a temporary, redundant macOS job with these assessments disabled. I'm hoping you can adopt it for a few weeks to help me collect more data on how this affects real projects. --- .github/workflows/test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7755466a0..47fa041e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,3 +12,13 @@ jobs: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v10 - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings + macos_perf_test: + runs-on: macos-latest + steps: + - name: Disable syspolicy assessments + run: | + spctl --status + sudo spctl --master-disable + - uses: actions/checkout@v2 + - uses: cachix/install-nix-action@v10 + - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings From a2c27022e9afc394e08d34d349587c8903fc1a97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jul 2020 15:54:32 +0200 Subject: [PATCH 4/9] LocalStore::addToStore(srcPath): Handle the flat case This helps nix-prefetch-url when using a local store. --- src/libstore/local-store.cc | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fa66c1c90..69f149841 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1097,16 +1097,13 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, { Path srcPath(absPath(_srcPath)); - if (method != FileIngestionMethod::Recursive) - return addToStoreFromDump(readFile(srcPath), name, method, hashAlgo, repair); - /* For computing the NAR hash. */ auto sha256Sink = std::make_unique(htSHA256); /* For computing the store path. In recursive SHA-256 mode, this is the same as the NAR hash, so no need to do it again. */ std::unique_ptr hashSink = - hashAlgo == htSHA256 + method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 ? nullptr : std::make_unique(hashAlgo); @@ -1139,7 +1136,10 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (!inMemory) sink(buf, len); }); - dumpPath(srcPath, sink2, filter); + if (method == FileIngestionMethod::Recursive) + dumpPath(srcPath, sink2, filter); + else + readFile(srcPath, sink2); }); std::unique_ptr delTempDir; @@ -1155,7 +1155,10 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, *source); + if (method == FileIngestionMethod::Recursive) + restorePath(tempPath, *source); + else + writeFile(tempPath, *source); } catch (EndOfFile &) { if (!inMemory) throw; @@ -1188,7 +1191,10 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, if (inMemory) { /* Restore from the NAR in memory. */ StringSource source(nar); - restorePath(realPath, source); + if (method == FileIngestionMethod::Recursive) + restorePath(realPath, source); + else + writeFile(realPath, source); } else { /* Move the temporary path we restored above. */ if (rename(tempPath.c_str(), realPath.c_str())) From 062a584f12a9216464e1664f019b782f86ce78df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 11:21:06 +0200 Subject: [PATCH 5/9] .dir-locals.el: Set c-block-comment-prefix --- .dir-locals.el | 1 + 1 file changed, 1 insertion(+) diff --git a/.dir-locals.el b/.dir-locals.el index 8b21d4e40..a2d1dc48d 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -1,6 +1,7 @@ ((c++-mode . ( (c-file-style . "k&r") (c-basic-offset . 4) + (c-block-comment-prefix . " ") (indent-tabs-mode . nil) (tab-width . 4) (show-trailing-whitespace . t) From 06e3dd9005c1904a17a14a62dcd19813e2c261dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 11:22:48 +0200 Subject: [PATCH 6/9] nix-prefetch-url: Run in constant memory when using RemoteStore Fixes #3684. --- src/nix-prefetch-url/nix-prefetch-url.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..8ca85e894 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -10,6 +10,7 @@ #include "../nix/legacy.hh" #include "progress-bar.hh" #include "tarfile.hh" +#include "archive.hh" #include @@ -200,8 +201,10 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - /* FIXME: inefficient; addToStore() will also hash - this. */ + /* FIXME: inefficient: we're reading/hashing 'tmpFile' + three times. */ + auto [narHash, narSize] = hashPath(htSHA256, tmpFile); + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) @@ -209,13 +212,17 @@ static int _main(int argc, char * * argv) const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - /* Copy the file to the Nix store. FIXME: if RemoteStore - implemented addToStoreFromDump() and downloadFile() - supported a sink, we could stream the download directly - into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); + storePath = store->makeFixedOutputPath(recursive, hash, name); - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + /* Copy the file to the Nix store. */ + ValidPathInfo info(*storePath); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = recursive, .hash = hash }; + auto source = sinkToSource([&](Sink & sink) { + dumpPath(tmpFile, sink); + }); + store->addToStore(info, *source); } stopProgressBar(); From 7f1a86d57c548b4db44f589ffae2f6491d3b2d4c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 12:51:56 +0200 Subject: [PATCH 7/9] nix-store --add-fixed: Run in constant memory --- src/nix-store/nix-store.cc | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..c0274d4b6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -186,8 +186,23 @@ static void opAddFixed(Strings opFlags, Strings opArgs) HashType hashAlgo = parseHashType(opArgs.front()); opArgs.pop_front(); - for (auto & i : opArgs) - cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); + for (auto & i : opArgs) { + auto hash = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, i).first + : hashFile(hashAlgo, i); + auto [narHash, narSize] = hashPath(htSHA256, i); + ValidPathInfo info(store->makeFixedOutputPath(method, hash, baseNameOf(i))); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + auto source = sinkToSource([&](Sink & sink) { + dumpPath(i, sink); + }); + store->addToStore(info, *source); + + std::cout << fmt("%s\n", store->printStorePath(info.path)); + } } From 5dff49f661cd221fc457d1a4660cd36f28266dc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 13:21:37 +0200 Subject: [PATCH 8/9] Factor out commonality between nix-prefetch-url and nix-store --add-fixed --- src/libstore/content-address.cc | 12 +++++++++ src/libstore/content-address.hh | 2 ++ src/libstore/store-api.cc | 32 ++++++++++++++++++++++ src/libstore/store-api.hh | 7 +++++ src/nix-prefetch-url/nix-prefetch-url.cc | 34 +++++++----------------- src/nix-store/nix-store.cc | 19 ++----------- 6 files changed, 64 insertions(+), 42 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3d753836f..6cb69d0a9 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional ca) { return ca ? renderContentAddress(*ca) : ""; } +Hash getContentAddressHash(const ContentAddress & ca) +{ + return std::visit(overloaded { + [](TextHash th) { + return th.hash; + }, + [](FixedOutputHash fsh) { + return fsh.hash; + } + }, ca); +} + } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index ba4797f5b..22a039242 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa); std::optional parseContentAddressOpt(std::string_view rawCaOpt); +Hash getContentAddressHash(const ContentAddress & ca); + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0a8bc9f6..46587a49a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "archive.hh" #include @@ -221,6 +222,37 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional expectedCAHash) +{ + /* FIXME: inefficient: we're reading/hashing 'tmpFile' three + times. */ + + auto hash = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath).first + : hashFile(hashAlgo, srcPath); + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + auto [narHash, narSize] = hashPath(htSHA256, srcPath); + ValidPathInfo info(makeFixedOutputPath(method, hash, name)); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) : Config(params) , state({(size_t) pathInfoCacheSize}) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b122e05d6..b1dd1f478 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -450,6 +450,13 @@ public: FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional expectedCAHash = {}); + // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 8ca85e894..961e7fb6d 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -10,7 +10,6 @@ #include "../nix/legacy.hh" #include "progress-bar.hh" #include "tarfile.hh" -#include "archive.hh" #include @@ -154,14 +153,15 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ - Hash hash, expectedHash(ht); + std::optional expectedHash; + Hash hash; std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, expectedHash, name); + storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); if (store->isValidPath(*storePath)) - hash = expectedHash; + hash = *expectedHash; else storePath.reset(); } @@ -201,28 +201,12 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - /* FIXME: inefficient: we're reading/hashing 'tmpFile' - three times. */ - auto [narHash, narSize] = hashPath(htSHA256, tmpFile); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); - - if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error("hash mismatch for '%1%'", uri); - - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - storePath = store->makeFixedOutputPath(recursive, hash, name); - - /* Copy the file to the Nix store. */ - ValidPathInfo info(*storePath); - info.narHash = narHash; - info.narSize = narSize; - info.ca = FixedOutputHash { .method = recursive, .hash = hash }; - auto source = sinkToSource([&](Sink & sink) { - dumpPath(tmpFile, sink); - }); - store->addToStore(info, *source); + auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c0274d4b6..4fa179105 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -186,23 +186,8 @@ static void opAddFixed(Strings opFlags, Strings opArgs) HashType hashAlgo = parseHashType(opArgs.front()); opArgs.pop_front(); - for (auto & i : opArgs) { - auto hash = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, i).first - : hashFile(hashAlgo, i); - auto [narHash, narSize] = hashPath(htSHA256, i); - ValidPathInfo info(store->makeFixedOutputPath(method, hash, baseNameOf(i))); - info.narHash = narHash; - info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; - - auto source = sinkToSource([&](Sink & sink) { - dumpPath(i, sink); - }); - store->addToStore(info, *source); - - std::cout << fmt("%s\n", store->printStorePath(info.path)); - } + for (auto & i : opArgs) + std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path)); } From 8efa23bb996161af74f89401902450e51e9d4b54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 15:56:24 +0200 Subject: [PATCH 9/9] Avoid a redundant hash --- src/libstore/store-api.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 46587a49a..8d46bb436 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -229,14 +229,17 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* FIXME: inefficient: we're reading/hashing 'tmpFile' three times. */ + auto [narHash, narSize] = hashPath(htSHA256, srcPath); + auto hash = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath).first + ? hashAlgo == htSHA256 + ? narHash + : hashPath(hashAlgo, srcPath).first : hashFile(hashAlgo, srcPath); if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); - auto [narHash, narSize] = hashPath(htSHA256, srcPath); ValidPathInfo info(makeFixedOutputPath(method, hash, name)); info.narHash = narHash; info.narSize = narSize;