From 06e3dd9005c1904a17a14a62dcd19813e2c261dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 11:22:48 +0200 Subject: [PATCH] nix-prefetch-url: Run in constant memory when using RemoteStore Fixes #3684. --- src/nix-prefetch-url/nix-prefetch-url.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 40b05a2f3..8ca85e894 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -10,6 +10,7 @@ #include "../nix/legacy.hh" #include "progress-bar.hh" #include "tarfile.hh" +#include "archive.hh" #include @@ -200,8 +201,10 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - /* FIXME: inefficient; addToStore() will also hash - this. */ + /* FIXME: inefficient: we're reading/hashing 'tmpFile' + three times. */ + auto [narHash, narSize] = hashPath(htSHA256, tmpFile); + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) @@ -209,13 +212,17 @@ static int _main(int argc, char * * argv) const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - /* Copy the file to the Nix store. FIXME: if RemoteStore - implemented addToStoreFromDump() and downloadFile() - supported a sink, we could stream the download directly - into the Nix store. */ - storePath = store->addToStore(name, tmpFile, recursive, ht); + storePath = store->makeFixedOutputPath(recursive, hash, name); - assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); + /* Copy the file to the Nix store. */ + ValidPathInfo info(*storePath); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = recursive, .hash = hash }; + auto source = sinkToSource([&](Sink & sink) { + dumpPath(tmpFile, sink); + }); + store->addToStore(info, *source); } stopProgressBar();