Make <nix/fetchurl.nix> run in constant memory

E.g.

  nix-build --store ~/my-nix/ -E 'import <nix/fetchurl.nix> { url = https://cache.nixos.org/nar/0nwi996rgq4b914qyx0mv2wq4k80hjac7xilikavagw7kxmn2iiv.nar.xz; sha256 = "0nwi996rgq4b914qyx0mv2wq4k80hjac7xilikavagw7kxmn2iiv"; }'

now runs in 17 MiB (was 70 MiB), while

  nix-build --store ~/my-nix/ -E 'import <nix/fetchurl.nix> { url = https://cache.nixos.org/nar/0nwi996rgq4b914qyx0mv2wq4k80hjac7xilikavagw7kxmn2iiv.nar.xz; sha256 = "0d2fxljdih3nc5dqx41hjzic3141ajil94m8kdbpryq569dpsbvb"; unpack = true; }'

runs in 17 MiB (was 346 MiB).

src/libstore/builtins/fetchurl.cc

@@ -22,52 +22,60 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
         return i->second;
     };
 
-    auto fetch = [&](const string & url) {
+    Path storePath = getAttr("out");
+    auto mainUrl = getAttr("url");
+
+    /* Note: have to use a fresh downloader here because we're in
+       a forked process. */
+    auto downloader = makeDownloader();
+
+    auto fetch = [&](const std::string & url) {
+
+        auto source = sinkToSource([&](Sink & sink) {
+
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
             DownloadRequest request(url);
             request.verifyTLS = false;
             request.decompress = false;
 
-        /* Note: have to use a fresh downloader here because we're in
+            downloader->download(std::move(request), sink);
-           a forked process. */
+        });
-        auto data = makeDownloader()->download(request);
-        assert(data.data);
 
-        return data.data;
+        if (get(drv.env, "unpack", "") == "1") {
-    };
 
-    std::shared_ptr<std::string> data;
+            if (hasSuffix(mainUrl, ".xz")) {
-
+                auto source2 = sinkToSource([&](Sink & sink) {
-    if (getAttr("outputHashMode") == "flat")
+                    decompress("xz", *source, sink);
-        for (auto hashedMirror : settings.hashedMirrors.get())
+                });
-            try {
+                restorePath(storePath, *source2);
-                if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
-                auto ht = parseHashType(getAttr("outputHashAlgo"));
-                data = fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
-                break;
-            } catch (Error & e) {
-                debug(e.what());
-            }
-
-    if (!data) data = fetch(getAttr("url"));
-
-    Path storePath = getAttr("out");
-
-    auto unpack = drv.env.find("unpack");
-    if (unpack != drv.env.end() && unpack->second == "1") {
-        if (string(*data, 0, 6) == string("\xfd" "7zXZ\0", 6))
-            data = decompress("xz", *data);
-        StringSource source(*data);
-        restorePath(storePath, source);
             } else
-        writeFile(storePath, *data);
+                restorePath(storePath, *source);
+
+        } else
+              writeFile(storePath, *source);
 
         auto executable = drv.env.find("executable");
         if (executable != drv.env.end() && executable->second == "1") {
             if (chmod(storePath.c_str(), 0755) == -1)
                 throw SysError(format("making '%1%' executable") % storePath);
         }
+    };
+
+    /* Try the hashed mirrors first. */
+    if (getAttr("outputHashMode") == "flat")
+        for (auto hashedMirror : settings.hashedMirrors.get())
+            try {
+                if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
+                auto ht = parseHashType(getAttr("outputHashAlgo"));
+                fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false));
+                return;
+            } catch (Error & e) {
+                debug(e.what());
+            }
+
+    /* Otherwise try the specified URL. */
+    fetch(mainUrl);
 }
 
 }

src/libutil/util.cc

@@ -328,6 +328,23 @@ void writeFile(const Path & path, const string & s, mode_t mode)
 }
 
 
+void writeFile(const Path & path, Source & source, mode_t mode)
+{
+    AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
+    if (!fd)
+        throw SysError(format("opening file '%1%'") % path);
+
+    std::vector<unsigned char> buf(64 * 1024);
+
+    while (true) {
+        try {
+            auto n = source.read(buf.data(), buf.size());
+            writeFull(fd.get(), (unsigned char *) buf.data(), n);
+        } catch (EndOfFile &) { break; }
+    }
+}
+
+
 string readLine(int fd)
 {
     string s;

src/libutil/util.hh

@@ -103,6 +103,8 @@ void readFile(const Path & path, Sink & sink);
 /* Write a string to a file. */
 void writeFile(const Path & path, const string & s, mode_t mode = 0666);
 
+void writeFile(const Path & path, Source & source, mode_t mode = 0666);
+
 /* Read a line from a file descriptor. */
 string readLine(int fd);