From 6185d25e523a3cd223dd6f6aca10cf6ff15b4823 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Mar 2018 23:36:30 +0200 Subject: [PATCH] Make 'nix copy --to daemon' run in constant memory (daemon side) Continuation of 97002b684c902dadcd351a67208f9c2a88ff8f8f. This makes the daemon use constant memory. For example, it reduces the daemon's maximum RSS on $ nix copy --from ~/my-nix --to daemon /nix/store/1n7x0yv8vq6zi90hfmian84vdhd04bgp-blender-2.79a from 264 MiB to 7 MiB. We now use a TunnelSource to prevent the connection from ending up in an undefined state if an exception is thrown while the NAR is being sent. Issue https://github.com/NixOS/nix/issues/1681. --- src/libstore/remote-store.cc | 5 +++-- src/libstore/worker-protocol.hh | 2 +- src/nix-daemon/nix-daemon.cc | 16 +++++++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 080cef93d..d53037558 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -411,8 +411,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.references << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca << repair << !checkSigs; - copyNAR(source, conn->to); - conn->processStderr(); + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) copyNAR(source, conn->to); + conn->processStderr(0, tunnel ? &source : nullptr); } } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 996e1d253..5ebdfaf13 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x114 +#define PROTOCOL_VERSION 0x115 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index baf169804..590f96211 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -690,12 +690,22 @@ static void performOp(TunnelLogger * logger, ref store, if (!trusted) info.ultimate = false; - TeeSink tee(from); - parseDump(tee, tee.source); + std::string saved; + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique(saved); + } logger->startWork(); - store.cast()->addToStore(info, tee.source.data, (RepairFlag) repair, + + store.cast()->addToStore(info, *source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + logger->stopWork(); break; }