Make adding paths via nix-store --serve run in constant memory

It adds a new operation, cmdAddToStoreNar, that does the same thing as
the corresponding nix-daemon operation, i.e. call addToStore(). This
replaces cmdImportPaths, which has the major issue that it sends the
NAR first and the store path second, thus requiring us to store the
incoming NAR either in memory or on disk until we decide what to do
with it.

For example, this reduces the memory usage of

  $ nix copy --to 'ssh://localhost?remote-store=/tmp/nix' /nix/store/95cwv4q54dc6giaqv6q6p4r02ia2km35-blender-2.79

from 267 MiB to 12 MiB.

Probably fixes #1988.
This commit is contained in:
Eelco Dolstra 2018-08-03 21:10:03 +02:00
parent 34c17fdae5
commit 2825e05d21
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
4 changed files with 70 additions and 15 deletions

View file

@ -28,6 +28,7 @@ struct LegacySSHStore : public Store
FdSink to; FdSink to;
FdSource from; FdSource from;
int remoteVersion; int remoteVersion;
bool good = true;
}; };
std::string host; std::string host;
@ -42,7 +43,7 @@ struct LegacySSHStore : public Store
, connections(make_ref<Pool<Connection>>( , connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections), std::max(1, (int) maxConnections),
[this]() { return openConnection(); }, [this]() { return openConnection(); },
[](const ref<Connection> & r) { return true; } [](const ref<Connection> & r) { return r->good; }
)) ))
, master( , master(
host, host,
@ -58,7 +59,7 @@ struct LegacySSHStore : public Store
{ {
auto conn = make_ref<Connection>(); auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand( conn->sshConn = master.startCommand(
fmt("%s --serve --write", remoteProgram) fmt("command time %s --serve --write", remoteProgram)
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
conn->to = FdSink(conn->sshConn->in.get()); conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get()); conn->from = FdSource(conn->sshConn->out.get());
@ -130,10 +131,38 @@ struct LegacySSHStore : public Store
auto conn(connections->get()); auto conn(connections->get());
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
conn->to
<< cmdAddToStoreNar
<< info.path
<< info.deriver
<< info.narHash.to_string(Base16, false)
<< info.references
<< info.registrationTime
<< info.narSize
<< info.ultimate
<< info.sigs
<< info.ca;
try {
copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to.flush();
} else {
conn->to conn->to
<< cmdImportPaths << cmdImportPaths
<< 1; << 1;
try {
copyNAR(source, conn->to); copyNAR(source, conn->to);
} catch (...) {
conn->good = false;
throw;
}
conn->to conn->to
<< exportMagic << exportMagic
<< info.path << info.path
@ -143,6 +172,8 @@ struct LegacySSHStore : public Store
<< 0; << 0;
conn->to.flush(); conn->to.flush();
}
if (readInt(conn->from) != 1) if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s', info.path, host"); throw Error("failed to add path '%s' to remote host '%s', info.path, host");
} }

View file

@ -5,7 +5,7 @@ namespace nix {
#define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_1 0x390c9deb
#define SERVE_MAGIC_2 0x5452eecb #define SERVE_MAGIC_2 0x5452eecb
#define SERVE_PROTOCOL_VERSION 0x204 #define SERVE_PROTOCOL_VERSION 0x205
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@ -18,6 +18,7 @@ typedef enum {
cmdBuildPaths = 6, cmdBuildPaths = 6,
cmdQueryClosure = 7, cmdQueryClosure = 7,
cmdBuildDerivation = 8, cmdBuildDerivation = 8,
cmdAddToStoreNar = 9,
} ServeCommand; } ServeCommand;
} }

View file

@ -707,6 +707,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
// FIXME: race if addToStore doesn't read source?
store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair, store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair,
dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);

View file

@ -924,6 +924,28 @@ static void opServe(Strings opFlags, Strings opArgs)
break; break;
} }
case cmdAddToStoreNar: {
if (!writeAllowed) throw Error("importing paths is not allowed");
ValidPathInfo info;
info.path = readStorePath(*store, in);
in >> info.deriver;
if (!info.deriver.empty())
store->assertStorePath(info.deriver);
info.narHash = Hash(readString(in), htSHA256);
info.references = readStorePaths<PathSet>(*store, in);
in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in);
in >> info.ca;
// FIXME: race if addToStore doesn't read source?
store->addToStore(info, in, NoRepair, NoCheckSigs);
out << 1; // indicate success
break;
}
default: default:
throw Error(format("unknown serve command %1%") % cmd); throw Error(format("unknown serve command %1%") % cmd);
} }