Make 'nix copy --from ssh://...' run in constant memory

For instance, this reduced the memory consumption of

  $ nix copy --from ssh://localhost --to ~/my-nix /nix/store/1n7x0yv8vq6zi90hfmian84vdhd04bgp-blender-2.79a

from 632 MiB to 16 MiB.
This commit is contained in:
Eelco Dolstra 2018-03-21 22:56:02 +01:00
parent 92dfc22327
commit 47f7e5585b
4 changed files with 22 additions and 24 deletions

View file

@ -151,12 +151,7 @@ struct LegacySSHStore : public Store
conn->to << cmdDumpStorePath << path; conn->to << cmdDumpStorePath << path;
conn->to.flush(); conn->to.flush();
copyNAR(conn->from, sink);
/* FIXME: inefficient. */
ParseSink parseSink; /* null sink; just parse the NAR */
TeeSource savedNAR(conn->from);
parseDump(parseSink, savedNAR);
sink(*savedNAR.data);
} }
PathSet queryAllValidPaths() override { unsupported(); } PathSet queryAllValidPaths() override { unsupported(); }

View file

@ -63,29 +63,12 @@ private:
}; };
}; };
class ForwardSource : public Source
{
Source & readSource;
Sink & writeSink;
public:
ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {}
size_t read(unsigned char * data, size_t len) override
{
auto n = readSource.read(data, len);
writeSink(data, n);
return n;
}
};
void SSHStore::narFromPath(const Path & path, Sink & sink) void SSHStore::narFromPath(const Path & path, Sink & sink)
{ {
auto conn(connections->get()); auto conn(connections->get());
conn->to << wopNarFromPath << path; conn->to << wopNarFromPath << path;
conn->processStderr(); conn->processStderr();
ParseSink ps; copyNAR(conn->from, sink);
auto fwd = ForwardSource(conn->from, sink);
parseDump(ps, fwd);
} }
ref<FSAccessor> SSHStore::getFSAccessor() ref<FSAccessor> SSHStore::getFSAccessor()

View file

@ -350,4 +350,21 @@ void restorePath(const Path & path, Source & source)
} }
void copyNAR(Source & source, Sink & sink)
{
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
ParseSink parseSink; /* null sink; just parse the NAR */
LambdaSource wrapper([&](unsigned char * data, size_t len) {
auto n = source.read(data, len);
sink(data, n);
return n;
});
parseDump(parseSink, wrapper);
}
} }

View file

@ -74,6 +74,9 @@ void parseDump(ParseSink & sink, Source & source);
void restorePath(const Path & path, Source & source); void restorePath(const Path & path, Source & source);
/* Read a NAR from 'source' and write it to 'sink'. */
void copyNAR(Source & source, Sink & sink);
// FIXME: global variables are bad m'kay. // FIXME: global variables are bad m'kay.
extern bool useCaseHack; extern bool useCaseHack;