Make 'nix copy --from ssh://...' run in constant memory
For instance, this reduced the memory consumption of $ nix copy --from ssh://localhost --to ~/my-nix /nix/store/1n7x0yv8vq6zi90hfmian84vdhd04bgp-blender-2.79a from 632 MiB to 16 MiB.
This commit is contained in:
parent
92dfc22327
commit
47f7e5585b
|
@ -151,12 +151,7 @@ struct LegacySSHStore : public Store
|
|||
|
||||
conn->to << cmdDumpStorePath << path;
|
||||
conn->to.flush();
|
||||
|
||||
/* FIXME: inefficient. */
|
||||
ParseSink parseSink; /* null sink; just parse the NAR */
|
||||
TeeSource savedNAR(conn->from);
|
||||
parseDump(parseSink, savedNAR);
|
||||
sink(*savedNAR.data);
|
||||
copyNAR(conn->from, sink);
|
||||
}
|
||||
|
||||
PathSet queryAllValidPaths() override { unsupported(); }
|
||||
|
|
|
@ -63,29 +63,12 @@ private:
|
|||
};
|
||||
};
|
||||
|
||||
|
||||
class ForwardSource : public Source
|
||||
{
|
||||
Source & readSource;
|
||||
Sink & writeSink;
|
||||
public:
|
||||
ForwardSource(Source & readSource, Sink & writeSink) : readSource(readSource), writeSink(writeSink) {}
|
||||
size_t read(unsigned char * data, size_t len) override
|
||||
{
|
||||
auto n = readSource.read(data, len);
|
||||
writeSink(data, n);
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
void SSHStore::narFromPath(const Path & path, Sink & sink)
|
||||
{
|
||||
auto conn(connections->get());
|
||||
conn->to << wopNarFromPath << path;
|
||||
conn->processStderr();
|
||||
ParseSink ps;
|
||||
auto fwd = ForwardSource(conn->from, sink);
|
||||
parseDump(ps, fwd);
|
||||
copyNAR(conn->from, sink);
|
||||
}
|
||||
|
||||
ref<FSAccessor> SSHStore::getFSAccessor()
|
||||
|
|
|
@ -350,4 +350,21 @@ void restorePath(const Path & path, Source & source)
|
|||
}
|
||||
|
||||
|
||||
void copyNAR(Source & source, Sink & sink)
|
||||
{
|
||||
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
|
||||
// we should just forward all data directly without parsing.
|
||||
|
||||
ParseSink parseSink; /* null sink; just parse the NAR */
|
||||
|
||||
LambdaSource wrapper([&](unsigned char * data, size_t len) {
|
||||
auto n = source.read(data, len);
|
||||
sink(data, n);
|
||||
return n;
|
||||
});
|
||||
|
||||
parseDump(parseSink, wrapper);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
@ -74,6 +74,9 @@ void parseDump(ParseSink & sink, Source & source);
|
|||
|
||||
void restorePath(const Path & path, Source & source);
|
||||
|
||||
/* Read a NAR from 'source' and write it to 'sink'. */
|
||||
void copyNAR(Source & source, Sink & sink);
|
||||
|
||||
|
||||
// FIXME: global variables are bad m'kay.
|
||||
extern bool useCaseHack;
|
||||
|
|
Loading…
Reference in a new issue