Print a warning when loading a large path into memory

I.e. if you have a derivation with

  src = ./huge-directory;

you'll get a warning that this is not a good idea.
This commit is contained in:
Eelco Dolstra 2014-06-10 13:30:09 +02:00
parent 3c6b8a5215
commit 829af22759
3 changed files with 33 additions and 4 deletions

View file

@ -402,7 +402,10 @@ Path RemoteStore::addToStore(const Path & _srcPath,
writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
writeInt(recursive ? 1 : 0, to); writeInt(recursive ? 1 : 0, to);
writeString(printHashType(hashAlgo), to); writeString(printHashType(hashAlgo), to);
to.written = 0;
to.warn = true;
dumpPath(srcPath, to, filter); dumpPath(srcPath, to, filter);
to.warn = false;
processStderr(); processStderr();
return readStorePath(from); return readStorePath(from);
} }

View file

@ -54,8 +54,24 @@ FdSink::~FdSink()
} }
size_t threshold = 256 * 1024 * 1024;
static void warnLargeDump()
{
printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory");
}
void FdSink::write(const unsigned char * data, size_t len) void FdSink::write(const unsigned char * data, size_t len)
{ {
static bool warned = false;
if (warn && !warned) {
written += len;
if (written > threshold) {
warnLargeDump();
warned = true;
}
}
writeFull(fd, data, len); writeFull(fd, data, len);
} }
@ -256,4 +272,15 @@ template Paths readStrings(Source & source);
template PathSet readStrings(Source & source); template PathSet readStrings(Source & source);
void StringSink::operator () (const unsigned char * data, size_t len)
{
static bool warned = false;
if (!warned && s.size() > threshold) {
warnLargeDump();
warned = true;
}
s.append((const char *) data, len);
}
} }

View file

@ -72,6 +72,8 @@ struct BufferedSource : Source
struct FdSink : BufferedSink struct FdSink : BufferedSink
{ {
int fd; int fd;
bool warn;
size_t written;
FdSink() : fd(-1) { } FdSink() : fd(-1) { }
FdSink(int fd) : fd(fd) { } FdSink(int fd) : fd(fd) { }
@ -95,10 +97,7 @@ struct FdSource : BufferedSource
struct StringSink : Sink struct StringSink : Sink
{ {
string s; string s;
void operator () (const unsigned char * data, size_t len) void operator () (const unsigned char * data, size_t len);
{
s.append((const char *) data, len);
}
}; };