* Use a bounded amount of memory in scanForReferences() by not reading

regular files into memory all at once.
This commit is contained in:
Eelco Dolstra 2006-09-22 13:10:30 +00:00
parent 385c6f8737
commit 666babbbfa

View file

@ -1,5 +1,3 @@
#define __STDC_LIMIT_MACROS
#include "references.hh" #include "references.hh"
#include "hash.hh" #include "hash.hh"
#include "util.hh" #include "util.hh"
@ -13,8 +11,6 @@
#include <dirent.h> #include <dirent.h>
#include <fcntl.h> #include <fcntl.h>
#include <stdint.h>
namespace nix { namespace nix {
@ -80,14 +76,35 @@ void checkPath(const string & path,
AutoCloseFD fd = open(path.c_str(), O_RDONLY); AutoCloseFD fd = open(path.c_str(), O_RDONLY);
if (fd == -1) throw SysError(format("opening file `%1%'") % path); if (fd == -1) throw SysError(format("opening file `%1%'") % path);
if (st.st_size >= SIZE_MAX) size_t bufSize = 1024 * 1024;
throw Error(format("cannot allocate %1% bytes") % st.st_size); assert(refLength <= bufSize);
unsigned char * buf = new unsigned char[bufSize];
unsigned char * buf = new unsigned char[st.st_size]; size_t left = st.st_size;
bool firstBlock = true;
while (left > 0) {
checkInterrupt();
size_t read = left > bufSize ? bufSize : left;
size_t copiedBytes = 0;
readFull(fd, buf, st.st_size); if (!firstBlock) {
/* Move the last (refLength - 1) bytes from the last
block to the start of the buffer to deal with
references that cross block boundaries. */
copiedBytes = refLength - 1;
if (read + copiedBytes > bufSize)
read -= copiedBytes;
memcpy(buf, buf + (bufSize - copiedBytes), copiedBytes);
}
firstBlock = false;
search(st.st_size, buf, ids, seen); readFull(fd, buf + copiedBytes, read);
left -= read;
search(copiedBytes + read, buf, ids, seen);
}
delete[] buf; /* !!! autodelete */ delete[] buf; /* !!! autodelete */
} }