From 666babbbfa2fc168b8d511a35065d352b4979dae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Sep 2006 13:10:30 +0000 Subject: [PATCH] * Use a bounded amount of memory in scanForReferences() by not reading regular files into memory all at once. --- src/libstore/references.cc | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 836d9aa9a..19cb288bd 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -1,5 +1,3 @@ -#define __STDC_LIMIT_MACROS - #include "references.hh" #include "hash.hh" #include "util.hh" @@ -13,8 +11,6 @@ #include #include -#include - namespace nix { @@ -80,14 +76,35 @@ void checkPath(const string & path, AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file `%1%'") % path); - if (st.st_size >= SIZE_MAX) - throw Error(format("cannot allocate %1% bytes") % st.st_size); + size_t bufSize = 1024 * 1024; + assert(refLength <= bufSize); + unsigned char * buf = new unsigned char[bufSize]; - unsigned char * buf = new unsigned char[st.st_size]; + size_t left = st.st_size; + bool firstBlock = true; + + while (left > 0) { + checkInterrupt(); + + size_t read = left > bufSize ? bufSize : left; + size_t copiedBytes = 0; - readFull(fd, buf, st.st_size); + if (!firstBlock) { + /* Move the last (refLength - 1) bytes from the last + block to the start of the buffer to deal with + references that cross block boundaries. */ + copiedBytes = refLength - 1; + if (read + copiedBytes > bufSize) + read -= copiedBytes; + memcpy(buf, buf + (bufSize - copiedBytes), copiedBytes); + } + firstBlock = false; - search(st.st_size, buf, ids, seen); + readFull(fd, buf + copiedBytes, read); + left -= read; + + search(copiedBytes + read, buf, ids, seen); + } delete[] buf; /* !!! autodelete */ }