From 82ae85de2759eaa68bb2411a1f0a640cf9f8e76a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Dec 2008 18:05:14 +0000 Subject: [PATCH] * addToStore() in nix-worker: don't write the NAR dump received from the client to a temporary directory, as that is highly inefficient. --- src/libstore/local-store.cc | 45 ++++++++++++++++------------ src/libstore/local-store.hh | 7 +++++ src/nix-worker/nix-worker.cc | 58 ++++++++++++++++++++++++++++++++---- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bb53caacc..4629402fb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -655,24 +655,12 @@ void LocalStore::invalidatePath(const Path & path) } -Path LocalStore::addToStore(const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter) +Path LocalStore::addToStoreFromDump(const string & dump, const string & name, + bool recursive, HashType hashAlgo) { - Path srcPath(absPath(_srcPath)); - debug(format("adding `%1%' to the store") % srcPath); + Hash h = hashString(hashAlgo, dump); - /* Read the whole path into memory. This is not a very scalable - method for very large paths, but `copyPath' is mainly used for - small files. */ - StringSink sink; - if (recursive) - dumpPath(srcPath, sink, filter); - else - sink.s = readFile(srcPath); - - Hash h = hashString(hashAlgo, sink.s); - - Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, baseNameOf(srcPath)); + Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); addTempRoot(dstPath); @@ -688,10 +676,10 @@ Path LocalStore::addToStore(const Path & _srcPath, if (pathExists(dstPath)) deletePathWrapped(dstPath); if (recursive) { - StringSource source(sink.s); + StringSource source(dump); restorePath(dstPath, source); } else - writeStringToFile(dstPath, sink.s); + writeStringToFile(dstPath, dump); canonicalisePathMetaData(dstPath); @@ -701,7 +689,7 @@ Path LocalStore::addToStore(const Path & _srcPath, sha256); otherwise, compute it here. */ registerValidPath(dstPath, (recursive && hashAlgo == htSHA256) ? h : - (recursive ? hashString(htSHA256, sink.s) : hashPath(htSHA256, dstPath)), + (recursive ? hashString(htSHA256, dump) : hashPath(htSHA256, dstPath)), PathSet(), ""); } @@ -712,6 +700,25 @@ Path LocalStore::addToStore(const Path & _srcPath, } +Path LocalStore::addToStore(const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter) +{ + Path srcPath(absPath(_srcPath)); + debug(format("adding `%1%' to the store") % srcPath); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) + dumpPath(srcPath, sink, filter); + else + sink.s = readFile(srcPath); + + return addToStoreFromDump(sink.s, baseNameOf(srcPath), recursive, hashAlgo); +} + + Path LocalStore::addTextToStore(const string & name, const string & s, const PathSet & references) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 77e46fc3c..f201ddbde 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -93,6 +93,13 @@ public: bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter); + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ + Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256); + Path addTextToStore(const string & name, const string & s, const PathSet & references); diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index fd34bea67..2c3f44cc0 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -223,6 +223,43 @@ struct TunnelSource : Source }; +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink +{ + string s; + + void createDirectory(const Path & path) + { + throw Error("regular file expected"); + } + + void receiveContents(unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } + + void createSymlink(const Path & path, const string & target) + { + throw Error("regular file expected"); + } +}; + + +/* Adapter class of a Source that saves all data read to `s'. */ +struct SavingSourceAdapter : Source +{ + Source & orig; + string s; + SavingSourceAdapter(Source & orig) : orig(orig) { } + void operator () (unsigned char * data, unsigned int len) + { + orig(data, len); + s.append((const char *) data, len); + } +}; + + static void performOp(unsigned int clientVersion, Source & from, Sink & to, unsigned int op) { @@ -299,13 +336,22 @@ static void performOp(unsigned int clientVersion, } HashType hashAlgo = parseHashType(s); - Path tmp = createTempDir(); - AutoDelete delTmp(tmp); - Path tmp2 = tmp + "/" + baseName; - restorePath(tmp2, from); - + SavingSourceAdapter savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else { + parseDump(savedRegular, from); + } + startWork(); - Path path = store->addToStore(tmp2, recursive, hashAlgo); + Path path = dynamic_cast(store.get()) + ->addToStoreFromDump(recursive ? savedNAR.s : savedRegular.s, baseName, recursive, hashAlgo); stopWork(); writeString(path, to);