From 273b288a7e862ac1918064537ff130cc751fa9fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2011 22:31:25 +0000 Subject: [PATCH] * importPath() -> importPaths(). Because of buffering of the input stream it's now necessary for the daemon to process the entire sequence of exported paths, rather than letting the client do it. --- src/libstore/local-store.cc | 17 ++++++++++++-- src/libstore/local-store.hh | 4 +++- src/libstore/remote-store.cc | 41 +++++++++++++++++---------------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 6 ++--- src/libstore/worker-protocol.hh | 4 ++-- src/libutil/serialise.cc | 16 +++++++++---- src/libutil/serialise.hh | 4 ++-- src/nix-store/nix-store.cc | 10 ++++---- src/nix-worker/nix-worker.cc | 28 +++++++++++----------- 10 files changed, 75 insertions(+), 57 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 65b1cdbc8..cf0e2ad1b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1156,7 +1156,7 @@ void LocalStore::exportPath(const Path & path, bool sign, PathSet references; queryReferences(path, references); - writeStringSet(references, hashAndWriteSink); + writeStrings(references, hashAndWriteSink); Path deriver = queryDeriver(path); writeString(deriver, hashAndWriteSink); @@ -1243,7 +1243,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path dstPath = readStorePath(hashAndReadSource); - PathSet references = readStorePaths(hashAndReadSource); + PathSet references = readStorePaths(hashAndReadSource); Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); @@ -1330,6 +1330,19 @@ Path LocalStore::importPath(bool requireSignature, Source & source) } +Paths LocalStore::importPaths(bool requireSignature, Source & source) +{ + Paths res; + while (true) { + unsigned long long n = readLongLong(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); + res.push_back(importPath(requireSignature, source)); + } + return res; +} + + void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7ef01b264..4cb905f67 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -146,7 +146,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); @@ -259,6 +259,8 @@ private: Path createTempDirInStore(); + Path importPath(bool requireSignature, Source & source); + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e976e8fa5..942c5bcf1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -27,13 +27,15 @@ Path readStorePath(Source & from) } -PathSet readStorePaths(Source & from) +template T readStorePaths(Source & from) { - PathSet paths = readStringSet(from); - foreach (PathSet::iterator, i, paths) assertStorePath(*i); + T paths = readStrings(from); + foreach (typename T::iterator, i, paths) assertStorePath(*i); return paths; } +template PathSet readStorePaths(Source & from); + RemoteStore::RemoteStore() { @@ -215,7 +217,7 @@ PathSet RemoteStore::queryValidPaths() openConnection(); writeInt(wopQueryValidPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -242,7 +244,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, if (reply == 0) return false; info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.downloadSize = readLongLong(from); info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; return true; @@ -260,7 +262,7 @@ ValidPathInfo RemoteStore::queryPathInfo(const Path & path) info.deriver = readString(from); if (info.deriver != "") assertStorePath(info.deriver); info.hash = parseHash(htSHA256, readString(from)); - info.references = readStorePaths(from); + info.references = readStorePaths(from); info.registrationTime = readInt(from); info.narSize = readLongLong(from); return info; @@ -285,7 +287,7 @@ void RemoteStore::queryReferences(const Path & path, writeInt(wopQueryReferences, to); writeString(path, to); processStderr(); - PathSet references2 = readStorePaths(from); + PathSet references2 = readStorePaths(from); references.insert(references2.begin(), references2.end()); } @@ -297,7 +299,7 @@ void RemoteStore::queryReferrers(const Path & path, writeInt(wopQueryReferrers, to); writeString(path, to); processStderr(); - PathSet referrers2 = readStorePaths(from); + PathSet referrers2 = readStorePaths(from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -320,7 +322,7 @@ PathSet RemoteStore::queryDerivationOutputs(const Path & path) writeInt(wopQueryDerivationOutputs, to); writeString(path, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -350,7 +352,7 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, writeInt(wopAddTextToStore, to); writeString(name, to); writeString(s, to); - writeStringSet(references, to); + writeStrings(references, to); processStderr(); return readStorePath(from); @@ -369,14 +371,14 @@ void RemoteStore::exportPath(const Path & path, bool sign, } -Path RemoteStore::importPath(bool requireSignature, Source & source) +Paths RemoteStore::importPaths(bool requireSignature, Source & source) { openConnection(); - writeInt(wopImportPath, to); + writeInt(wopImportPaths, to); /* We ignore requireSignature, since the worker forces it to true anyway. */ processStderr(0, &source); - return readStorePath(from); + return readStorePaths(from); } @@ -384,7 +386,7 @@ void RemoteStore::buildDerivations(const PathSet & drvPaths) { openConnection(); writeInt(wopBuildDerivations, to); - writeStringSet(drvPaths, to); + writeStrings(drvPaths, to); processStderr(); readInt(from); } @@ -451,7 +453,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) writeInt(wopCollectGarbage, to); writeInt(options.action, to); - writeStringSet(options.pathsToDelete, to); + writeStrings(options.pathsToDelete, to); writeInt(options.ignoreLiveness, to); writeLongLong(options.maxFreed, to); writeInt(options.maxLinks, to); @@ -463,7 +465,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) processStderr(); - results.paths = readStringSet(from); + results.paths = readStrings(from); results.bytesFreed = readLongLong(from); results.blocksFreed = readLongLong(from); } @@ -474,7 +476,7 @@ PathSet RemoteStore::queryFailedPaths() openConnection(); writeInt(wopQueryFailedPaths, to); processStderr(); - return readStorePaths(from); + return readStorePaths(from); } @@ -482,7 +484,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths) { openConnection(); writeInt(wopClearFailedPaths, to); - writeStringSet(paths, to); + writeStrings(paths, to); processStderr(); readInt(from); } @@ -504,8 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source) size_t len = readInt(from); unsigned char * buf = new unsigned char[len]; AutoDeleteArray d(buf); - size_t n = source->read(buf, len); - writeString(string((const char *) buf, n), to); // !!! inefficient + writeString(buf, source->read(buf, len), to); to.flush(); } else { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 519f46fd1..34a2d91df 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -56,7 +56,7 @@ public: void exportPath(const Path & path, bool sign, Sink & sink); - Path importPath(bool requireSignature, Source & source); + Paths importPaths(bool requireSignature, Source & source); void buildDerivations(const PathSet & drvPaths); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3a2c0daa..d4997c886 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -169,9 +169,9 @@ public: virtual void exportPath(const Path & path, bool sign, Sink & sink) = 0; - /* Import a NAR dump created by exportPath() into the Nix - store. */ - virtual Path importPath(bool requireSignature, Source & source) = 0; + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. */ + virtual Paths importPaths(bool requireSignature, Source & source) = 0; /* Ensure that the output paths of the derivation are valid. If they are already valid, this is a no-op. Otherwise, validity diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index acabd6ca3..760d08a74 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -29,7 +29,6 @@ typedef enum { wopSyncWithGC = 13, wopFindRoots = 14, wopExportPath = 16, - wopImportPath = 17, wopQueryDeriver = 18, wopSetOptions = 19, wopCollectGarbage = 20, @@ -39,6 +38,7 @@ typedef enum { wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, + wopImportPaths = 27, } WorkerOp; @@ -58,7 +58,7 @@ typedef enum { Path readStorePath(Source & from); -PathSet readStorePaths(Source & from); +template T readStorePaths(Source & from); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index ba549c214..c4563ffd1 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -163,13 +163,16 @@ void writeString(const string & s, Sink & sink) } -void writeStringSet(const StringSet & ss, Sink & sink) +template void writeStrings(const T & ss, Sink & sink) { writeInt(ss.size(), sink); - for (StringSet::iterator i = ss.begin(); i != ss.end(); ++i) + foreach (typename T::const_iterator, i, ss) writeString(*i, sink); } +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + void readPadding(size_t len, Source & source) { @@ -234,14 +237,17 @@ string readString(Source & source) } -StringSet readStringSet(Source & source) +template T readStrings(Source & source) { unsigned int count = readInt(source); - StringSet ss; + T ss; while (count--) - ss.insert(readString(source)); + ss.insert(ss.end(), readString(source)); return ss; } +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index efd8e2a06..ded4b12a0 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -116,14 +116,14 @@ void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const unsigned char * buf, size_t len, Sink & sink); void writeString(const string & s, Sink & sink); -void writeStringSet(const StringSet & ss, Sink & sink); +template void writeStrings(const T & ss, Sink & sink); void readPadding(size_t len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); -StringSet readStringSet(Source & source); +template T readStrings(Source & source); MakeError(SerialisationError, Error) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 740033b45..e92ccb153 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -600,12 +600,10 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - while (true) { - unsigned long long n = readLongLong(source); - if (n == 0) break; - if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); - cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; - } + Paths paths = store->importPaths(requireSignature, source); + + foreach (Paths::iterator, i, paths) + cout << format("%1%\n") % *i << std::flush; } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 85e2105b2..5f57b2981 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -327,7 +327,7 @@ static void performOp(unsigned int clientVersion, store->queryReferrers(path, paths); else paths = store->queryDerivationOutputs(path); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -377,7 +377,7 @@ static void performOp(unsigned int clientVersion, case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - PathSet refs = readStorePaths(from); + PathSet refs = readStorePaths(from); startWork(); Path path = store->addTextToStore(suffix, s, refs); stopWork(); @@ -396,19 +396,17 @@ static void performOp(unsigned int clientVersion, break; } - case wopImportPath: { + case wopImportPaths: { startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 9) - throw Error("import not supported; upgrade your client"); TunnelSource source(from); - Path path = store->importPath(true, source); + Paths paths = store->importPaths(true, source); stopWork(); - writeString(path, to); + writeStrings(paths, to); break; } case wopBuildDerivations: { - PathSet drvs = readStorePaths(from); + PathSet drvs = readStorePaths(from); startWork(); store->buildDerivations(drvs); stopWork(); @@ -466,7 +464,7 @@ static void performOp(unsigned int clientVersion, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths(from); + options.pathsToDelete = readStorePaths(from); options.ignoreLiveness = readInt(from); options.maxFreed = readLongLong(from); options.maxLinks = readInt(from); @@ -484,7 +482,7 @@ static void performOp(unsigned int clientVersion, store->collectGarbage(options, results); stopWork(); - writeStringSet(results.paths, to); + writeStrings(results.paths, to); writeLongLong(results.bytesFreed, to); writeLongLong(results.blocksFreed, to); @@ -522,7 +520,7 @@ static void performOp(unsigned int clientVersion, writeInt(res ? 1 : 0, to); if (res) { writeString(info.deriver, to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeLongLong(info.downloadSize, to); if (GET_PROTOCOL_MINOR(clientVersion) >= 7) writeLongLong(info.narSize, to); @@ -534,7 +532,7 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryValidPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } @@ -542,12 +540,12 @@ static void performOp(unsigned int clientVersion, startWork(); PathSet paths = store->queryFailedPaths(); stopWork(); - writeStringSet(paths, to); + writeStrings(paths, to); break; } case wopClearFailedPaths: { - PathSet paths = readStringSet(from); + PathSet paths = readStrings(from); startWork(); store->clearFailedPaths(paths); stopWork(); @@ -562,7 +560,7 @@ static void performOp(unsigned int clientVersion, stopWork(); writeString(info.deriver, to); writeString(printHash(info.hash), to); - writeStringSet(info.references, to); + writeStrings(info.references, to); writeInt(info.registrationTime, to); writeLongLong(info.narSize, to); break;