From c4a40949d945b4a3be85ad68b8cfb449843f34a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Mar 2017 13:52:54 +0100 Subject: [PATCH] Handle importing NARs containing files greater than 4 GiB Also templatize readInt() to work for various integer types. --- src/libstore/derivations.cc | 8 +++--- src/libstore/export-import.cc | 2 +- src/libstore/remote-store.cc | 24 +++++++--------- src/libutil/serialise.cc | 43 ++-------------------------- src/libutil/serialise.hh | 54 ++++++++++++++++++++++++++++++++--- src/nix-daemon/nix-daemon.cc | 32 +++++++++------------ src/nix-store/nix-store.cc | 2 +- 7 files changed, 84 insertions(+), 81 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 79526c594..ce1ac7d33 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -397,8 +397,8 @@ PathSet BasicDerivation::outputPaths() const Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) { drv.outputs.clear(); - auto nr = readInt(in); - for (unsigned int n = 0; n < nr; n++) { + auto nr = readNum(in); + for (size_t n = 0; n < nr; n++) { auto name = readString(in); DerivationOutput o; in >> o.path >> o.hashAlgo >> o.hash; @@ -410,8 +410,8 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) in >> drv.platform >> drv.builder; drv.args = readStrings(in); - nr = readInt(in); - for (unsigned int n = 0; n < nr; n++) { + nr = readNum(in); + for (size_t n = 0; n < nr; n++) { auto key = readString(in); auto value = readString(in); drv.env[key] = value; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index c5618c826..e584ae538 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -86,7 +86,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr accessor, { Paths res; while (true) { - unsigned long long n = readLongLong(source); + auto n = readNum(source); if (n == 0) break; if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’"); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7f398685a..642825914 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -108,7 +108,7 @@ void RemoteStore::initConnection(Connection & conn) unsigned int magic = readInt(conn.from); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); - conn.daemonVersion = readInt(conn.from); + conn.from >> conn.daemonVersion; if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) throw Error("Nix daemon protocol version not supported"); if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) @@ -170,8 +170,7 @@ bool RemoteStore::isValidPathUncached(const Path & path) auto conn(connections->get()); conn->to << wopIsValidPath << path; conn->processStderr(); - unsigned int reply = readInt(conn->from); - return reply != 0; + return readInt(conn->from); } @@ -246,8 +245,8 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, conn->to << wopQuerySubstitutablePathInfos << paths; conn->processStderr(); - unsigned int count = readInt(conn->from); - for (unsigned int n = 0; n < count; n++) { + size_t count = readNum(conn->from); + for (size_t n = 0; n < count; n++) { Path path = readStorePath(*this, conn->from); SubstitutablePathInfo & info(infos[path]); info.deriver = readString(conn->from); @@ -277,7 +276,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path, throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid = readInt(conn->from) != 0; + bool valid; conn->from >> valid; if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path); } auto info = std::make_shared(); @@ -286,12 +285,11 @@ void RemoteStore::queryPathInfoUncached(const Path & path, if (info->deriver != "") assertStorePath(info->deriver); info->narHash = parseHash(htSHA256, readString(conn->from)); info->references = readStorePaths(*this, conn->from); - info->registrationTime = readInt(conn->from); - info->narSize = readLongLong(conn->from); + conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - info->ultimate = readInt(conn->from) != 0; + conn->from >> info->ultimate; info->sigs = readStrings(conn->from); - info->ca = readString(conn->from); + conn->from >> info->ca; } return info; }); @@ -515,7 +513,7 @@ Roots RemoteStore::findRoots() auto conn(connections->get()); conn->to << wopFindRoots; conn->processStderr(); - unsigned int count = readInt(conn->from); + size_t count = readNum(conn->from); Roots result; while (count--) { Path link = readString(conn->from); @@ -563,7 +561,7 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair) auto conn(connections->get()); conn->to << wopVerifyStore << checkContents << repair; conn->processStderr(); - return readInt(conn->from) != 0; + return readInt(conn->from); } @@ -599,7 +597,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) } else if (msg == STDERR_READ) { if (!source) throw Error("no source"); - size_t len = readInt(from); + size_t len = readNum(from); auto buf = std::make_unique(len); writeString(buf.get(), source->read(buf.get(), len), to); to.flush(); diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index a68f7a0fa..6064e15f5 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -194,39 +194,9 @@ void readPadding(size_t len, Source & source) } -unsigned int readInt(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); - if (buf[4] || buf[5] || buf[6] || buf[7]) - throw SerialisationError("implementation cannot deal with > 32-bit integers"); - return - buf[0] | - (buf[1] << 8) | - (buf[2] << 16) | - (buf[3] << 24); -} - - -unsigned long long readLongLong(Source & source) -{ - unsigned char buf[8]; - source(buf, sizeof(buf)); - return - ((unsigned long long) buf[0]) | - ((unsigned long long) buf[1] << 8) | - ((unsigned long long) buf[2] << 16) | - ((unsigned long long) buf[3] << 24) | - ((unsigned long long) buf[4] << 32) | - ((unsigned long long) buf[5] << 40) | - ((unsigned long long) buf[6] << 48) | - ((unsigned long long) buf[7] << 56); -} - - size_t readString(unsigned char * buf, size_t max, Source & source) { - size_t len = readInt(source); + auto len = readNum(source); if (len > max) throw Error("string is too long"); source(buf, len); readPadding(len, source); @@ -236,7 +206,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source) string readString(Source & source) { - size_t len = readInt(source); + auto len = readNum(source); auto buf = std::make_unique(len); source(buf.get(), len); readPadding(len, source); @@ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s) } -Source & operator >> (Source & in, unsigned int & n) -{ - n = readInt(in); - return in; -} - - template T readStrings(Source & source) { - unsigned int count = readInt(source); + auto count = readNum(source); T ss; while (count--) ss.insert(ss.end(), readString(source)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5646d08c1..3072f422e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -177,18 +177,64 @@ Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); +MakeError(SerialisationError, Error) + + +template +T readNum(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + + uint64_t n = + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); + + if (n > std::numeric_limits::max()) + throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name()); + + return n; +} + + +inline unsigned int readInt(Source & source) +{ + return readNum(source); +} + + +inline uint64_t readLongLong(Source & source) +{ + return readNum(source); +} + + void readPadding(size_t len, Source & source); -unsigned int readInt(Source & source); -unsigned long long readLongLong(Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); string readString(Source & source); template T readStrings(Source & source); Source & operator >> (Source & in, string & s); -Source & operator >> (Source & in, unsigned int & n); +template +Source & operator >> (Source & in, T & n) +{ + n = readNum(in); + return in; +} -MakeError(SerialisationError, Error) +template +Source & operator >> (Source & in, bool & b) +{ + b = readNum(in); + return in; +} } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 9fbc43b82..a1b3f2f6f 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -273,10 +273,9 @@ static void performOp(ref store, bool trusted, unsigned int clientVe } case wopAddToStore: { - string baseName = readString(from); - bool fixed = readInt(from) == 1; /* obsolete */ - bool recursive = readInt(from) == 1; - string s = readString(from); + bool fixed, recursive; + std::string s, baseName; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; /* Compatibility hack. */ if (!fixed) { s = "sha256"; @@ -340,7 +339,7 @@ static void performOp(ref store, bool trusted, unsigned int clientVe PathSet drvs = readStorePaths(*store, from); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode)readInt(from); + mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" clients. */ @@ -417,8 +416,7 @@ static void performOp(ref store, bool trusted, unsigned int clientVe GCOptions options; options.action = (GCOptions::GCAction) readInt(from); options.pathsToDelete = readStorePaths(*store, from); - options.ignoreLiveness = readInt(from); - options.maxFreed = readLongLong(from); + from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); readInt(from); @@ -438,8 +436,8 @@ static void performOp(ref store, bool trusted, unsigned int clientVe } case wopSetOptions: { - settings.keepFailed = readInt(from) != 0; - settings.keepGoing = readInt(from) != 0; + from >> settings.keepFailed; + from >> settings.keepGoing; settings.set("build-fallback", readInt(from) ? "true" : "false"); verbosity = (Verbosity) readInt(from); settings.set("build-max-jobs", std::to_string(readInt(from))); @@ -539,8 +537,8 @@ static void performOp(ref store, bool trusted, unsigned int clientVe break; case wopVerifyStore: { - bool checkContents = readInt(from) != 0; - bool repair = readInt(from) != 0; + bool checkContents, repair; + from >> checkContents >> repair; startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); @@ -573,19 +571,17 @@ static void performOp(ref store, bool trusted, unsigned int clientVe case wopAddToStoreNar: { ValidPathInfo info; info.path = readStorePath(*store, from); - info.deriver = readString(from); + from >> info.deriver; if (!info.deriver.empty()) store->assertStorePath(info.deriver); info.narHash = parseHash(htSHA256, readString(from)); info.references = readStorePaths(*store, from); - info.registrationTime = readInt(from); - info.narSize = readLongLong(from); - info.ultimate = readLongLong(from); + from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = readString(from); + from >> info.ca; auto nar = make_ref(readString(from)); - auto repair = readInt(from) ? true : false; - auto dontCheckSigs = readInt(from) ? true : false; + bool repair, dontCheckSigs; + from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; startWork(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 0aabe66c5..bd889c851 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -839,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs) settings.maxSilentTime = readInt(in); settings.buildTimeout = readInt(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readInt(in); + in >> settings.maxLogSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { settings.set("build-repeat", std::to_string(readInt(in))); settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false");