Handle importing NARs containing files greater than 4 GiB

Also templatize readInt() to work for various integer types.

src/libstore/derivations.cc

@@ -397,8 +397,8 @@ PathSet BasicDerivation::outputPaths() const
 Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
 {
     drv.outputs.clear();
-    auto nr = readInt(in);
+    auto nr = readNum<size_t>(in);
-    for (unsigned int n = 0; n < nr; n++) {
+    for (size_t n = 0; n < nr; n++) {
         auto name = readString(in);
         DerivationOutput o;
         in >> o.path >> o.hashAlgo >> o.hash;
@@ -410,8 +410,8 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
-    nr = readInt(in);
+    nr = readNum<size_t>(in);
-    for (unsigned int n = 0; n < nr; n++) {
+    for (size_t n = 0; n < nr; n++) {
         auto key = readString(in);
         auto value = readString(in);
         drv.env[key] = value;

src/libstore/export-import.cc

@@ -86,7 +86,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
 {
     Paths res;
     while (true) {
-        unsigned long long n = readLongLong(source);
+        auto n = readNum<uint64_t>(source);
         if (n == 0) break;
         if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
 

src/libstore/remote-store.cc

@@ -108,7 +108,7 @@ void RemoteStore::initConnection(Connection & conn)
         unsigned int magic = readInt(conn.from);
         if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
 
-        conn.daemonVersion = readInt(conn.from);
+        conn.from >> conn.daemonVersion;
         if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
             throw Error("Nix daemon protocol version not supported");
         if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
@@ -170,8 +170,7 @@ bool RemoteStore::isValidPathUncached(const Path & path)
     auto conn(connections->get());
     conn->to << wopIsValidPath << path;
     conn->processStderr();
-    unsigned int reply = readInt(conn->from);
+    return readInt(conn->from);
-    return reply != 0;
 }
 
 
@@ -246,8 +245,8 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
 
         conn->to << wopQuerySubstitutablePathInfos << paths;
         conn->processStderr();
-        unsigned int count = readInt(conn->from);
+        size_t count = readNum<size_t>(conn->from);
-        for (unsigned int n = 0; n < count; n++) {
+        for (size_t n = 0; n < count; n++) {
             Path path = readStorePath(*this, conn->from);
             SubstitutablePathInfo & info(infos[path]);
             info.deriver = readString(conn->from);
@@ -277,7 +276,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
             throw;
         }
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
-            bool valid = readInt(conn->from) != 0;
+            bool valid; conn->from >> valid;
             if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path);
         }
         auto info = std::make_shared<ValidPathInfo>();
@@ -286,12 +285,11 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
         if (info->deriver != "") assertStorePath(info->deriver);
         info->narHash = parseHash(htSHA256, readString(conn->from));
         info->references = readStorePaths<PathSet>(*this, conn->from);
-        info->registrationTime = readInt(conn->from);
+        conn->from >> info->registrationTime >> info->narSize;
-        info->narSize = readLongLong(conn->from);
         if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
-            info->ultimate = readInt(conn->from) != 0;
+            conn->from >> info->ultimate;
             info->sigs = readStrings<StringSet>(conn->from);
-            info->ca = readString(conn->from);
+            conn->from >> info->ca;
         }
         return info;
     });
@@ -515,7 +513,7 @@ Roots RemoteStore::findRoots()
     auto conn(connections->get());
     conn->to << wopFindRoots;
     conn->processStderr();
-    unsigned int count = readInt(conn->from);
+    size_t count = readNum<size_t>(conn->from);
     Roots result;
     while (count--) {
         Path link = readString(conn->from);
@@ -563,7 +561,7 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair)
     auto conn(connections->get());
     conn->to << wopVerifyStore << checkContents << repair;
     conn->processStderr();
-    return readInt(conn->from) != 0;
+    return readInt(conn->from);
 }
 
 
@@ -599,7 +597,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
         }
         else if (msg == STDERR_READ) {
             if (!source) throw Error("no source");
-            size_t len = readInt(from);
+            size_t len = readNum<size_t>(from);
             auto buf = std::make_unique<unsigned char[]>(len);
             writeString(buf.get(), source->read(buf.get(), len), to);
             to.flush();

src/libutil/serialise.cc

@@ -194,39 +194,9 @@ void readPadding(size_t len, Source & source)
 }
 
 
-unsigned int readInt(Source & source)
-{
-    unsigned char buf[8];
-    source(buf, sizeof(buf));
-    if (buf[4] || buf[5] || buf[6] || buf[7])
-        throw SerialisationError("implementation cannot deal with > 32-bit integers");
-    return
-        buf[0] |
-        (buf[1] << 8) |
-        (buf[2] << 16) |
-        (buf[3] << 24);
-}
-
-
-unsigned long long readLongLong(Source & source)
-{
-    unsigned char buf[8];
-    source(buf, sizeof(buf));
-    return
-        ((unsigned long long) buf[0]) |
-        ((unsigned long long) buf[1] << 8) |
-        ((unsigned long long) buf[2] << 16) |
-        ((unsigned long long) buf[3] << 24) |
-        ((unsigned long long) buf[4] << 32) |
-        ((unsigned long long) buf[5] << 40) |
-        ((unsigned long long) buf[6] << 48) |
-        ((unsigned long long) buf[7] << 56);
-}
-
-
 size_t readString(unsigned char * buf, size_t max, Source & source)
 {
-    size_t len = readInt(source);
+    auto len = readNum<size_t>(source);
     if (len > max) throw Error("string is too long");
     source(buf, len);
     readPadding(len, source);
@@ -236,7 +206,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source)
 
 string readString(Source & source)
 {
-    size_t len = readInt(source);
+    auto len = readNum<size_t>(source);
     auto buf = std::make_unique<unsigned char[]>(len);
     source(buf.get(), len);
     readPadding(len, source);
@@ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s)
 }
 
 
-Source & operator >> (Source & in, unsigned int & n)
-{
-    n = readInt(in);
-    return in;
-}
-
-
 template<class T> T readStrings(Source & source)
 {
-    unsigned int count = readInt(source);
+    auto count = readNum<size_t>(source);
     T ss;
     while (count--)
         ss.insert(ss.end(), readString(source));

src/libutil/serialise.hh

@@ -177,18 +177,64 @@ Sink & operator << (Sink & sink, const Strings & s);
 Sink & operator << (Sink & sink, const StringSet & s);
 
 
+MakeError(SerialisationError, Error)
+
+
+template<typename T>
+T readNum(Source & source)
+{
+    unsigned char buf[8];
+    source(buf, sizeof(buf));
+
+    uint64_t n =
+        ((unsigned long long) buf[0]) |
+        ((unsigned long long) buf[1] << 8) |
+        ((unsigned long long) buf[2] << 16) |
+        ((unsigned long long) buf[3] << 24) |
+        ((unsigned long long) buf[4] << 32) |
+        ((unsigned long long) buf[5] << 40) |
+        ((unsigned long long) buf[6] << 48) |
+        ((unsigned long long) buf[7] << 56);
+
+    if (n > std::numeric_limits<T>::max())
+        throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name());
+
+    return n;
+}
+
+
+inline unsigned int readInt(Source & source)
+{
+    return readNum<unsigned int>(source);
+}
+
+
+inline uint64_t readLongLong(Source & source)
+{
+    return readNum<uint64_t>(source);
+}
+
+
 void readPadding(size_t len, Source & source);
-unsigned int readInt(Source & source);
-unsigned long long readLongLong(Source & source);
 size_t readString(unsigned char * buf, size_t max, Source & source);
 string readString(Source & source);
 template<class T> T readStrings(Source & source);
 
 Source & operator >> (Source & in, string & s);
-Source & operator >> (Source & in, unsigned int & n);
 
+template<typename T>
+Source & operator >> (Source & in, T & n)
+{
+    n = readNum<T>(in);
+    return in;
+}
 
-MakeError(SerialisationError, Error)
+template<typename T>
+Source & operator >> (Source & in, bool & b)
+{
+    b = readNum<uint64_t>(in);
+    return in;
+}
 
 
 }

src/nix-daemon/nix-daemon.cc

@@ -273,10 +273,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopAddToStore: {
-        string baseName = readString(from);
+        bool fixed, recursive;
-        bool fixed = readInt(from) == 1; /* obsolete */
+        std::string s, baseName;
-        bool recursive = readInt(from) == 1;
+        from >> baseName >> fixed /* obsolete */ >> recursive >> s;
-        string s = readString(from);
         /* Compatibility hack. */
         if (!fixed) {
             s = "sha256";
@@ -417,8 +416,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         GCOptions options;
         options.action = (GCOptions::GCAction) readInt(from);
         options.pathsToDelete = readStorePaths<PathSet>(*store, from);
-        options.ignoreLiveness = readInt(from);
+        from >> options.ignoreLiveness >> options.maxFreed;
-        options.maxFreed = readLongLong(from);
         // obsolete fields
         readInt(from);
         readInt(from);
@@ -438,8 +436,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     }
 
     case wopSetOptions: {
-        settings.keepFailed = readInt(from) != 0;
+        from >> settings.keepFailed;
-        settings.keepGoing = readInt(from) != 0;
+        from >> settings.keepGoing;
         settings.set("build-fallback", readInt(from) ? "true" : "false");
         verbosity = (Verbosity) readInt(from);
         settings.set("build-max-jobs", std::to_string(readInt(from)));
@@ -539,8 +537,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
         break;
 
     case wopVerifyStore: {
-        bool checkContents = readInt(from) != 0;
+        bool checkContents, repair;
-        bool repair = readInt(from) != 0;
+        from >> checkContents >> repair;
         startWork();
         if (repair && !trusted)
             throw Error("you are not privileged to repair paths");
@@ -573,19 +571,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
     case wopAddToStoreNar: {
         ValidPathInfo info;
         info.path = readStorePath(*store, from);
-        info.deriver = readString(from);
+        from >> info.deriver;
         if (!info.deriver.empty())
             store->assertStorePath(info.deriver);
         info.narHash = parseHash(htSHA256, readString(from));
         info.references = readStorePaths<PathSet>(*store, from);
-        info.registrationTime = readInt(from);
+        from >> info.registrationTime >> info.narSize >> info.ultimate;
-        info.narSize = readLongLong(from);
-        info.ultimate = readLongLong(from);
         info.sigs = readStrings<StringSet>(from);
-        info.ca = readString(from);
+        from >> info.ca;
         auto nar = make_ref<std::string>(readString(from));
-        auto repair = readInt(from) ? true : false;
+        bool repair, dontCheckSigs;
-        auto dontCheckSigs = readInt(from) ? true : false;
+        from >> repair >> dontCheckSigs;
         if (!trusted && dontCheckSigs)
             dontCheckSigs = false;
         startWork();

src/nix-store/nix-store.cc

@@ -839,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs)
         settings.maxSilentTime = readInt(in);
         settings.buildTimeout = readInt(in);
         if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
-            settings.maxLogSize = readInt(in);
+            in >> settings.maxLogSize;
         if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
             settings.set("build-repeat", std::to_string(readInt(in)));
             settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false");