Handle importing NARs containing files greater than 4 GiB

Also templatize readInt() to work for various integer types.
This commit is contained in:
Eelco Dolstra 2017-03-01 13:52:54 +01:00
parent 0780805246
commit c4a40949d9
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
7 changed files with 84 additions and 81 deletions

View file

@ -397,8 +397,8 @@ PathSet BasicDerivation::outputPaths() const
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
{ {
drv.outputs.clear(); drv.outputs.clear();
auto nr = readInt(in); auto nr = readNum<size_t>(in);
for (unsigned int n = 0; n < nr; n++) { for (size_t n = 0; n < nr; n++) {
auto name = readString(in); auto name = readString(in);
DerivationOutput o; DerivationOutput o;
in >> o.path >> o.hashAlgo >> o.hash; in >> o.path >> o.hashAlgo >> o.hash;
@ -410,8 +410,8 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
in >> drv.platform >> drv.builder; in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in); drv.args = readStrings<Strings>(in);
nr = readInt(in); nr = readNum<size_t>(in);
for (unsigned int n = 0; n < nr; n++) { for (size_t n = 0; n < nr; n++) {
auto key = readString(in); auto key = readString(in);
auto value = readString(in); auto value = readString(in);
drv.env[key] = value; drv.env[key] = value;

View file

@ -86,7 +86,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
{ {
Paths res; Paths res;
while (true) { while (true) {
unsigned long long n = readLongLong(source); auto n = readNum<uint64_t>(source);
if (n == 0) break; if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by nix-store --export"); if (n != 1) throw Error("input doesn't look like something created by nix-store --export");

View file

@ -108,7 +108,7 @@ void RemoteStore::initConnection(Connection & conn)
unsigned int magic = readInt(conn.from); unsigned int magic = readInt(conn.from);
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
conn.daemonVersion = readInt(conn.from); conn.from >> conn.daemonVersion;
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported"); throw Error("Nix daemon protocol version not supported");
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
@ -170,8 +170,7 @@ bool RemoteStore::isValidPathUncached(const Path & path)
auto conn(connections->get()); auto conn(connections->get());
conn->to << wopIsValidPath << path; conn->to << wopIsValidPath << path;
conn->processStderr(); conn->processStderr();
unsigned int reply = readInt(conn->from); return readInt(conn->from);
return reply != 0;
} }
@ -246,8 +245,8 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
conn->to << wopQuerySubstitutablePathInfos << paths; conn->to << wopQuerySubstitutablePathInfos << paths;
conn->processStderr(); conn->processStderr();
unsigned int count = readInt(conn->from); size_t count = readNum<size_t>(conn->from);
for (unsigned int n = 0; n < count; n++) { for (size_t n = 0; n < count; n++) {
Path path = readStorePath(*this, conn->from); Path path = readStorePath(*this, conn->from);
SubstitutablePathInfo & info(infos[path]); SubstitutablePathInfo & info(infos[path]);
info.deriver = readString(conn->from); info.deriver = readString(conn->from);
@ -277,7 +276,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
throw; throw;
} }
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid = readInt(conn->from) != 0; bool valid; conn->from >> valid;
if (!valid) throw InvalidPath(format("path %s is not valid") % path); if (!valid) throw InvalidPath(format("path %s is not valid") % path);
} }
auto info = std::make_shared<ValidPathInfo>(); auto info = std::make_shared<ValidPathInfo>();
@ -286,12 +285,11 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
if (info->deriver != "") assertStorePath(info->deriver); if (info->deriver != "") assertStorePath(info->deriver);
info->narHash = parseHash(htSHA256, readString(conn->from)); info->narHash = parseHash(htSHA256, readString(conn->from));
info->references = readStorePaths<PathSet>(*this, conn->from); info->references = readStorePaths<PathSet>(*this, conn->from);
info->registrationTime = readInt(conn->from); conn->from >> info->registrationTime >> info->narSize;
info->narSize = readLongLong(conn->from);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
info->ultimate = readInt(conn->from) != 0; conn->from >> info->ultimate;
info->sigs = readStrings<StringSet>(conn->from); info->sigs = readStrings<StringSet>(conn->from);
info->ca = readString(conn->from); conn->from >> info->ca;
} }
return info; return info;
}); });
@ -515,7 +513,7 @@ Roots RemoteStore::findRoots()
auto conn(connections->get()); auto conn(connections->get());
conn->to << wopFindRoots; conn->to << wopFindRoots;
conn->processStderr(); conn->processStderr();
unsigned int count = readInt(conn->from); size_t count = readNum<size_t>(conn->from);
Roots result; Roots result;
while (count--) { while (count--) {
Path link = readString(conn->from); Path link = readString(conn->from);
@ -563,7 +561,7 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair)
auto conn(connections->get()); auto conn(connections->get());
conn->to << wopVerifyStore << checkContents << repair; conn->to << wopVerifyStore << checkContents << repair;
conn->processStderr(); conn->processStderr();
return readInt(conn->from) != 0; return readInt(conn->from);
} }
@ -599,7 +597,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
} }
else if (msg == STDERR_READ) { else if (msg == STDERR_READ) {
if (!source) throw Error("no source"); if (!source) throw Error("no source");
size_t len = readInt(from); size_t len = readNum<size_t>(from);
auto buf = std::make_unique<unsigned char[]>(len); auto buf = std::make_unique<unsigned char[]>(len);
writeString(buf.get(), source->read(buf.get(), len), to); writeString(buf.get(), source->read(buf.get(), len), to);
to.flush(); to.flush();

View file

@ -194,39 +194,9 @@ void readPadding(size_t len, Source & source)
} }
unsigned int readInt(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
if (buf[4] || buf[5] || buf[6] || buf[7])
throw SerialisationError("implementation cannot deal with > 32-bit integers");
return
buf[0] |
(buf[1] << 8) |
(buf[2] << 16) |
(buf[3] << 24);
}
unsigned long long readLongLong(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
return
((unsigned long long) buf[0]) |
((unsigned long long) buf[1] << 8) |
((unsigned long long) buf[2] << 16) |
((unsigned long long) buf[3] << 24) |
((unsigned long long) buf[4] << 32) |
((unsigned long long) buf[5] << 40) |
((unsigned long long) buf[6] << 48) |
((unsigned long long) buf[7] << 56);
}
size_t readString(unsigned char * buf, size_t max, Source & source) size_t readString(unsigned char * buf, size_t max, Source & source)
{ {
size_t len = readInt(source); auto len = readNum<size_t>(source);
if (len > max) throw Error("string is too long"); if (len > max) throw Error("string is too long");
source(buf, len); source(buf, len);
readPadding(len, source); readPadding(len, source);
@ -236,7 +206,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source)
string readString(Source & source) string readString(Source & source)
{ {
size_t len = readInt(source); auto len = readNum<size_t>(source);
auto buf = std::make_unique<unsigned char[]>(len); auto buf = std::make_unique<unsigned char[]>(len);
source(buf.get(), len); source(buf.get(), len);
readPadding(len, source); readPadding(len, source);
@ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s)
} }
Source & operator >> (Source & in, unsigned int & n)
{
n = readInt(in);
return in;
}
template<class T> T readStrings(Source & source) template<class T> T readStrings(Source & source)
{ {
unsigned int count = readInt(source); auto count = readNum<size_t>(source);
T ss; T ss;
while (count--) while (count--)
ss.insert(ss.end(), readString(source)); ss.insert(ss.end(), readString(source));

View file

@ -177,18 +177,64 @@ Sink & operator << (Sink & sink, const Strings & s);
Sink & operator << (Sink & sink, const StringSet & s); Sink & operator << (Sink & sink, const StringSet & s);
MakeError(SerialisationError, Error)
template<typename T>
T readNum(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
uint64_t n =
((unsigned long long) buf[0]) |
((unsigned long long) buf[1] << 8) |
((unsigned long long) buf[2] << 16) |
((unsigned long long) buf[3] << 24) |
((unsigned long long) buf[4] << 32) |
((unsigned long long) buf[5] << 40) |
((unsigned long long) buf[6] << 48) |
((unsigned long long) buf[7] << 56);
if (n > std::numeric_limits<T>::max())
throw SerialisationError("serialised integer %d is too large for type %s", n, typeid(T).name());
return n;
}
inline unsigned int readInt(Source & source)
{
return readNum<unsigned int>(source);
}
inline uint64_t readLongLong(Source & source)
{
return readNum<uint64_t>(source);
}
void readPadding(size_t len, Source & source); void readPadding(size_t len, Source & source);
unsigned int readInt(Source & source);
unsigned long long readLongLong(Source & source);
size_t readString(unsigned char * buf, size_t max, Source & source); size_t readString(unsigned char * buf, size_t max, Source & source);
string readString(Source & source); string readString(Source & source);
template<class T> T readStrings(Source & source); template<class T> T readStrings(Source & source);
Source & operator >> (Source & in, string & s); Source & operator >> (Source & in, string & s);
Source & operator >> (Source & in, unsigned int & n);
template<typename T>
Source & operator >> (Source & in, T & n)
{
n = readNum<T>(in);
return in;
}
MakeError(SerialisationError, Error) template<typename T>
Source & operator >> (Source & in, bool & b)
{
b = readNum<uint64_t>(in);
return in;
}
} }

View file

@ -273,10 +273,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
} }
case wopAddToStore: { case wopAddToStore: {
string baseName = readString(from); bool fixed, recursive;
bool fixed = readInt(from) == 1; /* obsolete */ std::string s, baseName;
bool recursive = readInt(from) == 1; from >> baseName >> fixed /* obsolete */ >> recursive >> s;
string s = readString(from);
/* Compatibility hack. */ /* Compatibility hack. */
if (!fixed) { if (!fixed) {
s = "sha256"; s = "sha256";
@ -417,8 +416,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
GCOptions options; GCOptions options;
options.action = (GCOptions::GCAction) readInt(from); options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = readStorePaths<PathSet>(*store, from); options.pathsToDelete = readStorePaths<PathSet>(*store, from);
options.ignoreLiveness = readInt(from); from >> options.ignoreLiveness >> options.maxFreed;
options.maxFreed = readLongLong(from);
// obsolete fields // obsolete fields
readInt(from); readInt(from);
readInt(from); readInt(from);
@ -438,8 +436,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
} }
case wopSetOptions: { case wopSetOptions: {
settings.keepFailed = readInt(from) != 0; from >> settings.keepFailed;
settings.keepGoing = readInt(from) != 0; from >> settings.keepGoing;
settings.set("build-fallback", readInt(from) ? "true" : "false"); settings.set("build-fallback", readInt(from) ? "true" : "false");
verbosity = (Verbosity) readInt(from); verbosity = (Verbosity) readInt(from);
settings.set("build-max-jobs", std::to_string(readInt(from))); settings.set("build-max-jobs", std::to_string(readInt(from)));
@ -539,8 +537,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
break; break;
case wopVerifyStore: { case wopVerifyStore: {
bool checkContents = readInt(from) != 0; bool checkContents, repair;
bool repair = readInt(from) != 0; from >> checkContents >> repair;
startWork(); startWork();
if (repair && !trusted) if (repair && !trusted)
throw Error("you are not privileged to repair paths"); throw Error("you are not privileged to repair paths");
@ -573,19 +571,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
case wopAddToStoreNar: { case wopAddToStoreNar: {
ValidPathInfo info; ValidPathInfo info;
info.path = readStorePath(*store, from); info.path = readStorePath(*store, from);
info.deriver = readString(from); from >> info.deriver;
if (!info.deriver.empty()) if (!info.deriver.empty())
store->assertStorePath(info.deriver); store->assertStorePath(info.deriver);
info.narHash = parseHash(htSHA256, readString(from)); info.narHash = parseHash(htSHA256, readString(from));
info.references = readStorePaths<PathSet>(*store, from); info.references = readStorePaths<PathSet>(*store, from);
info.registrationTime = readInt(from); from >> info.registrationTime >> info.narSize >> info.ultimate;
info.narSize = readLongLong(from);
info.ultimate = readLongLong(from);
info.sigs = readStrings<StringSet>(from); info.sigs = readStrings<StringSet>(from);
info.ca = readString(from); from >> info.ca;
auto nar = make_ref<std::string>(readString(from)); auto nar = make_ref<std::string>(readString(from));
auto repair = readInt(from) ? true : false; bool repair, dontCheckSigs;
auto dontCheckSigs = readInt(from) ? true : false; from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs) if (!trusted && dontCheckSigs)
dontCheckSigs = false; dontCheckSigs = false;
startWork(); startWork();

View file

@ -839,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs)
settings.maxSilentTime = readInt(in); settings.maxSilentTime = readInt(in);
settings.buildTimeout = readInt(in); settings.buildTimeout = readInt(in);
if (GET_PROTOCOL_MINOR(clientVersion) >= 2) if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
settings.maxLogSize = readInt(in); in >> settings.maxLogSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
settings.set("build-repeat", std::to_string(readInt(in))); settings.set("build-repeat", std::to_string(readInt(in)));
settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false"); settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false");