lix-project/lix
20
55
Fork 31
Code Issues 343 Code Review (Gerrit) Projects 3 Releases Wiki Activity

Handle importing NARs containing files greater than 4 GiB

Browse source 
Also templatize readInt() to work for various integer types.
This commit is contained in:
Eelco Dolstra 2017-03-01 13:52:54 +01:00
parent 0780805246
commit c4a40949d9
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
7 changed files with 84 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/libstore/derivations.cc
View file

@ -397,8 +397,8 @@ PathSet BasicDerivation::outputPaths() const
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
{
drv.outputs.clear();
auto nr = readInt(in);
for (unsigned int n = 0; n < nr; n++) {
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto name = readString(in);
DerivationOutput o;
in >> o.path >> o.hashAlgo >> o.hash;
@ -410,8 +410,8 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv)
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
nr = readInt(in);
for (unsigned int n = 0; n < nr; n++) {
nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
auto key = readString(in);
auto value = readString(in);
drv.env[key] = value;

2
src/libstore/export-import.cc
View file

@ -86,7 +86,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
{
Paths res;
while (true) {
unsigned long long n = readLongLong(source);
auto n = readNum<uint64_t>(source);
if (n == 0) break;
if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");

24
src/libstore/remote-store.cc
View file

@ -108,7 +108,7 @@ void RemoteStore::initConnection(Connection & conn)
unsigned int magic = readInt(conn.from);
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
conn.daemonVersion = readInt(conn.from);
conn.from >> conn.daemonVersion;
if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported");
if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10)
@ -170,8 +170,7 @@ bool RemoteStore::isValidPathUncached(const Path & path)
auto conn(connections->get());
conn->to << wopIsValidPath << path;
conn->processStderr();
unsigned int reply = readInt(conn->from);
return reply != 0;
return readInt(conn->from);
}
@ -246,8 +245,8 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
conn->to << wopQuerySubstitutablePathInfos << paths;
conn->processStderr();
unsigned int count = readInt(conn->from);
for (unsigned int n = 0; n < count; n++) {
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
Path path = readStorePath(*this, conn->from);
SubstitutablePathInfo & info(infos[path]);
info.deriver = readString(conn->from);
@ -277,7 +276,7 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
throw;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) {
bool valid = readInt(conn->from) != 0;
bool valid; conn->from >> valid;
if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path);
}
auto info = std::make_shared<ValidPathInfo>();
@ -286,12 +285,11 @@ void RemoteStore::queryPathInfoUncached(const Path & path,
if (info->deriver != "") assertStorePath(info->deriver);
info->narHash = parseHash(htSHA256, readString(conn->from));
info->references = readStorePaths<PathSet>(*this, conn->from);
info->registrationTime = readInt(conn->from);
info->narSize = readLongLong(conn->from);
conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
info->ultimate = readInt(conn->from) != 0;
conn->from >> info->ultimate;
info->sigs = readStrings<StringSet>(conn->from);
info->ca = readString(conn->from);
conn->from >> info->ca;
}
return info;
});
@ -515,7 +513,7 @@ Roots RemoteStore::findRoots()
auto conn(connections->get());
conn->to << wopFindRoots;
conn->processStderr();
unsigned int count = readInt(conn->from);
size_t count = readNum<size_t>(conn->from);
Roots result;
while (count--) {
Path link = readString(conn->from);
@ -563,7 +561,7 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair)
auto conn(connections->get());
conn->to << wopVerifyStore << checkContents << repair;
conn->processStderr();
return readInt(conn->from) != 0;
return readInt(conn->from);
}
@ -599,7 +597,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
}
else if (msg == STDERR_READ) {
if (!source) throw Error("no source");
size_t len = readInt(from);
size_t len = readNum<size_t>(from);
auto buf = std::make_unique<unsigned char[]>(len);
writeString(buf.get(), source->read(buf.get(), len), to);
to.flush();

43
src/libutil/serialise.cc
View file

@ -194,39 +194,9 @@ void readPadding(size_t len, Source & source)
}
unsigned int readInt(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
if (buf[4] || buf[5] || buf[6] || buf[7])
throw SerialisationError("implementation cannot deal with > 32-bit integers");
return
buf[0] |
(buf[1] << 8) |
(buf[2] << 16) |
(buf[3] << 24);
}
unsigned long long readLongLong(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
return
((unsigned long long) buf[0]) |
((unsigned long long) buf[1] << 8) |
((unsigned long long) buf[2] << 16) |
((unsigned long long) buf[3] << 24) |
((unsigned long long) buf[4] << 32) |
((unsigned long long) buf[5] << 40) |
((unsigned long long) buf[6] << 48) |
((unsigned long long) buf[7] << 56);
}
size_t readString(unsigned char * buf, size_t max, Source & source)
{
size_t len = readInt(source);
auto len = readNum<size_t>(source);
if (len > max) throw Error("string is too long");
source(buf, len);
readPadding(len, source);
@ -236,7 +206,7 @@ size_t readString(unsigned char * buf, size_t max, Source & source)
string readString(Source & source)
{
size_t len = readInt(source);
auto len = readNum<size_t>(source);
auto buf = std::make_unique<unsigned char[]>(len);
source(buf.get(), len);
readPadding(len, source);
@ -250,16 +220,9 @@ Source & operator >> (Source & in, string & s)
}
Source & operator >> (Source & in, unsigned int & n)
{
n = readInt(in);
return in;
}
template<class T> T readStrings(Source & source)
{
unsigned int count = readInt(source);
auto count = readNum<size_t>(source);
T ss;
while (count--)
ss.insert(ss.end(), readString(source));

54
src/libutil/serialise.hh
View file

@ -177,18 +177,64 @@ Sink & operator << (Sink & sink, const Strings & s);
Sink & operator << (Sink & sink, const StringSet & s);
MakeError(SerialisationError, Error)
template<typename T>
T readNum(Source & source)
{
unsigned char buf[8];
source(buf, sizeof(buf));
uint64_t n =
((unsigned long long) buf[0]) |
((unsigned long long) buf[1] << 8) |
((unsigned long long) buf[2] << 16) |
((unsigned long long) buf[3] << 24) |
((unsigned long long) buf[4] << 32) |
((unsigned long long) buf[5] << 40) |
((unsigned long long) buf[6] << 48) |
((unsigned long long) buf[7] << 56);
if (n > std::numeric_limits<T>::max())
throw SerialisationError("serialised integer %d is too large for type ‘%s’", n, typeid(T).name());
return n;
}
inline unsigned int readInt(Source & source)
{
return readNum<unsigned int>(source);
}
inline uint64_t readLongLong(Source & source)
{
return readNum<uint64_t>(source);
}
void readPadding(size_t len, Source & source);
unsigned int readInt(Source & source);
unsigned long long readLongLong(Source & source);
size_t readString(unsigned char * buf, size_t max, Source & source);
string readString(Source & source);
template<class T> T readStrings(Source & source);
Source & operator >> (Source & in, string & s);
Source & operator >> (Source & in, unsigned int & n);
template<typename T>
Source & operator >> (Source & in, T & n)
{
n = readNum<T>(in);
return in;
}
MakeError(SerialisationError, Error)
template<typename T>
Source & operator >> (Source & in, bool & b)
{
b = readNum<uint64_t>(in);
return in;
}
}

30
src/nix-daemon/nix-daemon.cc
View file

@ -273,10 +273,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
}
case wopAddToStore: {
string baseName = readString(from);
bool fixed = readInt(from) == 1; /* obsolete */
bool recursive = readInt(from) == 1;
string s = readString(from);
bool fixed, recursive;
std::string s, baseName;
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
/* Compatibility hack. */
if (!fixed) {
s = "sha256";
@ -417,8 +416,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = readStorePaths<PathSet>(*store, from);
options.ignoreLiveness = readInt(from);
options.maxFreed = readLongLong(from);
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
readInt(from);
@ -438,8 +436,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
}
case wopSetOptions: {
settings.keepFailed = readInt(from) != 0;
settings.keepGoing = readInt(from) != 0;
from >> settings.keepFailed;
from >> settings.keepGoing;
settings.set("build-fallback", readInt(from) ? "true" : "false");
verbosity = (Verbosity) readInt(from);
settings.set("build-max-jobs", std::to_string(readInt(from)));
@ -539,8 +537,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
break;
case wopVerifyStore: {
bool checkContents = readInt(from) != 0;
bool repair = readInt(from) != 0;
bool checkContents, repair;
from >> checkContents >> repair;
startWork();
if (repair && !trusted)
throw Error("you are not privileged to repair paths");
@ -573,19 +571,17 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
case wopAddToStoreNar: {
ValidPathInfo info;
info.path = readStorePath(*store, from);
info.deriver = readString(from);
from >> info.deriver;
if (!info.deriver.empty())
store->assertStorePath(info.deriver);
info.narHash = parseHash(htSHA256, readString(from));
info.references = readStorePaths<PathSet>(*store, from);
info.registrationTime = readInt(from);
info.narSize = readLongLong(from);
info.ultimate = readLongLong(from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
info.ca = readString(from);
from >> info.ca;
auto nar = make_ref<std::string>(readString(from));
auto repair = readInt(from) ? true : false;
auto dontCheckSigs = readInt(from) ? true : false;
bool repair, dontCheckSigs;
from >> repair >> dontCheckSigs;
if (!trusted && dontCheckSigs)
dontCheckSigs = false;
startWork();

2
src/nix-store/nix-store.cc
View file

@ -839,7 +839,7 @@ static void opServe(Strings opFlags, Strings opArgs)
settings.maxSilentTime = readInt(in);
settings.buildTimeout = readInt(in);
if (GET_PROTOCOL_MINOR(clientVersion) >= 2)
settings.maxLogSize = readInt(in);
in >> settings.maxLogSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 3) {
settings.set("build-repeat", std::to_string(readInt(in)));
settings.set("enforce-determinism", readInt(in) != 0 ? "true" : "false");