lix/src/libutil/serialise.cc

475 lines
10 KiB
C++
Raw Normal View History

#include "serialise.hh"
#include "util.hh"
#include <cstring>
#include <cerrno>
#include <memory>
#include <boost/coroutine2/coroutine.hpp>
namespace nix {
2020-12-02 13:00:43 +00:00
void BufferedSink::operator () (std::string_view data)
{
2020-12-02 13:00:43 +00:00
if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
2015-07-19 23:16:16 +00:00
2020-12-02 13:00:43 +00:00
while (!data.empty()) {
/* Optimisation: bypass the buffer if the data exceeds the
buffer size. */
2020-12-02 13:00:43 +00:00
if (bufPos + data.size() >= bufSize) {
flush();
2020-12-02 13:00:43 +00:00
write(data);
break;
}
/* Otherwise, copy the bytes to the buffer. Flush the buffer
when it's full. */
2020-12-02 13:00:43 +00:00
size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size();
memcpy(buffer.get() + bufPos, data.data(), n);
data.remove_prefix(n); bufPos += n;
if (bufPos == bufSize) flush();
}
}
void BufferedSink::flush()
{
if (bufPos == 0) return;
2011-12-16 15:45:42 +00:00
size_t n = bufPos;
bufPos = 0; // don't trigger the assert() in ~BufferedSink()
2020-12-02 13:00:43 +00:00
write({buffer.get(), n});
2011-12-16 15:45:42 +00:00
}
FdSink::~FdSink()
{
try { flush(); } catch (...) { ignoreException(); }
}
size_t threshold = 256 * 1024 * 1024;
static void warnLargeDump()
{
Improve error formatting Changes: * The divider lines are gone. These were in practice a bit confusing, in particular with --show-trace or --keep-going, since then there were multiple lines, suggesting a start/end which wasn't the case. * Instead, multi-line error messages are now indented to align with the prefix (e.g. "error: "). * The 'description' field is gone since we weren't really using it. * 'hint' is renamed to 'msg' since it really wasn't a hint. * The error is now printed *before* the location info. * The 'name' field is no longer printed since most of the time it wasn't very useful since it was just the name of the exception (like EvalError). Ideally in the future this would be a unique, easily googleable error ID (like rustc). * "trace:" is now just "…". This assumes error contexts start with something like "while doing X". Example before: error: --- AssertionError ---------------------------------------------------------------------------------------- nix at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| assertion 'false' failed ----------------------------------------------------- show-trace ----------------------------------------------------- trace: while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { Example after: error: assertion 'false' failed at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| … while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) {
2021-01-20 23:27:36 +00:00
warn("dumping very large path (> 256 MiB); this may run out of memory");
}
2020-12-02 13:00:43 +00:00
void FdSink::write(std::string_view data)
{
2020-12-02 13:00:43 +00:00
written += data.size();
static bool warned = false;
if (warn && !warned) {
if (written > threshold) {
warnLargeDump();
warned = true;
}
}
try {
2020-12-02 13:00:43 +00:00
writeFull(fd, data);
} catch (SysError & e) {
_good = false;
throw;
}
}
bool FdSink::good()
{
return _good;
}
void Source::operator () (char * data, size_t len)
{
while (len) {
size_t n = read(data, len);
data += n; len -= n;
}
}
2020-09-13 12:39:11 +00:00
void Source::drainInto(Sink & sink)
{
std::string s;
std::vector<char> buf(8192);
while (true) {
size_t n;
try {
n = read(buf.data(), buf.size());
sink({buf.data(), n});
} catch (EndOfFile &) {
break;
}
}
2020-09-13 12:39:11 +00:00
}
std::string Source::drain()
{
StringSink s;
drainInto(s);
return *s.s;
}
size_t BufferedSource::read(char * data, size_t len)
{
if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
2016-07-13 10:03:37 +00:00
if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize);
2015-07-19 23:16:16 +00:00
/* Copy out the data in the buffer. */
size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
2016-07-13 10:03:37 +00:00
memcpy(data, buffer.get() + bufPosOut, n);
bufPosOut += n;
if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0;
return n;
}
bool BufferedSource::hasData()
{
return bufPosOut < bufPosIn;
}
size_t FdSource::readUnbuffered(char * data, size_t len)
{
ssize_t n;
do {
checkInterrupt();
n = ::read(fd, data, len);
} while (n == -1 && errno == EINTR);
if (n == -1) { _good = false; throw SysError("reading from file"); }
if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
2016-02-26 15:16:08 +00:00
read += n;
return n;
}
bool FdSource::good()
{
return _good;
}
size_t StringSource::read(char * data, size_t len)
{
if (pos == s.size()) throw EndOfFile("end of string reached");
size_t n = s.copy(data, len, pos);
pos += n;
return n;
}
#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600
#error Coroutines are broken in this version of Boost!
#endif
/* A concrete datatype allow virtual dispatch of stack allocation methods. */
struct VirtualStackAllocator {
StackAllocator *allocator = StackAllocator::defaultAllocator;
boost::context::stack_context allocate() {
return allocator->allocate();
}
void deallocate(boost::context::stack_context sctx) {
allocator->deallocate(sctx);
}
};
/* This class reifies the default boost coroutine stack allocation strategy with
a virtual interface. */
class DefaultStackAllocator : public StackAllocator {
boost::coroutines2::default_stack stack;
boost::context::stack_context allocate() {
return stack.allocate();
}
void deallocate(boost::context::stack_context sctx) {
2020-11-10 03:24:55 +00:00
stack.deallocate(sctx);
}
};
static DefaultStackAllocator defaultAllocatorSingleton;
StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
2019-12-10 08:47:38 +00:00
std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
{
struct SourceToSink : FinishSink
{
typedef boost::coroutines2::coroutine<bool> coro_t;
std::function<void(Source &)> fun;
std::optional<coro_t::push_type> coro;
SourceToSink(std::function<void(Source &)> fun) : fun(fun)
{
}
std::string_view cur;
void operator () (std::string_view in) override
{
if (in.empty()) return;
cur = in;
if (!coro)
coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
LambdaSource source([&](char *out, size_t out_len) {
if (cur.empty()) {
yield();
if (yield.get()) {
return (size_t)0;
}
}
size_t n = std::min(cur.size(), out_len);
memcpy(out, cur.data(), n);
cur.remove_prefix(n);
return n;
});
fun(source);
});
if (!*coro) { abort(); }
if (!cur.empty()) (*coro)(false);
}
2021-09-14 06:15:33 +00:00
void finish() override
{
2019-12-10 08:47:38 +00:00
if (!coro) return;
if (!*coro) abort();
(*coro)(true);
if (*coro) abort();
}
};
return std::make_unique<SourceToSink>(fun);
}
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof)
{
struct SinkToSource : Source
{
typedef boost::coroutines2::coroutine<std::string> coro_t;
std::function<void(Sink &)> fun;
std::function<void()> eof;
2019-02-12 12:43:32 +00:00
std::optional<coro_t::pull_type> coro;
SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
: fun(fun), eof(eof)
{
}
std::string cur;
size_t pos = 0;
size_t read(char * data, size_t len) override
{
if (!coro)
coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
2020-12-02 13:00:43 +00:00
LambdaSink sink([&](std::string_view data) {
if (!data.empty()) yield(std::string(data));
});
fun(sink);
});
if (!*coro) { eof(); abort(); }
if (pos == cur.size()) {
if (!cur.empty()) (*coro)();
cur = coro->get();
pos = 0;
}
auto n = std::min(cur.size() - pos, len);
memcpy(data, cur.data() + pos, n);
pos += n;
return n;
}
};
return std::make_unique<SinkToSource>(fun, eof);
}
void writePadding(size_t len, Sink & sink)
{
if (len % 8) {
char zero[8];
memset(zero, 0, sizeof(zero));
sink({zero, 8 - (len % 8)});
}
}
2020-12-02 13:00:43 +00:00
void writeString(std::string_view data, Sink & sink)
{
2020-12-02 13:00:43 +00:00
sink << data.size();
sink(data);
writePadding(data.size(), sink);
}
2015-07-19 23:16:16 +00:00
Sink & operator << (Sink & sink, const string & s)
{
2020-12-02 13:00:43 +00:00
writeString(s, sink);
2015-07-19 23:16:16 +00:00
return sink;
}
template<class T> void writeStrings(const T & ss, Sink & sink)
{
2015-07-19 23:16:16 +00:00
sink << ss.size();
2015-07-17 17:24:28 +00:00
for (auto & i : ss)
2015-07-19 23:16:16 +00:00
sink << i;
}
2015-07-19 23:16:16 +00:00
Sink & operator << (Sink & sink, const Strings & s)
{
2015-07-19 23:16:16 +00:00
writeStrings(s, sink);
return sink;
}
2015-07-19 23:16:16 +00:00
Sink & operator << (Sink & sink, const StringSet & s)
{
2015-07-19 23:16:16 +00:00
writeStrings(s, sink);
return sink;
}
Sink & operator << (Sink & sink, const Error & ex)
{
auto info = ex.info();
sink
<< "Error"
<< info.level
<< info.name
Improve error formatting Changes: * The divider lines are gone. These were in practice a bit confusing, in particular with --show-trace or --keep-going, since then there were multiple lines, suggesting a start/end which wasn't the case. * Instead, multi-line error messages are now indented to align with the prefix (e.g. "error: "). * The 'description' field is gone since we weren't really using it. * 'hint' is renamed to 'msg' since it really wasn't a hint. * The error is now printed *before* the location info. * The 'name' field is no longer printed since most of the time it wasn't very useful since it was just the name of the exception (like EvalError). Ideally in the future this would be a unique, easily googleable error ID (like rustc). * "trace:" is now just "…". This assumes error contexts start with something like "while doing X". Example before: error: --- AssertionError ---------------------------------------------------------------------------------------- nix at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| assertion 'false' failed ----------------------------------------------------- show-trace ----------------------------------------------------- trace: while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { Example after: error: assertion 'false' failed at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| … while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) {
2021-01-20 23:27:36 +00:00
<< info.msg.str()
<< 0 // FIXME: info.errPos
<< info.traces.size();
for (auto & trace : info.traces) {
sink << 0; // FIXME: trace.pos
sink << trace.hint.str();
}
return sink;
}
void readPadding(size_t len, Source & source)
{
if (len % 8) {
char zero[8];
size_t n = 8 - (len % 8);
source(zero, n);
for (unsigned int i = 0; i < n; i++)
if (zero[i]) throw SerialisationError("non-zero padding");
}
}
size_t readString(char * buf, size_t max, Source & source)
{
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
source(buf, len);
readPadding(len, source);
return len;
}
2015-07-19 23:16:16 +00:00
string readString(Source & source, size_t max)
{
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
std::string res(len, 0);
source(res.data(), len);
readPadding(len, source);
return res;
}
Source & operator >> (Source & in, string & s)
{
s = readString(in);
return in;
}
template<class T> T readStrings(Source & source)
{
auto count = readNum<size_t>(source);
T ss;
while (count--)
ss.insert(ss.end(), readString(source));
return ss;
}
template Paths readStrings(Source & source);
template PathSet readStrings(Source & source);
Error readError(Source & source)
{
auto type = readString(source);
assert(type == "Error");
Improve error formatting Changes: * The divider lines are gone. These were in practice a bit confusing, in particular with --show-trace or --keep-going, since then there were multiple lines, suggesting a start/end which wasn't the case. * Instead, multi-line error messages are now indented to align with the prefix (e.g. "error: "). * The 'description' field is gone since we weren't really using it. * 'hint' is renamed to 'msg' since it really wasn't a hint. * The error is now printed *before* the location info. * The 'name' field is no longer printed since most of the time it wasn't very useful since it was just the name of the exception (like EvalError). Ideally in the future this would be a unique, easily googleable error ID (like rustc). * "trace:" is now just "…". This assumes error contexts start with something like "while doing X". Example before: error: --- AssertionError ---------------------------------------------------------------------------------------- nix at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| assertion 'false' failed ----------------------------------------------------- show-trace ----------------------------------------------------- trace: while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { Example after: error: assertion 'false' failed at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| … while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) {
2021-01-20 23:27:36 +00:00
auto level = (Verbosity) readInt(source);
auto name = readString(source);
auto msg = readString(source);
ErrorInfo info {
.level = level,
.name = name,
.msg = hintformat(std::move(format("%s") % msg)),
};
auto havePos = readNum<size_t>(source);
assert(havePos == 0);
auto nrTraces = readNum<size_t>(source);
for (size_t i = 0; i < nrTraces; ++i) {
havePos = readNum<size_t>(source);
assert(havePos == 0);
info.traces.push_back(Trace {
.hint = hintformat(std::move(format("%s") % readString(source)))
});
}
return Error(std::move(info));
}
2020-12-02 13:00:43 +00:00
void StringSink::operator () (std::string_view data)
{
static bool warned = false;
2016-03-04 15:49:56 +00:00
if (!warned && s->size() > threshold) {
warnLargeDump();
warned = true;
}
2020-12-02 13:00:43 +00:00
s->append(data);
}
size_t ChainSource::read(char * data, size_t len)
{
if (useSecond) {
return source2.read(data, len);
} else {
try {
return source1.read(data, len);
} catch (EndOfFile &) {
useSecond = true;
return this->read(data, len);
}
}
}
}