diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6d64644d1..cdc9c09c7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -160,8 +160,7 @@ struct TunnelSink : Sink TunnelSink(Sink & to) : to(to) { } void operator () (std::string_view data) { - to << STDERR_WRITE; - writeString(data, to); + to << STDERR_WRITE << data; } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b2f8a285d..55a71f502 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -897,7 +897,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * if (!source) throw Error("no source"); size_t len = readNum(from); auto buf = std::make_unique(len); - writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); + to << std::string_view((const char *) buf.get(), source->read(buf.get(), len)); to.flush(); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 80b111f08..11bc183cc 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -330,55 +330,40 @@ void writePadding(size_t len, Sink & sink) } -void writeString(std::string_view data, Sink & sink) +WireFormatGenerator SerializingTransform::operator()(std::string_view s) { - sink << data.size(); - sink(data); - writePadding(data.size(), sink); + co_yield s.size(); + co_yield Bytes(s.begin(), s.size()); + co_yield SerializingTransform::padding(s.size()); } - -Sink & operator << (Sink & sink, std::string_view s) +WireFormatGenerator SerializingTransform::operator()(const Strings & ss) { - writeString(s, sink); - return sink; + co_yield ss.size(); + for (const auto & s : ss) + co_yield std::string_view(s); } - -template void writeStrings(const T & ss, Sink & sink) +WireFormatGenerator SerializingTransform::operator()(const StringSet & ss) { - sink << ss.size(); - for (auto & i : ss) - sink << i; + co_yield ss.size(); + for (const auto & s : ss) + co_yield std::string_view(s); } -Sink & operator << (Sink & sink, const Strings & s) -{ - writeStrings(s, sink); - return sink; -} - -Sink & operator << (Sink & sink, const StringSet & s) -{ - writeStrings(s, sink); - return sink; -} - -Sink & operator << (Sink & sink, const Error & ex) +WireFormatGenerator SerializingTransform::operator()(const Error & ex) { auto & info = ex.info(); - sink - << "Error" - << info.level - << "Error" // removed - << info.msg.str() - << 0 // FIXME: info.errPos - << info.traces.size(); + co_yield "Error"; + co_yield info.level; + co_yield "Error"; // removed + co_yield info.msg.str(); + co_yield 0; // FIXME: info.errPos + co_yield info.traces.size(); for (auto & trace : info.traces) { - sink << 0; // FIXME: trace.pos - sink << trace.hint.str(); + co_yield 0; // FIXME: trace.pos + co_yield trace.hint.str(); } - return sink; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 491b1987d..2651ec979 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -350,33 +350,82 @@ inline Sink & operator<<(Sink & sink, Generator && g) return sink; } -void writePadding(size_t len, Sink & sink); -void writeString(std::string_view s, Sink & sink); +struct SerializingTransform; +using WireFormatGenerator = Generator; -inline Sink & operator << (Sink & sink, uint64_t n) +struct SerializingTransform { - unsigned char buf[8]; - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - buf[4] = (n >> 32) & 0xff; - buf[5] = (n >> 40) & 0xff; - buf[6] = (n >> 48) & 0xff; - buf[7] = (unsigned char) (n >> 56) & 0xff; - sink({(char *) buf, sizeof(buf)}); - return sink; + std::array buf; + + Bytes operator()(uint64_t n) + { + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (unsigned char) (n >> 56) & 0xff; + return {reinterpret_cast(buf.begin()), 8}; + } + + static Bytes padding(size_t unpadded) + { + return Bytes("\0\0\0\0\0\0\0", unpadded % 8 ? 8 - unpadded % 8 : 0); + } + + // opt in to generator chaining. without this co_yielding + // another generator of any type will cause a type error. + auto operator()(Generator && g) + { + return std::move(g); + } + + // only choose this for *exactly* char spans, do not allow implicit + // conversions. this would cause ambiguities with strings literals, + // and resolving those with more string-like overloads needs a lot. + template + requires std::same_as> || std::same_as> + Bytes operator()(Span s) + { + return s; + } + WireFormatGenerator operator()(std::string_view s); + WireFormatGenerator operator()(const Strings & s); + WireFormatGenerator operator()(const StringSet & s); + WireFormatGenerator operator()(const Error & s); +}; + +void writePadding(size_t len, Sink & sink); + +inline Sink & operator<<(Sink & sink, uint64_t u) +{ + return sink << [&]() -> WireFormatGenerator { co_yield u; }(); } -Sink & operator << (Sink & in, const Error & ex); -Sink & operator << (Sink & sink, std::string_view s); -Sink & operator << (Sink & sink, const Strings & s); -Sink & operator << (Sink & sink, const StringSet & s); +inline Sink & operator<<(Sink & sink, std::string_view s) +{ + return sink << [&]() -> WireFormatGenerator { co_yield s; }(); +} +inline Sink & operator<<(Sink & sink, const Strings & s) +{ + return sink << [&]() -> WireFormatGenerator { co_yield s; }(); +} + +inline Sink & operator<<(Sink & sink, const StringSet & s) +{ + return sink << [&]() -> WireFormatGenerator { co_yield s; }(); +} + +inline Sink & operator<<(Sink & sink, const Error & ex) +{ + return sink << [&]() -> WireFormatGenerator { co_yield ex; }(); +} MakeError(SerialisationError, Error); - template T readNum(Source & source) { diff --git a/tests/unit/libutil/serialise.cc b/tests/unit/libutil/serialise.cc index 95ae43115..78882ad2c 100644 --- a/tests/unit/libutil/serialise.cc +++ b/tests/unit/libutil/serialise.cc @@ -2,30 +2,47 @@ #include "error.hh" #include "fmt.hh" #include "pos-table.hh" +#include "generator.hh" #include "ref.hh" #include "types.hh" +#include +#include +#include #include #include #include +#include +#include +#include namespace nix { -TEST(Sink, uint64_t) +// don't deduce the type of `val` for added insurance. +template +static std::string toWire(const std::type_identity_t & val) { - StringSink s; - s << 42; - ASSERT_EQ(s.s, std::string({42, 0, 0, 0, 0, 0, 0, 0})); + std::string result; + auto g = [] (const auto & val) -> WireFormatGenerator { co_yield val; }(val); + while (auto buffer = g.next()) { + result.append(buffer->data(), buffer->size()); + } + return result; } -TEST(Sink, string_view) +TEST(WireFormatGenerator, uint64_t) { - StringSink s; - s << ""; + auto s = toWire(42); + ASSERT_EQ(s, std::string({42, 0, 0, 0, 0, 0, 0, 0})); +} + +TEST(WireFormatGenerator, string_view) +{ + auto s = toWire(""); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 0, 0, 0, 0, 0, 0, 0, 0, @@ -34,11 +51,10 @@ TEST(Sink, string_view) ); // clang-format on - s = {}; - s << "test"; + s = toWire("test"); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 4, 0, 0, 0, 0, 0, 0, 0, @@ -50,11 +66,10 @@ TEST(Sink, string_view) ); // clang-format on - s = {}; - s << "longer string"; + s = toWire("longer string"); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 13, 0, 0, 0, 0, 0, 0, 0, @@ -67,13 +82,12 @@ TEST(Sink, string_view) // clang-format on } -TEST(Sink, StringSet) +TEST(WireFormatGenerator, StringSet) { - StringSink s; - s << StringSet{}; + auto s = toWire({}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 0, 0, 0, 0, 0, 0, 0, 0, @@ -82,11 +96,10 @@ TEST(Sink, StringSet) ); // clang-format on - s = {}; - s << StringSet{"a", ""}; + s = toWire({"a", ""}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 2, 0, 0, 0, 0, 0, 0, 0, @@ -99,13 +112,12 @@ TEST(Sink, StringSet) // clang-format on } -TEST(Sink, Strings) +TEST(WireFormatGenerator, Strings) { - StringSink s; - s << Strings{}; + auto s = toWire({}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 0, 0, 0, 0, 0, 0, 0, 0, @@ -114,11 +126,10 @@ TEST(Sink, Strings) ); // clang-format on - s = {}; - s << Strings{"a", ""}; + s = toWire({"a", ""}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 2, 0, 0, 0, 0, 0, 0, 0, @@ -131,23 +142,22 @@ TEST(Sink, Strings) // clang-format on } -TEST(Sink, Error) +TEST(WireFormatGenerator, Error) { PosTable pt; auto o = pt.addOrigin(Pos::String{make_ref("test")}, 4); - StringSink s; - s << Error{ErrorInfo{ + auto s = toWire(Error{ErrorInfo{ .level = lvlInfo, .msg = HintFmt("foo"), .pos = pt[pt.add(o, 1)], .traces = {{.pos = pt[pt.add(o, 2)], .hint = HintFmt("b %1%", "foo")}}, - }}; + }}); // NOTE position of the error and all traces are ignored // by the wire format // clang-format off ASSERT_EQ( - s.s, + s, std::string({ 5, 0, 0, 0, 0, 0, 0, 0, 'E', 'r', 'r', 'o', 'r', 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, @@ -163,4 +173,45 @@ TEST(Sink, Error) // clang-format on } +TEST(WireFormatGenerator, exampleMessage) +{ + auto gen = []() -> WireFormatGenerator { + std::set foo{"a", "longer string", ""}; + co_yield 42; + co_yield foo; + co_yield std::string_view("test"); + co_yield true; + }(); + + std::vector full; + while (auto s = gen.next()) { + full.insert(full.end(), s->begin(), s->end()); + } + + ASSERT_EQ( + full, + (std::vector{ + // clang-format off + // 42 + 42, 0, 0, 0, 0, 0, 0, 0, + // foo + 3, 0, 0, 0, 0, 0, 0, 0, + /// "" + 0, 0, 0, 0, 0, 0, 0, 0, + /// a + 1, 0, 0, 0, 0, 0, 0, 0, + 'a', 0, 0, 0, 0, 0, 0, 0, + /// longer string + 13, 0, 0, 0, 0, 0, 0, 0, + 'l', 'o', 'n', 'g', 'e', 'r', ' ', 's', 't', 'r', 'i', 'n', 'g', 0, 0, 0, + // foo done + // test + 4, 0, 0, 0, 0, 0, 0, 0, + 't', 'e', 's', 't', 0, 0, 0, 0, + // true + 1, 0, 0, 0, 0, 0, 0, 0, + //clang-format on + })); +} + }