From 05c04089ee5fb1f0ab20f65841efea8a0a044160 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Tue, 19 Mar 2024 22:22:18 +0100 Subject: [PATCH] libutil: prototype serialization mechanism on coroutines Change-Id: I361d89ff556354f6930d9204f55117565f2f7f20 --- src/libstore/daemon.cc | 3 +- src/libstore/remote-store.cc | 2 +- src/libutil/serialise.cc | 60 +++++++---------- src/libutil/serialise.hh | 98 +++++++++++++++++++++------ tests/unit/libutil/serialise.cc | 114 +++++++++++++++++++++++--------- 5 files changed, 186 insertions(+), 91 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ce0f39367..49b0d5e95 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -162,8 +162,7 @@ struct TunnelSink : Sink TunnelSink(Sink & to) : to(to) { } void operator () (std::string_view data) { - to << STDERR_WRITE; - writeString(data, to); + to << STDERR_WRITE << data; } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 12c8bbf00..dd95f00e1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1001,7 +1001,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * if (!source) throw Error("no source"); size_t len = readNum(from); auto buf = std::make_unique(len); - writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); + to << std::string_view((const char *) buf.get(), source->read(buf.get(), len)); to.flush(); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 222e88df8..77d433ffb 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -315,55 +315,43 @@ void writePadding(size_t len, Sink & sink) } -void writeString(std::string_view data, Sink & sink) +WireFormatGenerator SerializingTransform::operator()(std::string_view s) { - sink << data.size(); - sink(data); - writePadding(data.size(), sink); + co_yield s.size(); + co_yield raw(s.begin(), s.size()); + if (s.size() % 8) { + std::array pad{}; + co_yield raw(pad.begin(), 8 - s.size() % 8); + } } - -Sink & operator << (Sink & sink, std::string_view s) +WireFormatGenerator SerializingTransform::operator()(const Strings & ss) { - writeString(s, sink); - return sink; + co_yield ss.size(); + for (const auto & s : ss) + co_yield std::string_view(s); } - -template void writeStrings(const T & ss, Sink & sink) +WireFormatGenerator SerializingTransform::operator()(const StringSet & ss) { - sink << ss.size(); - for (auto & i : ss) - sink << i; + co_yield ss.size(); + for (const auto & s : ss) + co_yield std::string_view(s); } -Sink & operator << (Sink & sink, const Strings & s) -{ - writeStrings(s, sink); - return sink; -} - -Sink & operator << (Sink & sink, const StringSet & s) -{ - writeStrings(s, sink); - return sink; -} - -Sink & operator << (Sink & sink, const Error & ex) +WireFormatGenerator SerializingTransform::operator()(const Error & ex) { auto & info = ex.info(); - sink - << "Error" - << info.level - << "Error" // removed - << info.msg.str() - << 0 // FIXME: info.errPos - << info.traces.size(); + co_yield "Error"; + co_yield info.level; + co_yield "Error"; // removed + co_yield info.msg.str(); + co_yield 0; // FIXME: info.errPos + co_yield info.traces.size(); for (auto & trace : info.traces) { - sink << 0; // FIXME: trace.pos - sink << trace.hint.str(); + co_yield 0; // FIXME: trace.pos + co_yield trace.hint.str(); } - return sink; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 9b831fe56..89fff7a04 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,8 +1,10 @@ #pragma once ///@file +#include #include +#include "generator.hh" #include "types.hh" #include "util.hh" @@ -372,34 +374,92 @@ std::unique_ptr sinkToSource( throw EndOfFile("coroutine has finished"); }); +struct SerializingTransform; +using WireFormatGenerator = Generator, SerializingTransform>; -void writePadding(size_t len, Sink & sink); -void writeString(std::string_view s, Sink & sink); - -inline Sink & operator << (Sink & sink, uint64_t n) +inline void drainGenerator(WireFormatGenerator g, std::derived_from auto & into) { - unsigned char buf[8]; - buf[0] = n & 0xff; - buf[1] = (n >> 8) & 0xff; - buf[2] = (n >> 16) & 0xff; - buf[3] = (n >> 24) & 0xff; - buf[4] = (n >> 32) & 0xff; - buf[5] = (n >> 40) & 0xff; - buf[6] = (n >> 48) & 0xff; - buf[7] = (unsigned char) (n >> 56) & 0xff; - sink({(char *) buf, sizeof(buf)}); + while (g) { + auto bit = g(); + into(std::string_view(bit.data(), bit.size())); + } +} + +struct SerializingTransform +{ + std::array buf; + + static std::span raw(auto... args) + { + return std::span(args...); + } + + std::span operator()(uint64_t n) + { + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (unsigned char) (n >> 56) & 0xff; + return {buf.begin(), 8}; + } + + // only choose this for *exactly* char spans, do not allow implicit + // conversions. this would cause ambiguities with strings literals, + // and resolving those with more string-like overloads needs a lot. + template + requires std::same_as> || std::same_as> + std::span operator()(Span s) + { + return s; + } + WireFormatGenerator operator()(std::string_view s); + WireFormatGenerator operator()(const Strings & s); + WireFormatGenerator operator()(const StringSet & s); + WireFormatGenerator operator()(const Error & s); +}; + +inline Sink & operator<<(Sink & sink, WireFormatGenerator && g) +{ + while (g) { + auto bit = g(); + sink(std::string_view(bit.data(), bit.size())); + } return sink; } -Sink & operator << (Sink & in, const Error & ex); -Sink & operator << (Sink & sink, std::string_view s); -Sink & operator << (Sink & sink, const Strings & s); -Sink & operator << (Sink & sink, const StringSet & s); +void writePadding(size_t len, Sink & sink); +inline Sink & operator<<(Sink & sink, uint64_t u) +{ + return sink << [&]() -> WireFormatGenerator { co_yield u; }(); +} + +inline Sink & operator<<(Sink & sink, std::string_view s) +{ + return sink << [&]() -> WireFormatGenerator { co_yield s; }(); +} + +inline Sink & operator<<(Sink & sink, const Strings & s) +{ + return sink << [&]() -> WireFormatGenerator { co_yield s; }(); +} + +inline Sink & operator<<(Sink & sink, const StringSet & s) +{ + return sink << [&]() -> WireFormatGenerator { co_yield s; }(); +} + +inline Sink & operator<<(Sink & sink, const Error & ex) +{ + return sink << [&]() -> WireFormatGenerator { co_yield ex; }(); +} MakeError(SerialisationError, Error); - template T readNum(Source & source) { diff --git a/tests/unit/libutil/serialise.cc b/tests/unit/libutil/serialise.cc index b3bbdd33e..e2a5b234a 100644 --- a/tests/unit/libutil/serialise.cc +++ b/tests/unit/libutil/serialise.cc @@ -1,15 +1,19 @@ #include "serialise.hh" #include "error.hh" #include "fmt.hh" +#include "generator.hh" #include "libexpr/pos-table.hh" #include "ref.hh" #include "types.hh" #include "util.hh" +#include +#include #include #include #include +#include namespace nix { @@ -58,20 +62,29 @@ TEST(ChainSource, move) ASSERT_EQ(buf, "33"); } -TEST(Sink, uint64_t) +static std::string simpleToWire(const auto & val) { - StringSink s; - s << 42; - ASSERT_EQ(s.s, std::string({42, 0, 0, 0, 0, 0, 0, 0})); + std::string result; + auto g = [&] () -> WireFormatGenerator { co_yield val; }(); + while (g) { + auto bit = g(); + result.append(bit.data(), bit.size()); + } + return result; } -TEST(Sink, string_view) +TEST(WireFormatGenerator, uint64_t) { - StringSink s; - s << ""; + auto s = simpleToWire(42); + ASSERT_EQ(s, std::string({42, 0, 0, 0, 0, 0, 0, 0})); +} + +TEST(WireFormatGenerator, string_view) +{ + auto s = simpleToWire(""); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 0, 0, 0, 0, 0, 0, 0, 0, @@ -80,11 +93,10 @@ TEST(Sink, string_view) ); // clang-format on - s = {}; - s << "test"; + s = simpleToWire("test"); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 4, 0, 0, 0, 0, 0, 0, 0, @@ -96,11 +108,10 @@ TEST(Sink, string_view) ); // clang-format on - s = {}; - s << "longer string"; + s = simpleToWire("longer string"); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 13, 0, 0, 0, 0, 0, 0, 0, @@ -113,13 +124,12 @@ TEST(Sink, string_view) // clang-format on } -TEST(Sink, StringSet) +TEST(WireFormatGenerator, StringSet) { - StringSink s; - s << StringSet{}; + auto s = simpleToWire(StringSet{}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 0, 0, 0, 0, 0, 0, 0, 0, @@ -128,11 +138,10 @@ TEST(Sink, StringSet) ); // clang-format on - s = {}; - s << StringSet{"a", ""}; + s = simpleToWire(StringSet{"a", ""}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 2, 0, 0, 0, 0, 0, 0, 0, @@ -145,13 +154,12 @@ TEST(Sink, StringSet) // clang-format on } -TEST(Sink, Strings) +TEST(WireFormatGenerator, Strings) { - StringSink s; - s << Strings{}; + auto s = simpleToWire(Strings{}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 0, 0, 0, 0, 0, 0, 0, 0, @@ -160,11 +168,10 @@ TEST(Sink, Strings) ); // clang-format on - s = {}; - s << Strings{"a", ""}; + s = simpleToWire(Strings{"a", ""}); // clang-format off ASSERT_EQ( - s.s, + s, std::string({ // length 2, 0, 0, 0, 0, 0, 0, 0, @@ -177,23 +184,22 @@ TEST(Sink, Strings) // clang-format on } -TEST(Sink, Error) +TEST(WireFormatGenerator, Error) { PosTable pt; auto o = pt.addOrigin(Pos::String{make_ref("test")}, 4); - StringSink s; - s << Error{{ + auto s = simpleToWire(Error{{ .level = lvlInfo, .msg = HintFmt("foo"), .pos = pt[pt.add(o, 1)], .traces = {{.pos = pt[pt.add(o, 2)], .hint = HintFmt("b %1%", "foo")}}, - }}; + }}); // NOTE position of the error and all traces are ignored // by the wire format // clang-format off ASSERT_EQ( - s.s, + s, std::string({ 5, 0, 0, 0, 0, 0, 0, 0, 'E', 'r', 'r', 'o', 'r', 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, @@ -209,4 +215,46 @@ TEST(Sink, Error) // clang-format on } +TEST(FullFormatter, foo) +{ + auto gen = []() -> Generator, SerializingTransform> { + std::set foo{"a", "longer string", ""}; + co_yield 42; + co_yield foo; + co_yield std::string_view("test"); + co_yield 7; + }(); + + std::vector full; + while (gen) { + auto s = gen(); + full.insert(full.end(), s.begin(), s.end()); + } + + ASSERT_EQ( + full, + (std::vector{ + // clang-format off + // 32 + 42, 0, 0, 0, 0, 0, 0, 0, + // foo + 3, 0, 0, 0, 0, 0, 0, 0, + /// "" + 0, 0, 0, 0, 0, 0, 0, 0, + /// a + 1, 0, 0, 0, 0, 0, 0, 0, + 'a', 0, 0, 0, 0, 0, 0, 0, + /// longer string + 13, 0, 0, 0, 0, 0, 0, 0, + 'l', 'o', 'n', 'g', 'e', 'r', ' ', 's', 't', 'r', 'i', 'n', 'g', 0, 0, 0, + // foo done + // test + 4, 0, 0, 0, 0, 0, 0, 0, + 't', 'e', 's', 't', 0, 0, 0, 0, + // 7 + 7, 0, 0, 0, 0, 0, 0, 0, + //clang-format on + })); +} + }