lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

libutil: prototype serialization mechanism on coroutines

Browse source 
Change-Id: I361d89ff556354f6930d9204f55117565f2f7f20
This commit is contained in:
eldritch horrors 2024-03-19 22:22:18 +01:00
parent 1aa630bdf5
commit 05c04089ee
5 changed files with 186 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/libstore/daemon.cc
View file

@ -162,8 +162,7 @@ struct TunnelSink : Sink
TunnelSink(Sink & to) : to(to) { }
void operator () (std::string_view data)
{
to << STDERR_WRITE;
writeString(data, to);
to << STDERR_WRITE << data;
}
};

2
src/libstore/remote-store.cc
View file

@ -1001,7 +1001,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
if (!source) throw Error("no source");
size_t len = readNum<size_t>(from);
auto buf = std::make_unique<char[]>(len);
writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
to << std::string_view((const char *) buf.get(), source->read(buf.get(), len));
to.flush();
}

60
src/libutil/serialise.cc
View file

@ -315,55 +315,43 @@ void writePadding(size_t len, Sink & sink)
}
void writeString(std::string_view data, Sink & sink)
WireFormatGenerator SerializingTransform::operator()(std::string_view s)
{
sink << data.size();
sink(data);
writePadding(data.size(), sink);
co_yield s.size();
co_yield raw(s.begin(), s.size());
if (s.size() % 8) {
std::array<char, 8> pad{};
co_yield raw(pad.begin(), 8 - s.size() % 8);
}
}
Sink & operator << (Sink & sink, std::string_view s)
WireFormatGenerator SerializingTransform::operator()(const Strings & ss)
{
writeString(s, sink);
return sink;
co_yield ss.size();
for (const auto & s : ss)
co_yield std::string_view(s);
}
template<class T> void writeStrings(const T & ss, Sink & sink)
WireFormatGenerator SerializingTransform::operator()(const StringSet & ss)
{
sink << ss.size();
for (auto & i : ss)
sink << i;
co_yield ss.size();
for (const auto & s : ss)
co_yield std::string_view(s);
}
Sink & operator << (Sink & sink, const Strings & s)
{
writeStrings(s, sink);
return sink;
}
Sink & operator << (Sink & sink, const StringSet & s)
{
writeStrings(s, sink);
return sink;
}
Sink & operator << (Sink & sink, const Error & ex)
WireFormatGenerator SerializingTransform::operator()(const Error & ex)
{
auto & info = ex.info();
sink
<< "Error"
<< info.level
<< "Error" // removed
<< info.msg.str()
<< 0 // FIXME: info.errPos
<< info.traces.size();
co_yield "Error";
co_yield info.level;
co_yield "Error"; // removed
co_yield info.msg.str();
co_yield 0; // FIXME: info.errPos
co_yield info.traces.size();
for (auto & trace : info.traces) {
sink << 0; // FIXME: trace.pos
sink << trace.hint.str();
co_yield 0; // FIXME: trace.pos
co_yield trace.hint.str();
}
return sink;
}

98
src/libutil/serialise.hh
View file

@ -1,8 +1,10 @@
#pragma once
///@file
#include <concepts>
#include <memory>
#include "generator.hh"
#include "types.hh"
#include "util.hh"
@ -372,34 +374,92 @@ std::unique_ptr<Source> sinkToSource(
throw EndOfFile("coroutine has finished");
});
struct SerializingTransform;
using WireFormatGenerator = Generator<std::span<const char>, SerializingTransform>;
void writePadding(size_t len, Sink & sink);
void writeString(std::string_view s, Sink & sink);
inline Sink & operator << (Sink & sink, uint64_t n)
inline void drainGenerator(WireFormatGenerator g, std::derived_from<Sink> auto & into)
{
unsigned char buf[8];
buf[0] = n & 0xff;
buf[1] = (n >> 8) & 0xff;
buf[2] = (n >> 16) & 0xff;
buf[3] = (n >> 24) & 0xff;
buf[4] = (n >> 32) & 0xff;
buf[5] = (n >> 40) & 0xff;
buf[6] = (n >> 48) & 0xff;
buf[7] = (unsigned char) (n >> 56) & 0xff;
sink({(char *) buf, sizeof(buf)});
while (g) {
auto bit = g();
into(std::string_view(bit.data(), bit.size()));
}
}
struct SerializingTransform
{
std::array<char, 8> buf;
static std::span<const char> raw(auto... args)
{
return std::span<const char>(args...);
}
std::span<const char> operator()(uint64_t n)
{
buf[0] = n & 0xff;
buf[1] = (n >> 8) & 0xff;
buf[2] = (n >> 16) & 0xff;
buf[3] = (n >> 24) & 0xff;
buf[4] = (n >> 32) & 0xff;
buf[5] = (n >> 40) & 0xff;
buf[6] = (n >> 48) & 0xff;
buf[7] = (unsigned char) (n >> 56) & 0xff;
return {buf.begin(), 8};
}
// only choose this for *exactly* char spans, do not allow implicit
// conversions. this would cause ambiguities with strings literals,
// and resolving those with more string-like overloads needs a lot.
template<typename Span>
requires std::same_as<Span, std::span<char>> || std::same_as<Span, std::span<const char>>
std::span<const char> operator()(Span s)
{
return s;
}
WireFormatGenerator operator()(std::string_view s);
WireFormatGenerator operator()(const Strings & s);
WireFormatGenerator operator()(const StringSet & s);
WireFormatGenerator operator()(const Error & s);
};
inline Sink & operator<<(Sink & sink, WireFormatGenerator && g)
{
while (g) {
auto bit = g();
sink(std::string_view(bit.data(), bit.size()));
}
return sink;
}
Sink & operator << (Sink & in, const Error & ex);
Sink & operator << (Sink & sink, std::string_view s);
Sink & operator << (Sink & sink, const Strings & s);
Sink & operator << (Sink & sink, const StringSet & s);
void writePadding(size_t len, Sink & sink);
inline Sink & operator<<(Sink & sink, uint64_t u)
{
return sink << [&]() -> WireFormatGenerator { co_yield u; }();
}
inline Sink & operator<<(Sink & sink, std::string_view s)
{
return sink << [&]() -> WireFormatGenerator { co_yield s; }();
}
inline Sink & operator<<(Sink & sink, const Strings & s)
{
return sink << [&]() -> WireFormatGenerator { co_yield s; }();
}
inline Sink & operator<<(Sink & sink, const StringSet & s)
{
return sink << [&]() -> WireFormatGenerator { co_yield s; }();
}
inline Sink & operator<<(Sink & sink, const Error & ex)
{
return sink << [&]() -> WireFormatGenerator { co_yield ex; }();
}
MakeError(SerialisationError, Error);
template<typename T>
T readNum(Source & source)
{

114
tests/unit/libutil/serialise.cc
View file

@ -1,15 +1,19 @@
#include "serialise.hh"
#include "error.hh"
#include "fmt.hh"
#include "generator.hh"
#include "libexpr/pos-table.hh"
#include "ref.hh"
#include "types.hh"
#include "util.hh"
#include <concepts>
#include <initializer_list>
#include <limits.h>
#include <gtest/gtest.h>
#include <numeric>
#include <stdexcept>
namespace nix {
@ -58,20 +62,29 @@ TEST(ChainSource, move)
ASSERT_EQ(buf, "33");
}
TEST(Sink, uint64_t)
static std::string simpleToWire(const auto & val)
{
StringSink s;
s << 42;
ASSERT_EQ(s.s, std::string({42, 0, 0, 0, 0, 0, 0, 0}));
std::string result;
auto g = [&] () -> WireFormatGenerator { co_yield val; }();
while (g) {
auto bit = g();
result.append(bit.data(), bit.size());
}
return result;
}
TEST(Sink, string_view)
TEST(WireFormatGenerator, uint64_t)
{
StringSink s;
s << "";
auto s = simpleToWire(42);
ASSERT_EQ(s, std::string({42, 0, 0, 0, 0, 0, 0, 0}));
}
TEST(WireFormatGenerator, string_view)
{
auto s = simpleToWire("");
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
0, 0, 0, 0, 0, 0, 0, 0,
@ -80,11 +93,10 @@ TEST(Sink, string_view)
);
// clang-format on
s = {};
s << "test";
s = simpleToWire("test");
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
4, 0, 0, 0, 0, 0, 0, 0,
@ -96,11 +108,10 @@ TEST(Sink, string_view)
);
// clang-format on
s = {};
s << "longer string";
s = simpleToWire("longer string");
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
13, 0, 0, 0, 0, 0, 0, 0,
@ -113,13 +124,12 @@ TEST(Sink, string_view)
// clang-format on
}
TEST(Sink, StringSet)
TEST(WireFormatGenerator, StringSet)
{
StringSink s;
s << StringSet{};
auto s = simpleToWire(StringSet{});
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
0, 0, 0, 0, 0, 0, 0, 0,
@ -128,11 +138,10 @@ TEST(Sink, StringSet)
);
// clang-format on
s = {};
s << StringSet{"a", ""};
s = simpleToWire(StringSet{"a", ""});
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
2, 0, 0, 0, 0, 0, 0, 0,
@ -145,13 +154,12 @@ TEST(Sink, StringSet)
// clang-format on
}
TEST(Sink, Strings)
TEST(WireFormatGenerator, Strings)
{
StringSink s;
s << Strings{};
auto s = simpleToWire(Strings{});
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
0, 0, 0, 0, 0, 0, 0, 0,
@ -160,11 +168,10 @@ TEST(Sink, Strings)
);
// clang-format on
s = {};
s << Strings{"a", ""};
s = simpleToWire(Strings{"a", ""});
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
// length
2, 0, 0, 0, 0, 0, 0, 0,
@ -177,23 +184,22 @@ TEST(Sink, Strings)
// clang-format on
}
TEST(Sink, Error)
TEST(WireFormatGenerator, Error)
{
PosTable pt;
auto o = pt.addOrigin(Pos::String{make_ref<std::string>("test")}, 4);
StringSink s;
s << Error{{
auto s = simpleToWire(Error{{
.level = lvlInfo,
.msg = HintFmt("foo"),
.pos = pt[pt.add(o, 1)],
.traces = {{.pos = pt[pt.add(o, 2)], .hint = HintFmt("b %1%", "foo")}},
}};
}});
// NOTE position of the error and all traces are ignored
// by the wire format
// clang-format off
ASSERT_EQ(
s.s,
s,
std::string({
5, 0, 0, 0, 0, 0, 0, 0, 'E', 'r', 'r', 'o', 'r', 0, 0, 0,
3, 0, 0, 0, 0, 0, 0, 0,
@ -209,4 +215,46 @@ TEST(Sink, Error)
// clang-format on
}
TEST(FullFormatter, foo)
{
auto gen = []() -> Generator<std::span<const char>, SerializingTransform> {
std::set<std::string> foo{"a", "longer string", ""};
co_yield 42;
co_yield foo;
co_yield std::string_view("test");
co_yield 7;
}();
std::vector<char> full;
while (gen) {
auto s = gen();
full.insert(full.end(), s.begin(), s.end());
}
ASSERT_EQ(
full,
(std::vector<char>{
// clang-format off
// 32
42, 0, 0, 0, 0, 0, 0, 0,
// foo
3, 0, 0, 0, 0, 0, 0, 0,
/// ""
0, 0, 0, 0, 0, 0, 0, 0,
/// a
1, 0, 0, 0, 0, 0, 0, 0,
'a', 0, 0, 0, 0, 0, 0, 0,
/// longer string
13, 0, 0, 0, 0, 0, 0, 0,
'l', 'o', 'n', 'g', 'e', 'r', ' ', 's', 't', 'r', 'i', 'n', 'g', 0, 0, 0,
// foo done
// test
4, 0, 0, 0, 0, 0, 0, 0,
't', 'e', 's', 't', 0, 0, 0, 0,
// 7
7, 0, 0, 0, 0, 0, 0, 0,
//clang-format on
}));
}
}