forked from lix-project/lix
Merge pull request #9099 from obsidiansystems/common-proto
Factor out bits of the worker protocol to use elsewhere
(cherry picked from commit 4b1a97338f517f45e6169d3d8845c5caa5724e97)
Change-Id: If93afa0f8b1cf9b0e705b34fa71e6fd708752758
This commit is contained in:
parent
da0aa66d98
commit
6897e238bd
|
@ -58,6 +58,23 @@ INPUT = \
|
|||
src/nix-env \
|
||||
src/nix-store
|
||||
|
||||
# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
|
||||
# in the source code. If set to NO, only conditional compilation will be
|
||||
# performed. Macro expansion can be done in a controlled way by setting
|
||||
# EXPAND_ONLY_PREDEF to YES.
|
||||
# The default value is: NO.
|
||||
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
|
||||
|
||||
MACRO_EXPANSION = YES
|
||||
|
||||
# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
|
||||
# the macro expansion is limited to the macros specified with the PREDEFINED and
|
||||
# EXPAND_AS_DEFINED tags.
|
||||
# The default value is: NO.
|
||||
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
|
||||
|
||||
EXPAND_ONLY_PREDEF = YES
|
||||
|
||||
# The INCLUDE_PATH tag can be used to specify one or more directories that
|
||||
# contain include files that are not input files but should be processed by the
|
||||
# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
|
||||
|
@ -65,3 +82,16 @@ INPUT = \
|
|||
# This tag requires that the tag SEARCH_INCLUDES is set to YES.
|
||||
|
||||
INCLUDE_PATH = @RAPIDCHECK_HEADERS@
|
||||
|
||||
# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
|
||||
# tag can be used to specify a list of macro names that should be expanded. The
|
||||
# macro definition that is found in the sources will be used. Use the PREDEFINED
|
||||
# tag if you want to use a different macro definition that overrules the
|
||||
# definition found in the source code.
|
||||
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
|
||||
|
||||
EXPAND_AS_DEFINED = \
|
||||
DECLARE_COMMON_SERIALISER \
|
||||
DECLARE_WORKER_SERIALISER \
|
||||
DECLARE_SERVE_SERIALISER \
|
||||
LENGTH_PREFIXED_PROTO_HELPER
|
||||
|
|
|
@ -8,8 +8,8 @@
|
|||
#include "util.hh"
|
||||
#include "archive.hh"
|
||||
#include "compression.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
#include "topo-sort.hh"
|
||||
#include "callback.hh"
|
||||
#include "local-store.hh" // TODO remove, along with remaining downcasts
|
||||
|
@ -1185,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
throw;
|
||||
}
|
||||
|
||||
WorkerProto::WriteConn conn { hook->sink };
|
||||
CommonProto::WriteConn conn { hook->sink };
|
||||
|
||||
/* Tell the hook all the inputs that have to be copied to the
|
||||
remote system. */
|
||||
WorkerProto::write(worker.store, conn, inputPaths);
|
||||
CommonProto::write(worker.store, conn, inputPaths);
|
||||
|
||||
/* Tell the hooks the missing outputs that have to be copied back
|
||||
from the remote system. */
|
||||
|
@ -1200,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook()
|
|||
if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
|
||||
missingOutputs.insert(outputName);
|
||||
}
|
||||
WorkerProto::write(worker.store, conn, missingOutputs);
|
||||
CommonProto::write(worker.store, conn, missingOutputs);
|
||||
}
|
||||
|
||||
hook->sink = FdSink();
|
||||
|
|
41
src/libstore/common-protocol-impl.hh
Normal file
41
src/libstore/common-protocol-impl.hh
Normal file
|
@ -0,0 +1,41 @@
|
|||
#pragma once
|
||||
/**
|
||||
* @file
|
||||
*
|
||||
* Template implementations (as opposed to mere declarations).
|
||||
*
|
||||
* This file is an exmample of the "impl.hh" pattern. See the
|
||||
* contributing guide.
|
||||
*/
|
||||
|
||||
#include "common-protocol.hh"
|
||||
#include "length-prefixed-protocol-helper.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* protocol-agnostic templates */
|
||||
|
||||
#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
|
||||
TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \
|
||||
{ \
|
||||
return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
|
||||
} \
|
||||
TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
|
||||
{ \
|
||||
LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
|
||||
}
|
||||
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::set<T>)
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
#define COMMA_ ,
|
||||
COMMON_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K COMMA_ typename V>,
|
||||
std::map<K COMMA_ V>)
|
||||
#undef COMMA_
|
||||
|
||||
|
||||
/* protocol-specific templates */
|
||||
|
||||
}
|
98
src/libstore/common-protocol.cc
Normal file
98
src/libstore/common-protocol.cc
Normal file
|
@ -0,0 +1,98 @@
|
|||
#include "serialise.hh"
|
||||
#include "util.hh"
|
||||
#include "path-with-outputs.hh"
|
||||
#include "store-api.hh"
|
||||
#include "build-result.hh"
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
#include "archive.hh"
|
||||
#include "derivations.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* protocol-agnostic definitions */
|
||||
|
||||
std::string CommonProto::Serialise<std::string>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return readString(conn.from);
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
|
||||
{
|
||||
conn.to << str;
|
||||
}
|
||||
|
||||
|
||||
StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return store.parseStorePath(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
|
||||
{
|
||||
conn.to << store.printStorePath(storePath);
|
||||
}
|
||||
|
||||
|
||||
ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return ContentAddress::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
|
||||
{
|
||||
conn.to << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
|
||||
Realisation CommonProto::Serialise<Realisation>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
std::string rawInput = readString(conn.from);
|
||||
return Realisation::fromJSON(
|
||||
nlohmann::json::parse(rawInput),
|
||||
"remote-protocol"
|
||||
);
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
|
||||
{
|
||||
conn.to << realisation.toJSON().dump();
|
||||
}
|
||||
|
||||
|
||||
DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return DrvOutput::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
|
||||
{
|
||||
conn.to << drvOutput.to_string();
|
||||
}
|
||||
|
||||
|
||||
std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
auto s = readString(conn.from);
|
||||
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
|
||||
{
|
||||
conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
|
||||
}
|
||||
|
||||
|
||||
std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, CommonProto::ReadConn conn)
|
||||
{
|
||||
return ContentAddress::parseOpt(readString(conn.from));
|
||||
}
|
||||
|
||||
void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
|
||||
{
|
||||
conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
|
||||
}
|
||||
|
||||
}
|
106
src/libstore/common-protocol.hh
Normal file
106
src/libstore/common-protocol.hh
Normal file
|
@ -0,0 +1,106 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "serialise.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class Store;
|
||||
struct Source;
|
||||
|
||||
// items being serialized
|
||||
class StorePath;
|
||||
struct ContentAddress;
|
||||
struct DrvOutput;
|
||||
struct Realisation;
|
||||
|
||||
|
||||
/**
|
||||
* Shared serializers between the worker protocol, serve protocol, and a
|
||||
* few others.
|
||||
*
|
||||
* This `struct` is basically just a `namespace`; We use a type rather
|
||||
* than a namespace just so we can use it as a template argument.
|
||||
*/
|
||||
struct CommonProto
|
||||
{
|
||||
/**
|
||||
* A unidirectional read connection, to be used by the read half of the
|
||||
* canonical serializers below.
|
||||
*/
|
||||
struct ReadConn {
|
||||
Source & from;
|
||||
};
|
||||
|
||||
/**
|
||||
* A unidirectional write connection, to be used by the write half of the
|
||||
* canonical serializers below.
|
||||
*/
|
||||
struct WriteConn {
|
||||
Sink & to;
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct Serialise;
|
||||
|
||||
/**
|
||||
* Wrapper function around `CommonProto::Serialise<T>::write` that allows us to
|
||||
* infer the type instead of having to write it down explicitly.
|
||||
*/
|
||||
template<typename T>
|
||||
static void write(const Store & store, WriteConn conn, const T & t)
|
||||
{
|
||||
CommonProto::Serialise<T>::write(store, conn, t);
|
||||
}
|
||||
};
|
||||
|
||||
#define DECLARE_COMMON_SERIALISER(T) \
|
||||
struct CommonProto::Serialise< T > \
|
||||
{ \
|
||||
static T read(const Store & store, CommonProto::ReadConn conn); \
|
||||
static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
|
||||
}
|
||||
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(std::string);
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(StorePath);
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(ContentAddress);
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(DrvOutput);
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(Realisation);
|
||||
|
||||
template<typename T>
|
||||
DECLARE_COMMON_SERIALISER(std::vector<T>);
|
||||
template<typename T>
|
||||
DECLARE_COMMON_SERIALISER(std::set<T>);
|
||||
template<typename... Ts>
|
||||
DECLARE_COMMON_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
#define COMMA_ ,
|
||||
template<typename K, typename V>
|
||||
DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V>);
|
||||
#undef COMMA_
|
||||
|
||||
/**
|
||||
* These use the empty string for the null case, relying on the fact
|
||||
* that the underlying types never serialize to the empty string.
|
||||
*
|
||||
* We do this instead of a generic std::optional<T> instance because
|
||||
* ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
|
||||
* the same reason, we don't have a std::variant<T..> instances (ordinal
|
||||
* tags 0...n).
|
||||
*
|
||||
* We could the generic instances and then these as specializations for
|
||||
* compatability, but that's proven a bit finnicky, and also makes the
|
||||
* worker protocol harder to implement in other languages where such
|
||||
* specializations may not be allowed.
|
||||
*/
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(std::optional<StorePath>);
|
||||
template<>
|
||||
DECLARE_COMMON_SERIALISER(std::optional<ContentAddress>);
|
||||
|
||||
}
|
|
@ -4,8 +4,8 @@
|
|||
#include "globals.hh"
|
||||
#include "util.hh"
|
||||
#include "split.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
#include "fs-accessor.hh"
|
||||
#include <boost/container/small_vector.hpp>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
@ -895,8 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
|
|||
drv.outputs.emplace(std::move(name), std::move(output));
|
||||
}
|
||||
|
||||
drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store,
|
||||
WorkerProto::ReadConn { .from = in });
|
||||
drv.inputSrcs = CommonProto::Serialise<StorePathSet>::read(store,
|
||||
CommonProto::ReadConn { .from = in });
|
||||
in >> drv.platform >> drv.builder;
|
||||
drv.args = readStrings<Strings>(in);
|
||||
|
||||
|
@ -944,8 +944,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
|||
},
|
||||
}, i.second.raw);
|
||||
}
|
||||
WorkerProto::write(store,
|
||||
WorkerProto::WriteConn { .to = out },
|
||||
CommonProto::write(store,
|
||||
CommonProto::WriteConn { .to = out },
|
||||
drv.inputSrcs);
|
||||
out << drv.platform << drv.builder << drv.args;
|
||||
out << drv.env.size();
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
#include "serialise.hh"
|
||||
#include "store-api.hh"
|
||||
#include "archive.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
|
@ -46,8 +46,8 @@ void Store::exportPath(const StorePath & path, Sink & sink)
|
|||
teeSink
|
||||
<< exportMagic
|
||||
<< printStorePath(path);
|
||||
WorkerProto::write(*this,
|
||||
WorkerProto::WriteConn { .to = teeSink },
|
||||
CommonProto::write(*this,
|
||||
CommonProto::WriteConn { .to = teeSink },
|
||||
info->references);
|
||||
teeSink
|
||||
<< (info->deriver ? printStorePath(*info->deriver) : "")
|
||||
|
@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
|
|||
|
||||
//Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
|
||||
|
||||
auto references = WorkerProto::Serialise<StorePathSet>::read(*this,
|
||||
WorkerProto::ReadConn { .from = source });
|
||||
auto references = CommonProto::Serialise<StorePathSet>::read(*this,
|
||||
CommonProto::ReadConn { .from = source });
|
||||
auto deriver = readString(source);
|
||||
auto narHash = hashString(htSHA256, saved.s);
|
||||
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
#include "build-result.hh"
|
||||
#include "store-api.hh"
|
||||
#include "path-with-outputs.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
#include "ssh.hh"
|
||||
#include "derivations.hh"
|
||||
#include "callback.hh"
|
||||
|
@ -50,37 +50,37 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
bool good = true;
|
||||
|
||||
/**
|
||||
* Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
|
||||
* factored out worker protocol searlizers with a
|
||||
* Coercion to `CommonProto::ReadConn`. This makes it easy to use the
|
||||
* factored out common protocol serialisers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The worker protocol connection types are unidirectional, unlike
|
||||
* The common protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*
|
||||
* @todo Use server protocol serializers, not worker protocol
|
||||
* @todo Use server protocol serializers, not common protocol
|
||||
* serializers, once we have made that distiction.
|
||||
*/
|
||||
operator WorkerProto::ReadConn ()
|
||||
operator CommonProto::ReadConn ()
|
||||
{
|
||||
return WorkerProto::ReadConn {
|
||||
return CommonProto::ReadConn {
|
||||
.from = from,
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
|
||||
* factored out worker protocol searlizers with a
|
||||
* Coercion to `CommonProto::WriteConn`. This makes it easy to use the
|
||||
* factored out common protocol searlizers with a
|
||||
* `LegacySSHStore::Connection`.
|
||||
*
|
||||
* The worker protocol connection types are unidirectional, unlike
|
||||
* The common protocol connection types are unidirectional, unlike
|
||||
* this type.
|
||||
*
|
||||
* @todo Use server protocol serializers, not worker protocol
|
||||
* @todo Use server protocol serializers, not common protocol
|
||||
* serializers, once we have made that distiction.
|
||||
*/
|
||||
operator WorkerProto::WriteConn ()
|
||||
operator CommonProto::WriteConn ()
|
||||
{
|
||||
return WorkerProto::WriteConn {
|
||||
return CommonProto::WriteConn {
|
||||
.to = to,
|
||||
};
|
||||
}
|
||||
|
@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
auto deriver = readString(conn->from);
|
||||
if (deriver != "")
|
||||
info->deriver = parseStorePath(deriver);
|
||||
info->references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
info->references = CommonProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
readLongLong(conn->from); // download size
|
||||
info->narSize = readLongLong(conn->from);
|
||||
|
||||
|
@ -217,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
<< printStorePath(info.path)
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< info.narHash.to_string(Base16, false);
|
||||
WorkerProto::write(*this, *conn, info.references);
|
||||
CommonProto::write(*this, *conn, info.references);
|
||||
conn->to
|
||||
<< info.registrationTime
|
||||
<< info.narSize
|
||||
|
@ -246,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
|
|||
conn->to
|
||||
<< exportMagic
|
||||
<< printStorePath(info.path);
|
||||
WorkerProto::write(*this, *conn, info.references);
|
||||
CommonProto::write(*this, *conn, info.references);
|
||||
conn->to
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< 0
|
||||
|
@ -331,7 +331,7 @@ public:
|
|||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
|
||||
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
|
||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
|
||||
auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn);
|
||||
auto builtOutputs = CommonProto::Serialise<DrvOutputs>::read(*this, *conn);
|
||||
for (auto && [output, realisation] : builtOutputs)
|
||||
status.builtOutputs.insert_or_assign(
|
||||
std::move(output.outputName),
|
||||
|
@ -409,10 +409,10 @@ public:
|
|||
conn->to
|
||||
<< ServeProto::Command::QueryClosure
|
||||
<< includeOutputs;
|
||||
WorkerProto::write(*this, *conn, paths);
|
||||
CommonProto::write(*this, *conn, paths);
|
||||
conn->to.flush();
|
||||
|
||||
for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, *conn))
|
||||
for (auto & i : CommonProto::Serialise<StorePathSet>::read(*this, *conn))
|
||||
out.insert(i);
|
||||
}
|
||||
|
||||
|
@ -425,10 +425,10 @@ public:
|
|||
<< ServeProto::Command::QueryValidPaths
|
||||
<< false // lock
|
||||
<< maybeSubstitute;
|
||||
WorkerProto::write(*this, *conn, paths);
|
||||
CommonProto::write(*this, *conn, paths);
|
||||
conn->to.flush();
|
||||
|
||||
return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
return CommonProto::Serialise<StorePathSet>::read(*this, *conn);
|
||||
}
|
||||
|
||||
void connect() override
|
||||
|
|
162
src/libstore/length-prefixed-protocol-helper.hh
Normal file
162
src/libstore/length-prefixed-protocol-helper.hh
Normal file
|
@ -0,0 +1,162 @@
|
|||
#pragma once
|
||||
/**
|
||||
* @file Reusable serialisers for serialization container types in a
|
||||
* length-prefixed manner.
|
||||
*
|
||||
* Used by both the Worker and Serve protocols.
|
||||
*/
|
||||
|
||||
#include "types.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class Store;
|
||||
|
||||
/**
|
||||
* Reusable serialisers for serialization container types in a
|
||||
* length-prefixed manner.
|
||||
*
|
||||
* @param T The type of the collection being serialised
|
||||
*
|
||||
* @param Inner This the most important parameter; this is the "inner"
|
||||
* protocol. The user of this will substitute `MyProtocol` or similar
|
||||
* when making a `MyProtocol::Serialiser<Collection<T>>`. Note that the
|
||||
* inside is allowed to call to call `Inner::Serialiser` on different
|
||||
* types. This is especially important for `std::map` which doesn't have
|
||||
* a single `T` but one `K` and one `V`.
|
||||
*/
|
||||
template<class Inner, typename T>
|
||||
struct LengthPrefixedProtoHelper;
|
||||
|
||||
/*!
|
||||
* \typedef LengthPrefixedProtoHelper::S
|
||||
*
|
||||
* Read this as simply `using S = Inner::Serialise;`.
|
||||
*
|
||||
* It would be nice to use that directly, but C++ doesn't seem to allow
|
||||
* it. The `typename` keyword needed to refer to `Inner` seems to greedy
|
||||
* (low precedence), and then C++ complains that `Serialise` is not a
|
||||
* type parameter but a real type.
|
||||
*
|
||||
* Making this `S` alias seems to be the only way to avoid these issues.
|
||||
*/
|
||||
|
||||
#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \
|
||||
struct LengthPrefixedProtoHelper< Inner, T > \
|
||||
{ \
|
||||
static T read(const Store & store, typename Inner::ReadConn conn); \
|
||||
static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \
|
||||
private: \
|
||||
template<typename U> using S = typename Inner::template Serialise<U>; \
|
||||
}
|
||||
|
||||
template<class Inner, typename T>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector<T>);
|
||||
|
||||
template<class Inner, typename T>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T>);
|
||||
|
||||
template<class Inner, typename... Ts>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>);
|
||||
|
||||
template<class Inner, typename K, typename V>
|
||||
#define _X std::map<K, V>
|
||||
LENGTH_PREFIXED_PROTO_HELPER(Inner, _X);
|
||||
#undef _X
|
||||
|
||||
template<class Inner, typename T>
|
||||
std::vector<T>
|
||||
LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
|
||||
const Store & store, typename Inner::ReadConn conn)
|
||||
{
|
||||
std::vector<T> resSet;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
resSet.push_back(S<T>::read(store, conn));
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<class Inner, typename T>
|
||||
void
|
||||
LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
|
||||
const Store & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
|
||||
{
|
||||
conn.to << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
S<T>::write(store, conn, key);
|
||||
}
|
||||
}
|
||||
|
||||
template<class Inner, typename T>
|
||||
std::set<T>
|
||||
LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
|
||||
const Store & store, typename Inner::ReadConn conn)
|
||||
{
|
||||
std::set<T> resSet;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
resSet.insert(S<T>::read(store, conn));
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<class Inner, typename T>
|
||||
void
|
||||
LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
|
||||
const Store & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
|
||||
{
|
||||
conn.to << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
S<T>::write(store, conn, key);
|
||||
}
|
||||
}
|
||||
|
||||
template<class Inner, typename K, typename V>
|
||||
std::map<K, V>
|
||||
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
|
||||
const Store & store, typename Inner::ReadConn conn)
|
||||
{
|
||||
std::map<K, V> resMap;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
auto k = S<K>::read(store, conn);
|
||||
auto v = S<V>::read(store, conn);
|
||||
resMap.insert_or_assign(std::move(k), std::move(v));
|
||||
}
|
||||
return resMap;
|
||||
}
|
||||
|
||||
template<class Inner, typename K, typename V>
|
||||
void
|
||||
LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
|
||||
const Store & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
|
||||
{
|
||||
conn.to << resMap.size();
|
||||
for (auto & i : resMap) {
|
||||
S<K>::write(store, conn, i.first);
|
||||
S<V>::write(store, conn, i.second);
|
||||
}
|
||||
}
|
||||
|
||||
template<class Inner, typename... Ts>
|
||||
std::tuple<Ts...>
|
||||
LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
|
||||
const Store & store, typename Inner::ReadConn conn)
|
||||
{
|
||||
return std::tuple<Ts...> {
|
||||
S<Ts>::read(store, conn)...,
|
||||
};
|
||||
}
|
||||
|
||||
template<class Inner, typename... Ts>
|
||||
void
|
||||
LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::write(
|
||||
const Store & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
|
||||
{
|
||||
std::apply([&]<typename... Us>(const Us &... args) {
|
||||
(S<Us>::write(store, conn, args), ...);
|
||||
}, res);
|
||||
}
|
||||
|
||||
}
|
|
@ -9,86 +9,51 @@
|
|||
*/
|
||||
|
||||
#include "worker-protocol.hh"
|
||||
#include "length-prefixed-protocol-helper.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* protocol-agnostic templates */
|
||||
|
||||
#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
|
||||
TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \
|
||||
{ \
|
||||
return LengthPrefixedProtoHelper<WorkerProto, T >::read(store, conn); \
|
||||
} \
|
||||
TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
|
||||
{ \
|
||||
LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
|
||||
}
|
||||
|
||||
WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
|
||||
WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::set<T>)
|
||||
WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
|
||||
|
||||
#define COMMA_ ,
|
||||
WORKER_USE_LENGTH_PREFIX_SERIALISER(
|
||||
template<typename K COMMA_ typename V>,
|
||||
std::map<K COMMA_ V>)
|
||||
#undef COMMA_
|
||||
|
||||
/**
|
||||
* Use `CommonProto` where possible.
|
||||
*/
|
||||
template<typename T>
|
||||
std::vector<T> WorkerProto::Serialise<std::vector<T>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
struct WorkerProto::Serialise
|
||||
{
|
||||
std::vector<T> resSet;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
resSet.push_back(WorkerProto::Serialise<T>::read(store, conn));
|
||||
static T read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return CommonProto::Serialise<T>::read(store,
|
||||
CommonProto::ReadConn { .from = conn.from });
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void WorkerProto::Serialise<std::vector<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector<T> & resSet)
|
||||
{
|
||||
conn.to << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
WorkerProto::Serialise<T>::write(store, conn, key);
|
||||
static void write(const Store & store, WorkerProto::WriteConn conn, const T & t)
|
||||
{
|
||||
CommonProto::Serialise<T>::write(store,
|
||||
CommonProto::WriteConn { .to = conn.to },
|
||||
t);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
std::set<T> WorkerProto::Serialise<std::set<T>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
std::set<T> resSet;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
resSet.insert(WorkerProto::Serialise<T>::read(store, conn));
|
||||
}
|
||||
return resSet;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void WorkerProto::Serialise<std::set<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::set<T> & resSet)
|
||||
{
|
||||
conn.to << resSet.size();
|
||||
for (auto & key : resSet) {
|
||||
WorkerProto::Serialise<T>::write(store, conn, key);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename K, typename V>
|
||||
std::map<K, V> WorkerProto::Serialise<std::map<K, V>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
std::map<K, V> resMap;
|
||||
auto size = readNum<size_t>(conn.from);
|
||||
while (size--) {
|
||||
auto k = WorkerProto::Serialise<K>::read(store, conn);
|
||||
auto v = WorkerProto::Serialise<V>::read(store, conn);
|
||||
resMap.insert_or_assign(std::move(k), std::move(v));
|
||||
}
|
||||
return resMap;
|
||||
}
|
||||
|
||||
template<typename K, typename V>
|
||||
void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, WorkerProto::WriteConn conn, const std::map<K, V> & resMap)
|
||||
{
|
||||
conn.to << resMap.size();
|
||||
for (auto & i : resMap) {
|
||||
WorkerProto::Serialise<K>::write(store, conn, i.first);
|
||||
WorkerProto::Serialise<V>::write(store, conn, i.second);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename... Ts>
|
||||
std::tuple<Ts...> WorkerProto::Serialise<std::tuple<Ts...>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return std::tuple<Ts...> {
|
||||
WorkerProto::Serialise<Ts>::read(store, conn)...,
|
||||
};
|
||||
}
|
||||
|
||||
template<typename... Ts>
|
||||
void WorkerProto::Serialise<std::tuple<Ts...>>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple<Ts...> & res)
|
||||
{
|
||||
std::apply([&]<typename... Us>(const Us &... args) {
|
||||
(WorkerProto::Serialise<Us>::write(store, conn, args), ...);
|
||||
}, res);
|
||||
}
|
||||
/* protocol-specific templates */
|
||||
|
||||
}
|
||||
|
|
|
@ -12,27 +12,7 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
std::string WorkerProto::Serialise<std::string>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return readString(conn.from);
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<std::string>::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str)
|
||||
{
|
||||
conn.to << str;
|
||||
}
|
||||
|
||||
|
||||
StorePath WorkerProto::Serialise<StorePath>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return store.parseStorePath(readString(conn.from));
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<StorePath>::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath)
|
||||
{
|
||||
conn.to << store.printStorePath(storePath);
|
||||
}
|
||||
|
||||
/* protocol-specific definitions */
|
||||
|
||||
std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
|
@ -68,17 +48,6 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & sto
|
|||
}
|
||||
|
||||
|
||||
ContentAddress WorkerProto::Serialise<ContentAddress>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return ContentAddress::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<ContentAddress>::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca)
|
||||
{
|
||||
conn.to << renderContentAddress(ca);
|
||||
}
|
||||
|
||||
|
||||
DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
auto s = readString(conn.from);
|
||||
|
@ -91,32 +60,6 @@ void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto
|
|||
}
|
||||
|
||||
|
||||
Realisation WorkerProto::Serialise<Realisation>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
std::string rawInput = readString(conn.from);
|
||||
return Realisation::fromJSON(
|
||||
nlohmann::json::parse(rawInput),
|
||||
"remote-protocol"
|
||||
);
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<Realisation>::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation)
|
||||
{
|
||||
conn.to << realisation.toJSON().dump();
|
||||
}
|
||||
|
||||
|
||||
DrvOutput WorkerProto::Serialise<DrvOutput>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return DrvOutput::parse(readString(conn.from));
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<DrvOutput>::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput)
|
||||
{
|
||||
conn.to << drvOutput.to_string();
|
||||
}
|
||||
|
||||
|
||||
KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn);
|
||||
|
@ -168,26 +111,4 @@ void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto
|
|||
}
|
||||
|
||||
|
||||
std::optional<StorePath> WorkerProto::Serialise<std::optional<StorePath>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
auto s = readString(conn.from);
|
||||
return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<std::optional<StorePath>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
|
||||
{
|
||||
conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
|
||||
}
|
||||
|
||||
|
||||
std::optional<ContentAddress> WorkerProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, WorkerProto::ReadConn conn)
|
||||
{
|
||||
return ContentAddress::parseOpt(readString(conn.from));
|
||||
}
|
||||
|
||||
void WorkerProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
|
||||
{
|
||||
conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "serialise.hh"
|
||||
#include "common-protocol.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
@ -28,11 +28,7 @@ class Store;
|
|||
struct Source;
|
||||
|
||||
// items being serialised
|
||||
class StorePath;
|
||||
struct ContentAddress;
|
||||
struct DerivedPath;
|
||||
struct DrvOutput;
|
||||
struct Realisation;
|
||||
struct BuildResult;
|
||||
struct KeyedBuildResult;
|
||||
enum TrustedFlag : bool;
|
||||
|
@ -193,60 +189,32 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
|
|||
* be legal specialization syntax. See below for what that looks like in
|
||||
* practice.
|
||||
*/
|
||||
#define MAKE_WORKER_PROTO(T) \
|
||||
struct WorkerProto::Serialise< T > { \
|
||||
#define DECLARE_WORKER_SERIALISER(T) \
|
||||
struct WorkerProto::Serialise< T > \
|
||||
{ \
|
||||
static T read(const Store & store, WorkerProto::ReadConn conn); \
|
||||
static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
|
||||
};
|
||||
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::string);
|
||||
DECLARE_WORKER_SERIALISER(DerivedPath);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(StorePath);
|
||||
DECLARE_WORKER_SERIALISER(BuildResult);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(ContentAddress);
|
||||
DECLARE_WORKER_SERIALISER(KeyedBuildResult);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(DerivedPath);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(DrvOutput);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(Realisation);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(BuildResult);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(KeyedBuildResult);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
|
||||
DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
|
||||
|
||||
template<typename T>
|
||||
MAKE_WORKER_PROTO(std::vector<T>);
|
||||
DECLARE_WORKER_SERIALISER(std::vector<T>);
|
||||
template<typename T>
|
||||
MAKE_WORKER_PROTO(std::set<T>);
|
||||
DECLARE_WORKER_SERIALISER(std::set<T>);
|
||||
template<typename... Ts>
|
||||
MAKE_WORKER_PROTO(std::tuple<Ts...>);
|
||||
DECLARE_WORKER_SERIALISER(std::tuple<Ts...>);
|
||||
|
||||
#define COMMA_ ,
|
||||
template<typename K, typename V>
|
||||
#define X_ std::map<K, V>
|
||||
MAKE_WORKER_PROTO(X_);
|
||||
#undef X_
|
||||
|
||||
/**
|
||||
* These use the empty string for the null case, relying on the fact
|
||||
* that the underlying types never serialise to the empty string.
|
||||
*
|
||||
* We do this instead of a generic std::optional<T> instance because
|
||||
* ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
|
||||
* the same reason, we don't have a std::variant<T..> instances (ordinal
|
||||
* tags 0...n).
|
||||
*
|
||||
* We could the generic instances and then these as specializations for
|
||||
* compatability, but that's proven a bit finnicky, and also makes the
|
||||
* worker protocol harder to implement in other languages where such
|
||||
* specializations may not be allowed.
|
||||
*/
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::optional<StorePath>);
|
||||
template<>
|
||||
MAKE_WORKER_PROTO(std::optional<ContentAddress>);
|
||||
DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V>);
|
||||
#undef COMMA_
|
||||
|
||||
}
|
||||
|
|
|
@ -11,8 +11,8 @@
|
|||
#include "serve-protocol.hh"
|
||||
#include "shared.hh"
|
||||
#include "util.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "worker-protocol-impl.hh"
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
#include "graphml.hh"
|
||||
#include "legacy.hh"
|
||||
#include "path-with-outputs.hh"
|
||||
|
@ -821,8 +821,8 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
out.flush();
|
||||
unsigned int clientVersion = readInt(in);
|
||||
|
||||
WorkerProto::ReadConn rconn { .from = in };
|
||||
WorkerProto::WriteConn wconn { .to = out };
|
||||
CommonProto::ReadConn rconn { .from = in };
|
||||
CommonProto::WriteConn wconn { .to = out };
|
||||
|
||||
auto getBuildSettings = [&]() {
|
||||
// FIXME: changing options here doesn't work if we're
|
||||
|
@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
case ServeProto::Command::QueryValidPaths: {
|
||||
bool lock = readInt(in);
|
||||
bool substitute = readInt(in);
|
||||
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
auto paths = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
if (lock && writeAllowed)
|
||||
for (auto & path : paths)
|
||||
store->addTempRoot(path);
|
||||
|
@ -876,19 +876,19 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
store->substitutePaths(paths);
|
||||
}
|
||||
|
||||
WorkerProto::write(*store, wconn, store->queryValidPaths(paths));
|
||||
CommonProto::write(*store, wconn, store->queryValidPaths(paths));
|
||||
break;
|
||||
}
|
||||
|
||||
case ServeProto::Command::QueryPathInfos: {
|
||||
auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
auto paths = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
// !!! Maybe we want a queryPathInfos?
|
||||
for (auto & i : paths) {
|
||||
try {
|
||||
auto info = store->queryPathInfo(i);
|
||||
out << store->printStorePath(info->path)
|
||||
<< (info->deriver ? store->printStorePath(*info->deriver) : "");
|
||||
WorkerProto::write(*store, wconn, info->references);
|
||||
CommonProto::write(*store, wconn, info->references);
|
||||
// !!! Maybe we want compression?
|
||||
out << info->narSize // downloadSize
|
||||
<< info->narSize;
|
||||
|
@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
|
||||
case ServeProto::Command::ExportPaths: {
|
||||
readInt(in); // obsolete
|
||||
store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, rconn), out);
|
||||
store->exportPaths(CommonProto::Serialise<StorePathSet>::read(*store, rconn), out);
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
DrvOutputs builtOutputs;
|
||||
for (auto & [output, realisation] : status.builtOutputs)
|
||||
builtOutputs.insert_or_assign(realisation.id, realisation);
|
||||
WorkerProto::write(*store, wconn, builtOutputs);
|
||||
CommonProto::write(*store, wconn, builtOutputs);
|
||||
}
|
||||
|
||||
break;
|
||||
|
@ -971,9 +971,9 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
case ServeProto::Command::QueryClosure: {
|
||||
bool includeOutputs = readInt(in);
|
||||
StorePathSet closure;
|
||||
store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, rconn),
|
||||
store->computeFSClosure(CommonProto::Serialise<StorePathSet>::read(*store, rconn),
|
||||
closure, false, includeOutputs);
|
||||
WorkerProto::write(*store, wconn, closure);
|
||||
CommonProto::write(*store, wconn, closure);
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -988,7 +988,7 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
};
|
||||
if (deriver != "")
|
||||
info.deriver = store->parseStorePath(deriver);
|
||||
info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
info.references = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
|
||||
in >> info.registrationTime >> info.narSize >> info.ultimate;
|
||||
info.sigs = readStrings<StringSet>(in);
|
||||
info.ca = ContentAddress::parseOpt(readString(in));
|
||||
|
|
23
tests/unit/libstore/characterization.hh
Normal file
23
tests/unit/libstore/characterization.hh
Normal file
|
@ -0,0 +1,23 @@
|
|||
#pragma once
|
||||
///@file
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* The path to the `unit-test-data` directory. See the contributing
|
||||
* guide in the manual for further details.
|
||||
*/
|
||||
static Path getUnitTestData() {
|
||||
return getEnv("_NIX_TEST_UNIT_DATA").value();
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether we should update "golden masters" instead of running tests
|
||||
* against them. See the contributing guide in the manual for further
|
||||
* details.
|
||||
*/
|
||||
static bool testAccept() {
|
||||
return getEnv("_NIX_TEST_ACCEPT") == "1";
|
||||
}
|
||||
|
||||
}
|
152
tests/unit/libstore/common-protocol.cc
Normal file
152
tests/unit/libstore/common-protocol.cc
Normal file
|
@ -0,0 +1,152 @@
|
|||
#include <regex>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "common-protocol.hh"
|
||||
#include "common-protocol-impl.hh"
|
||||
#include "build-result.hh"
|
||||
#include "protocol.hh"
|
||||
#include "characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
const char commonProtoDir[] = "common-protocol";
|
||||
|
||||
using CommonProtoTest = ProtoTest<CommonProto, commonProtoDir>;
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
string,
|
||||
"string",
|
||||
(std::tuple<std::string, std::string, std::string, std::string, std::string> {
|
||||
"",
|
||||
"hi",
|
||||
"white rabbit",
|
||||
"大白兔",
|
||||
"oh no \0\0\0 what was that!",
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
storePath,
|
||||
"store-path",
|
||||
(std::tuple<StorePath, StorePath> {
|
||||
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
|
||||
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" },
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
contentAddress,
|
||||
"content-address",
|
||||
(std::tuple<ContentAddress, ContentAddress, ContentAddress> {
|
||||
ContentAddress {
|
||||
.method = TextIngestionMethod {},
|
||||
.hash = hashString(HashType::htSHA256, "Derive(...)"),
|
||||
},
|
||||
ContentAddress {
|
||||
.method = FileIngestionMethod::Flat,
|
||||
.hash = hashString(HashType::htSHA1, "blob blob..."),
|
||||
},
|
||||
ContentAddress {
|
||||
.method = FileIngestionMethod::Recursive,
|
||||
.hash = hashString(HashType::htSHA256, "(...)"),
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
(std::tuple<DrvOutput, DrvOutput> {
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
DrvOutput {
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
realisation,
|
||||
"realisation",
|
||||
(std::tuple<Realisation, Realisation> {
|
||||
Realisation {
|
||||
.id = DrvOutput {
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
|
||||
.signatures = { "asdf", "qwer" },
|
||||
},
|
||||
Realisation {
|
||||
.id = {
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
.outputName = "baz",
|
||||
},
|
||||
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
|
||||
.signatures = { "asdf", "qwer" },
|
||||
.dependentRealisations = {
|
||||
{
|
||||
DrvOutput {
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
vector,
|
||||
"vector",
|
||||
(std::tuple<std::vector<std::string>, std::vector<std::string>, std::vector<std::string>, std::vector<std::vector<std::string>>> {
|
||||
{ },
|
||||
{ "" },
|
||||
{ "", "foo", "bar" },
|
||||
{ {}, { "" }, { "", "1", "2" } },
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
set,
|
||||
"set",
|
||||
(std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
|
||||
{ },
|
||||
{ "" },
|
||||
{ "", "foo", "bar" },
|
||||
{ {}, { "" }, { "", "1", "2" } },
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
optionalStorePath,
|
||||
"optional-store-path",
|
||||
(std::tuple<std::optional<StorePath>, std::optional<StorePath>> {
|
||||
std::nullopt,
|
||||
std::optional {
|
||||
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" },
|
||||
},
|
||||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
CommonProtoTest,
|
||||
optionalContentAddress,
|
||||
"optional-content-address",
|
||||
(std::tuple<std::optional<ContentAddress>, std::optional<ContentAddress>> {
|
||||
std::nullopt,
|
||||
std::optional {
|
||||
ContentAddress {
|
||||
.method = FileIngestionMethod::Flat,
|
||||
.hash = hashString(HashType::htSHA1, "blob blob..."),
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
}
|
88
tests/unit/libstore/protocol.hh
Normal file
88
tests/unit/libstore/protocol.hh
Normal file
|
@ -0,0 +1,88 @@
|
|||
#include <nlohmann/json.hpp>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "tests/libstore.hh"
|
||||
#include "characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
template<class Proto, const char * protocolDir>
|
||||
class ProtoTest : public LibStoreTest
|
||||
{
|
||||
/**
|
||||
* Read this as simply `using S = Inner::Serialise;`.
|
||||
*
|
||||
* See `LengthPrefixedProtoHelper::S` for the same trick, and its
|
||||
* rationale.
|
||||
*/
|
||||
template<typename U> using S = typename Proto::template Serialise<U>;
|
||||
|
||||
public:
|
||||
Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir;
|
||||
|
||||
Path goldenMaster(std::string_view testStem) {
|
||||
return unitTestData + "/" + testStem + ".bin";
|
||||
}
|
||||
|
||||
/**
|
||||
* Golden test for `T` reading
|
||||
*/
|
||||
template<typename T>
|
||||
void readTest(PathView testStem, T value)
|
||||
{
|
||||
if (testAccept())
|
||||
{
|
||||
GTEST_SKIP() << "Cannot read golden master because another test is also updating it";
|
||||
}
|
||||
else
|
||||
{
|
||||
auto expected = readFile(goldenMaster(testStem));
|
||||
|
||||
T got = ({
|
||||
StringSource from { expected };
|
||||
S<T>::read(
|
||||
*store,
|
||||
typename Proto::ReadConn { .from = from });
|
||||
});
|
||||
|
||||
ASSERT_EQ(got, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Golden test for `T` write
|
||||
*/
|
||||
template<typename T>
|
||||
void writeTest(PathView testStem, const T & value)
|
||||
{
|
||||
auto file = goldenMaster(testStem);
|
||||
|
||||
StringSink to;
|
||||
Proto::write(
|
||||
*store,
|
||||
typename Proto::WriteConn { .to = to },
|
||||
value);
|
||||
|
||||
if (testAccept())
|
||||
{
|
||||
createDirs(dirOf(file));
|
||||
writeFile(file, to.s);
|
||||
GTEST_SKIP() << "Updating golden master";
|
||||
}
|
||||
else
|
||||
{
|
||||
auto expected = readFile(file);
|
||||
ASSERT_EQ(to.s, expected);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#define CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VALUE) \
|
||||
TEST_F(FIXTURE, NAME ## _read) { \
|
||||
readTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(FIXTURE, NAME ## _write) { \
|
||||
writeTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
}
|
|
@ -7,85 +7,17 @@
|
|||
#include "worker-protocol-impl.hh"
|
||||
#include "derived-path.hh"
|
||||
#include "build-result.hh"
|
||||
#include "tests/libstore.hh"
|
||||
#include "protocol.hh"
|
||||
#include "characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class WorkerProtoTest : public LibStoreTest
|
||||
{
|
||||
public:
|
||||
Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol";
|
||||
const char workerProtoDir[] = "worker-protocol";
|
||||
|
||||
bool testAccept() {
|
||||
return getEnv("_NIX_TEST_ACCEPT") == "1";
|
||||
}
|
||||
|
||||
Path goldenMaster(std::string_view testStem) {
|
||||
return unitTestData + "/" + testStem + ".bin";
|
||||
}
|
||||
|
||||
/**
|
||||
* Golden test for `T` reading
|
||||
*/
|
||||
template<typename T>
|
||||
void readTest(PathView testStem, T value)
|
||||
{
|
||||
if (testAccept())
|
||||
{
|
||||
GTEST_SKIP() << "Cannot read golden master because another test is also updating it";
|
||||
}
|
||||
else
|
||||
{
|
||||
auto expected = readFile(goldenMaster(testStem));
|
||||
|
||||
T got = ({
|
||||
StringSource from { expected };
|
||||
WorkerProto::Serialise<T>::read(
|
||||
*store,
|
||||
WorkerProto::ReadConn { .from = from });
|
||||
});
|
||||
|
||||
ASSERT_EQ(got, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Golden test for `T` write
|
||||
*/
|
||||
template<typename T>
|
||||
void writeTest(PathView testStem, const T & value)
|
||||
{
|
||||
auto file = goldenMaster(testStem);
|
||||
|
||||
StringSink to;
|
||||
WorkerProto::write(
|
||||
*store,
|
||||
WorkerProto::WriteConn { .to = to },
|
||||
value);
|
||||
|
||||
if (testAccept())
|
||||
{
|
||||
createDirs(dirOf(file));
|
||||
writeFile(file, to.s);
|
||||
GTEST_SKIP() << "Updating golden master";
|
||||
}
|
||||
else
|
||||
{
|
||||
auto expected = readFile(file);
|
||||
ASSERT_EQ(to.s, expected);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(WorkerProtoTest, NAME ## _read) { \
|
||||
readTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(WorkerProtoTest, NAME ## _write) { \
|
||||
writeTest(STEM, VALUE); \
|
||||
}
|
||||
using WorkerProtoTest = ProtoTest<WorkerProto, workerProtoDir>;
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
string,
|
||||
"string",
|
||||
(std::tuple<std::string, std::string, std::string, std::string, std::string> {
|
||||
|
@ -97,6 +29,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
storePath,
|
||||
"store-path",
|
||||
(std::tuple<StorePath, StorePath> {
|
||||
|
@ -105,6 +38,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
contentAddress,
|
||||
"content-address",
|
||||
(std::tuple<ContentAddress, ContentAddress, ContentAddress> {
|
||||
|
@ -123,6 +57,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
derivedPath,
|
||||
"derived-path",
|
||||
(std::tuple<DerivedPath, DerivedPath> {
|
||||
|
@ -138,6 +73,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
drvOutput,
|
||||
"drv-output",
|
||||
(std::tuple<DrvOutput, DrvOutput> {
|
||||
|
@ -152,6 +88,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
realisation,
|
||||
"realisation",
|
||||
(std::tuple<Realisation, Realisation> {
|
||||
|
@ -183,6 +120,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
buildResult,
|
||||
"build-result",
|
||||
({
|
||||
|
@ -240,6 +178,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
keyedBuildResult,
|
||||
"keyed-build-result",
|
||||
({
|
||||
|
@ -275,6 +214,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
optionalTrustedFlag,
|
||||
"optional-trusted-flag",
|
||||
(std::tuple<std::optional<TrustedFlag>, std::optional<TrustedFlag>, std::optional<TrustedFlag>> {
|
||||
|
@ -284,6 +224,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
vector,
|
||||
"vector",
|
||||
(std::tuple<std::vector<std::string>, std::vector<std::string>, std::vector<std::string>, std::vector<std::vector<std::string>>> {
|
||||
|
@ -294,6 +235,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
set,
|
||||
"set",
|
||||
(std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
|
||||
|
@ -304,6 +246,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
optionalStorePath,
|
||||
"optional-store-path",
|
||||
(std::tuple<std::optional<StorePath>, std::optional<StorePath>> {
|
||||
|
@ -314,6 +257,7 @@ CHARACTERIZATION_TEST(
|
|||
}))
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
optionalContentAddress,
|
||||
"optional-content-address",
|
||||
(std::tuple<std::optional<ContentAddress>, std::optional<ContentAddress>> {
|
||||
|
|
BIN
unit-test-data/libstore/common-protocol/content-address.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/content-address.bin
Normal file
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/drv-output.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/drv-output.bin
Normal file
Binary file not shown.
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/optional-store-path.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/optional-store-path.bin
Normal file
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/realisation.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/realisation.bin
Normal file
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/set.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/set.bin
Normal file
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/store-path.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/store-path.bin
Normal file
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/string.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/string.bin
Normal file
Binary file not shown.
BIN
unit-test-data/libstore/common-protocol/vector.bin
Normal file
BIN
unit-test-data/libstore/common-protocol/vector.bin
Normal file
Binary file not shown.
Loading…
Reference in a new issue