philiptaron/lix
0
0
Fork 0
forked from lix-project/lix
Code Pull requests Activity

WIP systematize more of the worker protocol

Browse source 
This refactor should *not* change the wire protocol.
This commit is contained in:
John Ericson 2020-08-04 19:02:05 +00:00
parent fbeb8695fb
commit 2f2ae993dc
9 changed files with 132 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/libstore/build.cc
View file

@ -1864,11 +1864,11 @@ HookReply DerivationGoal::tryBuildHook()
/* Tell the hook all the inputs that have to be copied to the
remote system. */
writeStorePaths(worker.store, hook->sink, inputPaths);
write(worker.store, hook->sink, inputPaths);
/* Tell the hooks the missing outputs that have to be copied back
from the remote system. */
writeStorePaths(worker.store, hook->sink, missingPaths);
write(worker.store, hook->sink, missingPaths);
hook->sink = FdSink();
hook->toHook.writeSide = -1;

34
src/libstore/daemon.cc
View file

@ -256,11 +256,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryValidPaths: {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = read(*store, from, Proxy<StorePathSet> {});
logger->startWork();
auto res = store->queryValidPaths(paths);
logger->stopWork();
writeStorePaths(*store, to, res);
write(*store, to, res);
break;
}
@ -276,11 +276,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePaths: {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = read(*store, from, Proxy<StorePathSet> {});
logger->startWork();
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
writeStorePaths(*store, to, res);
write(*store, to, res);
break;
}
@ -309,7 +309,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
paths = store->queryValidDerivers(path);
else paths = store->queryDerivationOutputs(path);
logger->stopWork();
writeStorePaths(*store, to, paths);
write(*store, to, paths);
break;
}
@ -397,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
auto refs = readStorePaths<StorePathSet>(*store, from);
auto refs = read(*store, from, Proxy<StorePathSet> {});
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
logger->stopWork();
@ -518,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopCollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
options.pathsToDelete = read(*store, from, Proxy<StorePathSet> {});
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
@ -587,7 +587,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
to << 1
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
writeStorePaths(*store, to, i->second.references);
write(*store, to, i->second.references);
to << i->second.downloadSize
<< i->second.narSize;
}
@ -598,11 +598,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
SubstitutablePathInfos infos;
StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = read(*store, from, Proxy<StorePathSet> {});
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = readStorePathCAMap(*store, from);
pathsMap = read(*store, from, Proxy<StorePathCAMap> {});
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
@ -610,7 +610,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
for (auto & i : infos) {
to << store->printStorePath(i.first)
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
writeStorePaths(*store, to, i.second.references);
write(*store, to, i.second.references);
to << i.second.downloadSize << i.second.narSize;
}
break;
@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto paths = store->queryAllValidPaths();
logger->stopWork();
writeStorePaths(*store, to, paths);
write(*store, to, paths);
break;
}
@ -639,7 +639,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
write(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
@ -699,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.narHash = Hash(readString(from), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, from);
info.references = read(*store, from, Proxy<StorePathSet> {});
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
info.ca = parseContentAddressOpt(readString(from));
@ -799,9 +799,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);
writeStorePaths(*store, to, willSubstitute);
writeStorePaths(*store, to, unknown);
write(*store, to, willBuild);
write(*store, to, willSubstitute);
write(*store, to, unknown);
to << downloadSize << narSize;
break;
}

4
src/libstore/derivations.cc
View file

@ -477,7 +477,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
drv.outputs.emplace(std::move(name), std::move(output));
}
drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
drv.inputSrcs = read(store, in, Proxy<StorePathSet> {});
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);
@ -505,7 +505,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
out << "" << "";
}
}
writeStorePaths(store, out, drv.inputSrcs);
write(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)

4
src/libstore/export-import.cc
View file

@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
teeSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, teeSink, info->references);
write(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);
info.references = readStorePaths<StorePathSet>(*this, source);
info.references = read(*this, source, Proxy<StorePathSet> {});
auto deriver = readString(source);
if (deriver != "")

14
src/libstore/legacy-ssh-store.cc
View file

@ -107,7 +107,7 @@ struct LegacySSHStore : public Store
auto deriver = readString(conn->from);
if (deriver != "")
info->deriver = parseStorePath(deriver);
info->references = readStorePaths<StorePathSet>(*this, conn->from);
info->references = read(*this, conn->from, Proxy<StorePathSet> {});
readLongLong(conn->from); // download size
info->narSize = readLongLong(conn->from);
@ -139,7 +139,7 @@ struct LegacySSHStore : public Store
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
write(*this, conn->to, info.references);
conn->to
<< info.registrationTime
<< info.narSize
@ -168,7 +168,7 @@ struct LegacySSHStore : public Store
conn->to
<< exportMagic
<< printStorePath(info.path);
writeStorePaths(*this, conn->to, info.references);
write(*this, conn->to, info.references);
conn->to
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
@ -251,10 +251,10 @@ struct LegacySSHStore : public Store
conn->to
<< cmdQueryClosure
<< includeOutputs;
writeStorePaths(*this, conn->to, paths);
write(*this, conn->to, paths);
conn->to.flush();
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
for (auto & i : read(*this, conn->from, Proxy<StorePathSet> {}))
out.insert(i);
}
@ -267,10 +267,10 @@ struct LegacySSHStore : public Store
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute;
writeStorePaths(*this, conn->to, paths);
write(*this, conn->to, paths);
conn->to.flush();
return readStorePaths<StorePathSet>(*this, conn->from);
return read(*this, conn->from, Proxy<StorePathSet> {});
}
void connect() override

116
src/libstore/remote-store.cc
View file

@ -23,66 +23,44 @@
namespace nix {
template<> StorePathSet readStorePaths(const Store & store, Source & from)
{
StorePathSet paths;
for (auto & i : readStrings<Strings>(from))
paths.insert(store.parseStorePath(i));
return paths;
}
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths)
void write(const Store & store, Sink & out, const StorePathSet & paths)
{
out << paths.size();
for (auto & i : paths)
out << store.printStorePath(i);
}
std::string read(const Store & store, Source & from, Proxy<std::string> _)
{
return readString(from);
}
void write(const Store & store, Sink & out, const std::string & str)
{
out << str;
}
StorePath read(const Store & store, Source & from, Proxy<StorePath> _)
{
auto path = readString(from);
return store.parseStorePath(path);
return store.parseStorePath(readString(from));
}
StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
{
StorePathCAMap paths;
auto count = readNum<size_t>(from);
while (count--)
paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
return paths;
}
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
{
out << paths.size();
for (auto & i : paths) {
out << store.printStorePath(i.first);
out << renderContentAddress(i.second);
}
}
std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
{
std::map<string, StorePath> pathMap;
auto rawInput = readStrings<Strings>(from);
if (rawInput.size() % 2)
throw Error("got an odd number of elements from the daemon when trying to read a output path map");
auto curInput = rawInput.begin();
while (curInput != rawInput.end()) {
auto thisKey = *curInput++;
auto thisValue = *curInput++;
pathMap.emplace(thisKey, store.parseStorePath(thisValue));
}
return pathMap;
}
void write(const Store & store, Sink & out, const StorePath & storePath)
{
auto path = store.printStorePath(storePath);
out << path;
out << store.printStorePath(storePath);
}
ContentAddress read(const Store & store, Source & from, Proxy<ContentAddress> _)
{
return parseContentAddress(readString(from));
}
void write(const Store & store, Sink & out, const ContentAddress & ca)
{
out << renderContentAddress(ca);
}
@ -319,9 +297,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
return res;
} else {
conn->to << wopQueryValidPaths;
writeStorePaths(*this, conn->to, paths);
write(*this, conn->to, paths);
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
return read(*this, conn->from, Proxy<StorePathSet> {});
}
}
@ -331,7 +309,7 @@ StorePathSet RemoteStore::queryAllValidPaths()
auto conn(getConnection());
conn->to << wopQueryAllValidPaths;
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
return read(*this, conn->from, Proxy<StorePathSet> {});
}
@ -348,9 +326,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
return res;
} else {
conn->to << wopQuerySubstitutablePaths;
writeStorePaths(*this, conn->to, paths);
write(*this, conn->to, paths);
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
return read(*this, conn->from, Proxy<StorePathSet> {});
}
}
@ -372,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.references = read(*this, conn->from, Proxy<StorePathSet> {});
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
infos.insert_or_assign(i.first, std::move(info));
@ -385,9 +363,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
StorePathSet paths;
for (auto & path : pathsMap)
paths.insert(path.first);
writeStorePaths(*this, conn->to, paths);
write(*this, conn->to, paths);
} else
writeStorePathCAMap(*this, conn->to, pathsMap);
write(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
@ -395,7 +373,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
auto deriver = readString(conn->from);
if (deriver != "")
info.deriver = parseStorePath(deriver);
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.references = read(*this, conn->from, Proxy<StorePathSet> {});
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
}
@ -428,7 +406,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
auto deriver = readString(conn->from);
if (deriver != "") info->deriver = parseStorePath(deriver);
info->narHash = Hash(readString(conn->from), htSHA256);
info->references = readStorePaths<StorePathSet>(*this, conn->from);
info->references = read(*this, conn->from, Proxy<StorePathSet> {});
conn->from >> info->registrationTime >> info->narSize;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
conn->from >> info->ultimate;
@ -447,7 +425,7 @@ void RemoteStore::queryReferrers(const StorePath & path,
auto conn(getConnection());
conn->to << wopQueryReferrers << printStorePath(path);
conn.processStderr();
for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
for (auto & i : read(*this, conn->from, Proxy<StorePathSet> {}))
referrers.insert(i);
}
@ -457,7 +435,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
auto conn(getConnection());
conn->to << wopQueryValidDerivers << printStorePath(path);
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
return read(*this, conn->from, Proxy<StorePathSet> {});
}
@ -469,7 +447,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
}
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
return read(*this, conn->from, Proxy<StorePathSet> {});
}
@ -508,7 +486,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
sink
<< exportMagic
<< printStorePath(info.path);
writeStorePaths(*this, sink, info.references);
write(*this, sink, info.references);
sink
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0 // == no legacy signature
@ -518,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn.processStderr(0, source2.get());
auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from);
auto importedPaths = read(*this, conn->from, Proxy<StorePathSet> {});
assert(importedPaths.size() <= 1);
}
@ -527,7 +505,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
write(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
@ -660,7 +638,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
auto conn(getConnection());
conn->to << wopAddTextToStore << name << s;
writeStorePaths(*this, conn->to, references);
write(*this, conn->to, references);
conn.processStderr();
return parseStorePath(readString(conn->from));
@ -762,7 +740,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
conn->to
<< wopCollectGarbage << options.action;
writeStorePaths(*this, conn->to, options.pathsToDelete);
write(*this, conn->to, options.pathsToDelete);
conn->to << options.ignoreLiveness
<< options.maxFreed
/* removed options */
@ -824,9 +802,9 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
ss.push_back(p.to_string(*this));
conn->to << ss;
conn.processStderr();
willBuild = readStorePaths<StorePathSet>(*this, conn->from);
willSubstitute = readStorePaths<StorePathSet>(*this, conn->from);
unknown = readStorePaths<StorePathSet>(*this, conn->from);
willBuild = read(*this, conn->from, Proxy<StorePathSet> {});
willSubstitute = read(*this, conn->from, Proxy<StorePathSet> {});
unknown = read(*this, conn->from, Proxy<StorePathSet> {});
conn->from >> downloadSize >> narSize;
return;
}

59
src/libstore/worker-protocol.hh
View file

@ -66,33 +66,50 @@ typedef enum {
class Store;
struct Source;
template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
/* To guide overloading */
template<typename T>
struct Proxy {};
template<typename T>
std::map<std::string, T> read(const Store & store, Source & from, Proxy<std::map<std::string, T>> _)
std::set<T> read(const Store & store, Source & from, Proxy<std::set<T>> _)
{
std::map<string, T> resMap;
auto size = (size_t)readInt(from);
std::set<T> resSet;
auto size = readNum<size_t>(from);
while (size--) {
auto thisKey = readString(from);
resMap.insert_or_assign(std::move(thisKey), read(store, from, Proxy<T> {}));
resSet.insert(read(store, from, Proxy<T> {}));
}
return resSet;
}
template<typename T>
void write(const Store & store, Sink & out, const std::set<T> & resSet)
{
out << resSet.size();
for (auto & key : resSet) {
write(store, out, key);
}
}
template<typename K, typename V>
std::map<K, V> read(const Store & store, Source & from, Proxy<std::map<K, V>> _)
{
std::map<K, V> resMap;
auto size = readNum<size_t>(from);
while (size--) {
resMap.insert_or_assign(
read(store, from, Proxy<K> {}),
read(store, from, Proxy<V> {}));
}
return resMap;
}
template<typename T>
void write(const Store & store, Sink & out, const std::map<string, T> & resMap)
template<typename K, typename V>
void write(const Store & store, Sink & out, const std::map<K, V> & resMap)
{
out << resMap.size();
for (auto & i : resMap) {
out << i.first;
write(store, out, i.second);
for (auto & [key, value] : resMap) {
write(store, out, key);
write(store, out, value);
}
}
@ -106,26 +123,28 @@ std::optional<T> read(const Store & store, Source & from, Proxy<std::optional<T>
case 1:
return read(store, from, Proxy<T> {});
default:
throw Error("got an invalid tag bit for std::optional: %#04x", tag);
throw Error("got an invalid tag bit for std::optional: %#04x", (size_t)tag);
}
}
template<typename T>
void write(const Store & store, Sink & out, const std::optional<T> & optVal)
{
out << (optVal ? 1 : 0);
out << (uint64_t) (optVal ? 1 : 0);
if (optVal)
write(store, out, *optVal);
}
std::string read(const Store & store, Source & from, Proxy<std::string> _);
void write(const Store & store, Sink & out, const std::string & str);
StorePath read(const Store & store, Source & from, Proxy<StorePath> _);
void write(const Store & store, Sink & out, const StorePath & storePath);
StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
ContentAddress read(const Store & store, Source & from, Proxy<ContentAddress> _);
void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
void write(const Store & store, Sink & out, const ContentAddress & ca);
}

16
src/nix-store/nix-store.cc
View file

@ -815,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdQueryValidPaths: {
bool lock = readInt(in);
bool substitute = readInt(in);
auto paths = readStorePaths<StorePathSet>(*store, in);
auto paths = read(*store, in, Proxy<StorePathSet> {});
if (lock && writeAllowed)
for (auto & path : paths)
store->addTempRoot(path);
@ -845,19 +845,19 @@ static void opServe(Strings opFlags, Strings opArgs)
}
}
writeStorePaths(*store, out, store->queryValidPaths(paths));
write(*store, out, store->queryValidPaths(paths));
break;
}
case cmdQueryPathInfos: {
auto paths = readStorePaths<StorePathSet>(*store, in);
auto paths = read(*store, in, Proxy<StorePathSet> {});
// !!! Maybe we want a queryPathInfos?
for (auto & i : paths) {
try {
auto info = store->queryPathInfo(i);
out << store->printStorePath(info->path)
<< (info->deriver ? store->printStorePath(*info->deriver) : "");
writeStorePaths(*store, out, info->references);
write(*store, out, info->references);
// !!! Maybe we want compression?
out << info->narSize // downloadSize
<< info->narSize;
@ -885,7 +885,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdExportPaths: {
readInt(in); // obsolete
store->exportPaths(readStorePaths<StorePathSet>(*store, in), out);
store->exportPaths(read(*store, in, Proxy<StorePathSet> {}), out);
break;
}
@ -934,9 +934,9 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdQueryClosure: {
bool includeOutputs = readInt(in);
StorePathSet closure;
store->computeFSClosure(readStorePaths<StorePathSet>(*store, in),
store->computeFSClosure(read(*store, in, Proxy<StorePathSet> {}),
closure, false, includeOutputs);
writeStorePaths(*store, out, closure);
write(*store, out, closure);
break;
}
@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs)
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.narHash = Hash(readString(in), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, in);
info.references = read(*store, in, Proxy<StorePathSet> {});
in >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(in);
info.ca = parseContentAddressOpt(readString(in));