From cb5052d98fa9a5d64d1700fe434c1c37a72e45d1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 May 2023 22:04:59 -0400 Subject: [PATCH] Revert "Revert "Use template structs instead of phantoms"" This is the more typically way to do [Argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)-leveraging generic serializers in C++. It makes the relationship between the `read` and `write` methods more clear and rigorous, and also looks more familiar to users coming from other languages that do not have C++'s libertine ad-hoc overloading. I am returning to this because during the review in https://github.com/NixOS/nix/pull/6223, it came up as something that would make the code easier to read --- easier today hopefully already, but definitely easier if we were have multiple codified protocols with code sharing between them as that PR seeks to accomplish. If I recall correctly, the main criticism of this the first time around (in 2020) was that having to specify the type when writing, e.g. `WorkerProto::write`, was too verbose and cumbersome. This is now addressed with the `workerProtoWrite` wrapper function. This method is also the way `nlohmann::json`, which we have used for a number of years now, does its serializers, for what its worth. This reverts commit 45a0ed82f089158a79c8c25ef844c55e4a74fc35. That commit in turn reverted 9ab07e99f527d1fa3adfa02839da477a1528d64b. --- src/libstore/build/derivation-goal.cc | 4 +- src/libstore/daemon.cc | 50 ++++++------ src/libstore/derivations.cc | 4 +- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 16 ++-- src/libstore/path-info.cc | 4 +- src/libstore/remote-store.cc | 62 +++++++------- src/libstore/worker-protocol.cc | 58 ++++++------- src/libstore/worker-protocol.hh | 113 ++++++++++++++++---------- src/nix-store/nix-store.cc | 18 ++-- 10 files changed, 182 insertions(+), 151 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5b1c923cd..df7d21e54 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1152,7 +1152,7 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); + workerProtoWrite(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1163,7 +1163,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - worker_proto::write(worker.store, hook->sink, missingOutputs); + workerProtoWrite(worker.store, hook->sink, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5083497a9..b6dd83684 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -263,7 +263,7 @@ static std::vector readDerivedPaths(Store & store, unsigned int cli { std::vector reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = worker_proto::read(store, from, Phantom> {}); + reqs = WorkerProto>::read(store, from); } else { for (auto & s : readStrings(from)) reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); @@ -287,7 +287,7 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQueryValidPaths: { - auto paths = worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); SubstituteFlag substitute = NoSubstitute; if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { @@ -300,7 +300,7 @@ static void performOp(TunnelLogger * logger, ref store, } auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); - worker_proto::write(*store, to, res); + workerProtoWrite(*store, to, res); break; } @@ -316,11 +316,11 @@ static void performOp(TunnelLogger * logger, ref store, } case wopQuerySubstitutablePaths: { - auto paths = worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - worker_proto::write(*store, to, res); + workerProtoWrite(*store, to, res); break; } @@ -349,7 +349,7 @@ static void performOp(TunnelLogger * logger, ref store, paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - worker_proto::write(*store, to, paths); + workerProtoWrite(*store, to, paths); break; } @@ -367,7 +367,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - worker_proto::write(*store, to, outputs); + workerProtoWrite(*store, to, outputs); break; } @@ -393,7 +393,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = worker_proto::read(*store, from, Phantom {}); + auto refs = WorkerProto::read(*store, from); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -495,7 +495,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopAddTextToStore: { std::string suffix = readString(from); std::string s = readString(from); - auto refs = worker_proto::read(*store, from, Phantom {}); + auto refs = WorkerProto::read(*store, from); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -567,7 +567,7 @@ static void performOp(TunnelLogger * logger, ref store, auto results = store->buildPathsWithResults(drvs, mode); logger->stopWork(); - worker_proto::write(*store, to, results); + workerProtoWrite(*store, to, results); break; } @@ -644,7 +644,7 @@ static void performOp(TunnelLogger * logger, ref store, DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - worker_proto::write(*store, to, builtOutputs); + workerProtoWrite(*store, to, builtOutputs); } break; } @@ -709,7 +709,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = worker_proto::read(*store, from, Phantom {}); + options.pathsToDelete = WorkerProto::read(*store, from); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -779,7 +779,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + workerProtoWrite(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -790,11 +790,11 @@ static void performOp(TunnelLogger * logger, ref store, SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = worker_proto::read(*store, from, Phantom {}); + auto paths = WorkerProto::read(*store, from); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = worker_proto::read(*store, from, Phantom {}); + pathsMap = WorkerProto::read(*store, from); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -802,7 +802,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + workerProtoWrite(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -812,7 +812,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - worker_proto::write(*store, to, paths); + workerProtoWrite(*store, to, paths); break; } @@ -884,7 +884,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom {}); + info.references = WorkerProto::read(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = ContentAddress::parseOpt(readString(from)); @@ -935,9 +935,9 @@ static void performOp(TunnelLogger * logger, ref store, uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - worker_proto::write(*store, to, willBuild); - worker_proto::write(*store, to, willSubstitute); - worker_proto::write(*store, to, unknown); + workerProtoWrite(*store, to, willBuild); + workerProtoWrite(*store, to, willSubstitute); + workerProtoWrite(*store, to, unknown); to << downloadSize << narSize; break; } @@ -950,7 +950,7 @@ static void performOp(TunnelLogger * logger, ref store, store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { - auto realisation = worker_proto::read(*store, from, Phantom()); + auto realisation = WorkerProto::read(*store, from); store->registerDrvOutput(realisation); } logger->stopWork(); @@ -965,11 +965,11 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + workerProtoWrite(*store, to, outPaths); } else { std::set realisations; if (info) realisations.insert(*info); - worker_proto::write(*store, to, realisations); + workerProtoWrite(*store, to, realisations); } break; } @@ -1045,7 +1045,7 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - worker_proto::write(*store, to, temp); + workerProtoWrite(*store, to, temp); } /* Send startup error messages to the client. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d56dc727b..15d3e07b3 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -748,7 +748,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = worker_proto::read(store, in, Phantom {}); + drv.inputSrcs = WorkerProto::read(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -796,7 +796,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw()); } - worker_proto::write(store, out, drv.inputSrcs); + workerProtoWrite(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4eb838b68..5ea263a86 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + workerProtoWrite(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = worker_proto::read(*this, source, Phantom {}); + auto references = WorkerProto::read(*this, source); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2012584e0..9a470a99e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -146,7 +146,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom {}); + info->references = WorkerProto::read(*this, conn->from); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -180,7 +180,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + workerProtoWrite(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -209,7 +209,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + workerProtoWrite(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -294,7 +294,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + auto builtOutputs = WorkerProto::read(*this, conn->from); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -358,10 +358,10 @@ public: conn->to << cmdQueryClosure << includeOutputs; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); conn->to.flush(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) out.insert(i); } @@ -374,10 +374,10 @@ public: << cmdQueryValidPaths << false // lock << maybeSubstitute; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); conn->to.flush(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } void connect() override diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index e60d7abe0..9dd4a3ae7 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -131,7 +131,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom {}); + info.references = WorkerProto::read(store, source); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -152,7 +152,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + workerProtoWrite(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 988b47473..c3dfb5979 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -100,7 +100,7 @@ void RemoteStore::initConnection(Connection & conn) } if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { - conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom> {}); + conn.remoteTrustsUs = WorkerProto>::read(*this, conn.from); } else { // We don't know the answer; protocol to old. conn.remoteTrustsUs = std::nullopt; @@ -227,12 +227,12 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -242,7 +242,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -259,9 +259,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -283,7 +283,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -296,9 +296,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); + workerProtoWrite(*this, conn->to, paths); } else - worker_proto::write(*this, conn->to, pathsMap); + workerProtoWrite(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -306,7 +306,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -349,7 +349,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) referrers.insert(i); } @@ -359,7 +359,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -371,7 +371,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -381,7 +381,7 @@ std::map> RemoteStore::queryPartialDerivat auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); + return WorkerProto>>::read(*this, conn->from); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -427,7 +427,7 @@ ref RemoteStore::addCAToStore( << wopAddToStore << name << caMethod.render(hashType); - worker_proto::write(*this, conn->to, references); + workerProtoWrite(*this, conn->to, references); conn->to << repair; // The dump source may invoke the store, so we need to make some room. @@ -452,7 +452,7 @@ ref RemoteStore::addCAToStore( name, printHashType(hashType)); std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); + workerProtoWrite(*this, conn->to, references); conn.processStderr(); }, [&](const FileIngestionMethod & fim) -> void { @@ -518,7 +518,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + workerProtoWrite(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -528,7 +528,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); + auto importedPaths = WorkerProto::read(*this, conn->from); assert(importedPaths.size() <= 1); } @@ -537,7 +537,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + workerProtoWrite(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -610,7 +610,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { - worker_proto::write(*this, conn->to, info); + workerProtoWrite(*this, conn->to, info); } conn.processStderr(); } @@ -632,14 +632,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, auto real = [&]() -> std::shared_ptr { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read( - *this, conn->from, Phantom> {}); + auto outPaths = WorkerProto>::read( + *this, conn->from); if (outPaths.empty()) return nullptr; return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); } else { - auto realisations = worker_proto::read( - *this, conn->from, Phantom> {}); + auto realisations = WorkerProto>::read( + *this, conn->from); if (realisations.empty()) return nullptr; return std::make_shared(*realisations.begin()); @@ -653,7 +653,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector & reqs) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { - worker_proto::write(store, conn->to, reqs); + workerProtoWrite(store, conn->to, reqs); } else { Strings ss; for (auto & p : reqs) { @@ -723,7 +723,7 @@ std::vector RemoteStore::buildPathsWithResults( writeDerivedPaths(*this, conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom> {}); + return WorkerProto>::read(*this, conn->from); } else { // Avoid deadlock. conn_.reset(); @@ -806,7 +806,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + auto builtOutputs = WorkerProto::read(*this, conn->from); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -865,7 +865,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); + workerProtoWrite(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -924,9 +924,9 @@ void RemoteStore::queryMissing(const std::vector & targets, conn->to << wopQueryMissing; writeDerivedPaths(*this, conn, targets); conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); + willBuild = WorkerProto::read(*this, conn->from); + willSubstitute = WorkerProto::read(*this, conn->from); + unknown = WorkerProto::read(*this, conn->from); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index cb0f3f321..51bb12026 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -9,31 +9,31 @@ #include -namespace nix::worker_proto { +namespace nix { -std::string read(const Store & store, Source & from, Phantom _) +std::string WorkerProto::read(const Store & store, Source & from) { return readString(from); } -void write(const Store & store, Sink & out, const std::string & str) +void WorkerProto::write(const Store & store, Sink & out, const std::string & str) { out << str; } -StorePath read(const Store & store, Source & from, Phantom _) +StorePath WorkerProto::read(const Store & store, Source & from) { return store.parseStorePath(readString(from)); } -void write(const Store & store, Sink & out, const StorePath & storePath) +void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) { out << store.printStorePath(storePath); } -std::optional read(const Store & store, Source & from, Phantom> _) +std::optional WorkerProto>::read(const Store & store, Source & from) { auto temp = readNum(from); switch (temp) { @@ -48,7 +48,7 @@ std::optional read(const Store & store, Source & from, Phantom & optTrusted) +void WorkerProto>::write(const Store & store, Sink & out, const std::optional & optTrusted) { if (!optTrusted) out << (uint8_t)0; @@ -67,30 +67,30 @@ void write(const Store & store, Sink & out, const std::optional & o } -ContentAddress read(const Store & store, Source & from, Phantom _) +ContentAddress WorkerProto::read(const Store & store, Source & from) { return ContentAddress::parse(readString(from)); } -void write(const Store & store, Sink & out, const ContentAddress & ca) +void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) { out << renderContentAddress(ca); } -DerivedPath read(const Store & store, Source & from, Phantom _) +DerivedPath WorkerProto::read(const Store & store, Source & from) { auto s = readString(from); return DerivedPath::parseLegacy(store, s); } -void write(const Store & store, Sink & out, const DerivedPath & req) +void WorkerProto::write(const Store & store, Sink & out, const DerivedPath & req) { out << req.to_string_legacy(store); } -Realisation read(const Store & store, Source & from, Phantom _) +Realisation WorkerProto::read(const Store & store, Source & from) { std::string rawInput = readString(from); return Realisation::fromJSON( @@ -99,41 +99,41 @@ Realisation read(const Store & store, Source & from, Phantom _) ); } -void write(const Store & store, Sink & out, const Realisation & realisation) +void WorkerProto::write(const Store & store, Sink & out, const Realisation & realisation) { out << realisation.toJSON().dump(); } -DrvOutput read(const Store & store, Source & from, Phantom _) +DrvOutput WorkerProto::read(const Store & store, Source & from) { return DrvOutput::parse(readString(from)); } -void write(const Store & store, Sink & out, const DrvOutput & drvOutput) +void WorkerProto::write(const Store & store, Sink & out, const DrvOutput & drvOutput) { out << drvOutput.to_string(); } -KeyedBuildResult read(const Store & store, Source & from, Phantom _) +KeyedBuildResult WorkerProto::read(const Store & store, Source & from) { - auto path = read(store, from, Phantom {}); - auto br = read(store, from, Phantom {}); + auto path = WorkerProto::read(store, from); + auto br = WorkerProto::read(store, from); return KeyedBuildResult { std::move(br), /* .path = */ std::move(path), }; } -void write(const Store & store, Sink & to, const KeyedBuildResult & res) +void WorkerProto::write(const Store & store, Sink & to, const KeyedBuildResult & res) { - write(store, to, res.path); - write(store, to, static_cast(res)); + workerProtoWrite(store, to, res.path); + workerProtoWrite(store, to, static_cast(res)); } -BuildResult read(const Store & store, Source & from, Phantom _) +BuildResult WorkerProto::read(const Store & store, Source & from) { BuildResult res; res.status = (BuildResult::Status) readInt(from); @@ -143,7 +143,7 @@ BuildResult read(const Store & store, Source & from, Phantom _) >> res.isNonDeterministic >> res.startTime >> res.stopTime; - auto builtOutputs = read(store, from, Phantom {}); + auto builtOutputs = WorkerProto::read(store, from); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -151,7 +151,7 @@ BuildResult read(const Store & store, Source & from, Phantom _) return res; } -void write(const Store & store, Sink & to, const BuildResult & res) +void WorkerProto::write(const Store & store, Sink & to, const BuildResult & res) { to << res.status @@ -163,28 +163,28 @@ void write(const Store & store, Sink & to, const BuildResult & res) DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - write(store, to, builtOutputs); + workerProtoWrite(store, to, builtOutputs); } -std::optional read(const Store & store, Source & from, Phantom> _) +std::optional WorkerProto>::read(const Store & store, Source & from) { auto s = readString(from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void write(const Store & store, Sink & out, const std::optional & storePathOpt) +void WorkerProto>::write(const Store & store, Sink & out, const std::optional & storePathOpt) { out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional read(const Store & store, Source & from, Phantom> _) +std::optional WorkerProto>::read(const Store & store, Source & from) { return ContentAddress::parseOpt(readString(from)); } -void write(const Store & store, Sink & out, const std::optional & caOpt) +void WorkerProto>::write(const Store & store, Sink & out, const std::optional & caOpt) { out << (caOpt ? renderContentAddress(*caOpt) : ""); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 34b2fc17b..28fd1a462 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -80,40 +80,71 @@ class Store; struct Source; /** - * Used to guide overloading + * Data type for canonical pairs of serializers for the worker protocol. * * See https://en.cppreference.com/w/cpp/language/adl for the broader * concept of what is going on here. */ template -struct Phantom {}; +struct WorkerProto { + static T read(const Store & store, Source & from); + static void write(const Store & store, Sink & out, const T & t); +}; +/** + * Wrapper function around `WorkerProto::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ +template +void workerProtoWrite(const Store & store, Sink & out, const T & t) +{ + WorkerProto::write(store, out, t); +} -namespace worker_proto { -/* FIXME maybe move more stuff inside here */ +/** + * Declare a canonical serializer pair for the worker protocol. + * + * We specialize the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define MAKE_WORKER_PROTO(T) \ + struct WorkerProto< T > { \ + static T read(const Store & store, Source & from); \ + static void write(const Store & store, Sink & out, const T & t); \ + }; -#define MAKE_WORKER_PROTO(TEMPLATE, T) \ - TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ - TEMPLATE void write(const Store & store, Sink & out, const T & str) +template<> +MAKE_WORKER_PROTO(std::string); +template<> +MAKE_WORKER_PROTO(StorePath); +template<> +MAKE_WORKER_PROTO(ContentAddress); +template<> +MAKE_WORKER_PROTO(DerivedPath); +template<> +MAKE_WORKER_PROTO(Realisation); +template<> +MAKE_WORKER_PROTO(DrvOutput); +template<> +MAKE_WORKER_PROTO(BuildResult); +template<> +MAKE_WORKER_PROTO(KeyedBuildResult); +template<> +MAKE_WORKER_PROTO(std::optional); -MAKE_WORKER_PROTO(, std::string); -MAKE_WORKER_PROTO(, StorePath); -MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, DerivedPath); -MAKE_WORKER_PROTO(, Realisation); -MAKE_WORKER_PROTO(, DrvOutput); -MAKE_WORKER_PROTO(, BuildResult); -MAKE_WORKER_PROTO(, KeyedBuildResult); -MAKE_WORKER_PROTO(, std::optional); +template +MAKE_WORKER_PROTO(std::vector); +template +MAKE_WORKER_PROTO(std::set); -MAKE_WORKER_PROTO(template, std::vector); -MAKE_WORKER_PROTO(template, std::set); - -#define X_ template -#define Y_ std::map -MAKE_WORKER_PROTO(X_, Y_); +template +#define X_ std::map +MAKE_WORKER_PROTO(X_); #undef X_ -#undef Y_ /** * These use the empty string for the null case, relying on the fact @@ -129,72 +160,72 @@ MAKE_WORKER_PROTO(X_, Y_); * worker protocol harder to implement in other languages where such * specializations may not be allowed. */ -MAKE_WORKER_PROTO(, std::optional); -MAKE_WORKER_PROTO(, std::optional); +template<> +MAKE_WORKER_PROTO(std::optional); +template<> +MAKE_WORKER_PROTO(std::optional); template -std::vector read(const Store & store, Source & from, Phantom> _) +std::vector WorkerProto>::read(const Store & store, Source & from) { std::vector resSet; auto size = readNum(from); while (size--) { - resSet.push_back(read(store, from, Phantom {})); + resSet.push_back(WorkerProto::read(store, from)); } return resSet; } template -void write(const Store & store, Sink & out, const std::vector & resSet) +void WorkerProto>::write(const Store & store, Sink & out, const std::vector & resSet) { out << resSet.size(); for (auto & key : resSet) { - write(store, out, key); + WorkerProto::write(store, out, key); } } template -std::set read(const Store & store, Source & from, Phantom> _) +std::set WorkerProto>::read(const Store & store, Source & from) { std::set resSet; auto size = readNum(from); while (size--) { - resSet.insert(read(store, from, Phantom {})); + resSet.insert(WorkerProto::read(store, from)); } return resSet; } template -void write(const Store & store, Sink & out, const std::set & resSet) +void WorkerProto>::write(const Store & store, Sink & out, const std::set & resSet) { out << resSet.size(); for (auto & key : resSet) { - write(store, out, key); + WorkerProto::write(store, out, key); } } template -std::map read(const Store & store, Source & from, Phantom> _) +std::map WorkerProto>::read(const Store & store, Source & from) { std::map resMap; auto size = readNum(from); while (size--) { - auto k = read(store, from, Phantom {}); - auto v = read(store, from, Phantom {}); + auto k = WorkerProto::read(store, from); + auto v = WorkerProto::read(store, from); resMap.insert_or_assign(std::move(k), std::move(v)); } return resMap; } template -void write(const Store & store, Sink & out, const std::map & resMap) +void WorkerProto>::write(const Store & store, Sink & out, const std::map & resMap) { out << resMap.size(); for (auto & i : resMap) { - write(store, out, i.first); - write(store, out, i.second); + WorkerProto::write(store, out, i.first); + WorkerProto::write(store, out, i.second); } } } - -} diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 40f30eb63..61c189efb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -849,7 +849,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = worker_proto::read(*store, in, Phantom {}); + auto paths = WorkerProto::read(*store, in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -858,19 +858,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - worker_proto::write(*store, out, store->queryValidPaths(paths)); + workerProtoWrite(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - auto paths = worker_proto::read(*store, in, Phantom {}); + auto paths = WorkerProto::read(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + workerProtoWrite(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(worker_proto::read(*store, in, Phantom {}), out); + store->exportPaths(WorkerProto::read(*store, in), out); break; } @@ -944,7 +944,7 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - worker_proto::write(*store, out, builtOutputs); + workerProtoWrite(*store, out, builtOutputs); } break; @@ -953,9 +953,9 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(worker_proto::read(*store, in, Phantom {}), + store->computeFSClosure(WorkerProto::read(*store, in), closure, false, includeOutputs); - worker_proto::write(*store, out, closure); + workerProtoWrite(*store, out, closure); break; } @@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom {}); + info.references = WorkerProto::read(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in));