queue runner: introduce some parallelism for remote paths lookup

Each output for a given step being ingested is looked up in parallel,
which should basically multiply the speed of builds ingestion by the
average number of outputs per derivation.
This commit is contained in:
Pierre Bourdon 2024-04-20 22:18:13 +02:00
parent f31b95d371
commit 6606a7f86e
Signed by: delroth
GPG key ID: 6FB80DCD84DA0F1C
2 changed files with 40 additions and 6 deletions

View file

@ -1,6 +1,7 @@
#include "state.hh" #include "state.hh"
#include "hydra-build-result.hh" #include "hydra-build-result.hh"
#include "globals.hh" #include "globals.hh"
#include "thread-pool.hh"
#include <cstring> #include <cstring>
@ -403,6 +404,34 @@ void State::processQueueChange(Connection & conn)
} }
std::map<DrvOutput, std::optional<StorePath>> State::getMissingRemotePaths(
ref<Store> destStore,
const std::map<DrvOutput, std::optional<StorePath>> & paths)
{
Sync<std::map<DrvOutput, std::optional<StorePath>>> missing_;
ThreadPool tp;
for (auto & [output, maybeOutputPath] : paths) {
if (!maybeOutputPath) {
auto missing(missing_.lock());
missing->insert({output, maybeOutputPath});
} else {
tp.enqueue([&] {
if (!destStore->isValidPath(*maybeOutputPath)) {
auto missing(missing_.lock());
missing->insert({output, maybeOutputPath});
}
});
}
}
tp.process();
auto missing(missing_.lock());
return *missing;
}
Step::ptr State::createStep(ref<Store> destStore, Step::ptr State::createStep(ref<Store> destStore,
Connection & conn, Build::ptr build, const StorePath & drvPath, Connection & conn, Build::ptr build, const StorePath & drvPath,
Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs, Build::ptr referringBuild, Step::ptr referringStep, std::set<StorePath> & finishedDrvs,
@ -483,16 +512,15 @@ Step::ptr State::createStep(ref<Store> destStore,
/* Are all outputs valid? */ /* Are all outputs valid? */
auto outputHashes = staticOutputHashes(*localStore, *(step->drv)); auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
bool valid = true; std::map<DrvOutput, std::optional<StorePath>> paths;
std::map<DrvOutput, std::optional<StorePath>> missing;
for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) { for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
auto outputHash = outputHashes.at(outputName); auto outputHash = outputHashes.at(outputName);
if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath)) paths.insert({{outputHash, outputName}, maybeOutputPath});
continue;
valid = false;
missing.insert({{outputHash, outputName}, maybeOutputPath});
} }
auto missing = getMissingRemotePaths(destStore, paths);
bool valid = missing.empty();
/* Try to copy the missing paths from the local store or from /* Try to copy the missing paths from the local store or from
substitutes. */ substitutes. */
if (!missing.empty()) { if (!missing.empty()) {

View file

@ -553,6 +553,12 @@ private:
BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore, BuildOutput getBuildOutputCached(Connection & conn, nix::ref<nix::Store> destStore,
const nix::StorePath & drvPath); const nix::StorePath & drvPath);
/* Returns paths missing from the remote store. Paths are processed in
* parallel to work around the possible latency of remote stores. */
std::map<nix::DrvOutput, std::optional<nix::StorePath>> getMissingRemotePaths(
nix::ref<nix::Store> destStore,
const std::map<nix::DrvOutput, std::optional<nix::StorePath>> & paths);
Step::ptr createStep(nix::ref<nix::Store> store, Step::ptr createStep(nix::ref<nix::Store> store,
Connection & conn, Build::ptr build, const nix::StorePath & drvPath, Connection & conn, Build::ptr build, const nix::StorePath & drvPath,
Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs, Build::ptr referringBuild, Step::ptr referringStep, std::set<nix::StorePath> & finishedDrvs,