lix-project/lix
20
57
Fork 31
Code Issues 347 Code Review (Gerrit) Projects 3 Releases Wiki Activity

libstore: move Store::queryMissing into a fresh struct

Browse source 
we want to get rid of all these local lambdas, they make coroutines impossible.

Change-Id: I94336f0844e9cdb0aa27788d0ee5f99925f24e0c
This commit is contained in:
eldritch horrors 2024-11-17 14:43:57 +01:00
parent 9cf91b7385
commit 74d820e5e8
1 changed files with 66 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

101
lix/libstore/misc.cc
View file

@ -7,6 +7,7 @@
#include "lix/libutil/closure.hh" #include "lix/libutil/closure.hh"
#include "lix/libstore/filetransfer.hh" #include "lix/libstore/filetransfer.hh"
#include "lix/libutil/strings.hh" #include "lix/libutil/strings.hh"
#include <kj/common.h>
namespace nix { namespace nix {
@ -77,16 +78,10 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv)
return nullptr; return nullptr;
} }
void Store::queryMissing(const std::vector<DerivedPath> & targets, namespace {
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, struct QueryMissingContext
uint64_t & downloadSize_, uint64_t & narSize_)
{ {
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); Store & store;
downloadSize_ = narSize_ = 0;
// FIXME: make async.
ThreadPool pool("queryMissing pool", fileTransferSettings.httpConnections);
struct State struct State
{ {
@ -104,22 +99,40 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
DrvState(size_t left) : left(left) { } DrvState(size_t left) : left(left) { }
}; };
Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); Sync<State> state_;
std::function<void(DerivedPath)> doPath; // FIXME: make async.
ThreadPool pool{"queryMissing pool", fileTransferSettings.httpConnections};
std::function<void(ref<SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> enqueueDerivedPaths; explicit QueryMissingContext(
Store & store,
StorePathSet & willBuild_,
StorePathSet & willSubstitute_,
StorePathSet & unknown_,
uint64_t & downloadSize_,
uint64_t & narSize_
)
: store(store)
, state_{{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}}
{
}
enqueueDerivedPaths = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) { KJ_DISALLOW_COPY_AND_MOVE(QueryMissingContext);
void queryMissing(const std::vector<DerivedPath> & targets);
void enqueueDerivedPaths(ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode)
{
if (!inputNode.value.empty()) if (!inputNode.value.empty())
pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value })); pool.enqueue([this, path{DerivedPath::Built { inputDrv, inputNode.value }}] { doPath(path); });
for (const auto & [outputName, childNode] : inputNode.childMap) for (const auto & [outputName, childNode] : inputNode.childMap)
enqueueDerivedPaths( enqueueDerivedPaths(
make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }), make_ref<SingleDerivedPath>(SingleDerivedPath::Built { inputDrv, outputName }),
childNode); childNode);
}; }
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { void mustBuildDrv(const StorePath & drvPath, const Derivation & drv)
{
{ {
auto state(state_.lock()); auto state(state_.lock());
state->willBuild.insert(drvPath); state->willBuild.insert(drvPath);
@ -128,16 +141,16 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) { for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) {
enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode); enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode);
} }
}; }
auto checkOutput = [&]( void checkOutput(
const StorePath & drvPath, ref<Derivation> drv, const StorePath & outPath, ref<Sync<DrvState>> drvState_) const StorePath & drvPath, ref<Derivation> drv, const StorePath & outPath, ref<Sync<DrvState>> drvState_)
{ {
if (drvState_->lock()->done) return; if (drvState_->lock()->done) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
auto * cap = getDerivationCA(*drv); auto * cap = getDerivationCA(*drv);
querySubstitutablePathInfos({ store.querySubstitutablePathInfos({
{ {
outPath, outPath,
cap ? std::optional { *cap } : std::nullopt, cap ? std::optional { *cap } : std::nullopt,
@ -156,17 +169,17 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
drvState->outPaths.insert(outPath); drvState->outPaths.insert(outPath);
if (!drvState->left) { if (!drvState->left) {
for (auto & path : drvState->outPaths) for (auto & path : drvState->outPaths)
pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } )); pool.enqueue([this, path{DerivedPath::Opaque { path }}] { doPath(path); });
}
} }
} }
} }
};
doPath = [&](const DerivedPath & req) {
void doPath(const DerivedPath & req)
{
{ {
auto state(state_.lock()); auto state(state_.lock());
if (!state->done.insert(req.to_string(*this)).second) return; if (!state->done.insert(req.to_string(store)).second) return;
} }
std::visit(overloaded { std::visit(overloaded {
@ -174,12 +187,12 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
auto drvPathP = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath); auto drvPathP = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath);
if (!drvPathP) { if (!drvPathP) {
// TODO make work in this case. // TODO make work in this case.
warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(store));
return; return;
} }
auto & drvPath = drvPathP->path; auto & drvPath = drvPathP->path;
if (!isValidPath(drvPath)) { if (!store.isValidPath(drvPath)) {
// FIXME: we could try to substitute the derivation. // FIXME: we could try to substitute the derivation.
auto state(state_.lock()); auto state(state_.lock());
state->unknown.insert(drvPath); state->unknown.insert(drvPath);
@ -190,17 +203,17 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
/* true for regular derivations, and CA derivations for which we /* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */ have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true; auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { for (auto & [outputName, pathOpt] : store.queryPartialDerivationOutputMap(drvPath)) {
if (!pathOpt) { if (!pathOpt) {
knownOutputPaths = false; knownOutputPaths = false;
break; break;
} }
if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) if (bfd.outputs.contains(outputName) && !store.isValidPath(*pathOpt))
invalid.insert(*pathOpt); invalid.insert(*pathOpt);
} }
if (knownOutputPaths && invalid.empty()) return; if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(drvPath)); auto drv = make_ref<Derivation>(store.derivationFromPath(drvPath));
ParsedDerivation parsedDrv(StorePath(drvPath), *drv); ParsedDerivation parsedDrv(StorePath(drvPath), *drv);
if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
@ -208,7 +221,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
// If there are unknown output paths, attempt to find if the // If there are unknown output paths, attempt to find if the
// paths are known to substituters through a realisation. // paths are known to substituters through a realisation.
auto outputHashes = staticOutputHashes(*this, *drv); auto outputHashes = staticOutputHashes(store, *drv);
knownOutputPaths = true; knownOutputPaths = true;
for (auto [outputName, hash] : outputHashes) { for (auto [outputName, hash] : outputHashes) {
@ -221,7 +234,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
if (!realisation) if (!realisation)
continue; continue;
found = true; found = true;
if (!isValidPath(realisation->outPath)) if (!store.isValidPath(realisation->outPath))
invalid.insert(realisation->outPath); invalid.insert(realisation->outPath);
break; break;
} }
@ -236,17 +249,17 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid) for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); pool.enqueue([=, this] { checkOutput(drvPath, drv, output, drvState); });
} else } else
mustBuildDrv(drvPath, *drv); mustBuildDrv(drvPath, *drv);
}, },
[&](const DerivedPath::Opaque & bo) { [&](const DerivedPath::Opaque & bo) {
if (isValidPath(bo.path)) return; if (store.isValidPath(bo.path)) return;
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos); store.querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos);
if (infos.empty()) { if (infos.empty()) {
auto state(state_.lock()); auto state(state_.lock());
@ -265,17 +278,33 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
} }
for (auto & ref : info->second.references) for (auto & ref : info->second.references)
pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); pool.enqueue([this, path{DerivedPath::Opaque { ref }}] { doPath(path); });
}, },
}, req.raw()); }, req.raw());
}
}; };
}
void QueryMissingContext::queryMissing(const std::vector<DerivedPath> & targets)
{
for (auto & path : targets) for (auto & path : targets)
pool.enqueue(std::bind(doPath, path)); pool.enqueue([=, this] { doPath(path); });
pool.process(); pool.process();
} }
void Store::queryMissing(const std::vector<DerivedPath> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
uint64_t & downloadSize_, uint64_t & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
downloadSize_ = narSize_ = 0;
QueryMissingContext{*this, willBuild_, willSubstitute_, unknown_, downloadSize_, narSize_}
.queryMissing(targets);
}
StorePaths Store::topoSortPaths(const StorePathSet & paths) StorePaths Store::topoSortPaths(const StorePathSet & paths)
{ {