From 1f3ff0d193c270f7b97af4aa3e463be01dbe5f2d Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 26 May 2021 16:09:02 +0200 Subject: [PATCH] Aso track the output path of the realisation dependencies --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/local-store.cc | 34 +++++++++++++++------------ src/libstore/misc.cc | 17 ++++++-------- src/libstore/realisation.cc | 20 ++++++++-------- src/libstore/realisation.hh | 8 ++++++- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 +- 7 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 93fc54440..8c9ef0101 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -927,7 +927,7 @@ void DerivationGoal::resolvedFinished() { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; newRealisation.signatures.clear(); - newRealisation.drvOutputDeps = drvOutputReferences(worker.store, *drv, realisation->outPath); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } else { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ce0e94865..debe70037 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -711,19 +711,19 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { settings.requireExperimentalFeature("ca-derivations"); - auto state(_state.lock()); retrySQLite([&]() { + auto state(_state.lock()); state->stmts->RegisterRealisedOutput .use()(info.id.strHash())(info.id.outputName)(printStorePath( info.outPath))(concatStringsSep(" ", info.signatures)) .exec(); + uint64_t myId = state->db.getLastInsertedRowId(); + for (auto & [outputId, _] : info.dependentRealisations) { + state->stmts->AddRealisationRealisationReference + .use()(myId)(outputId.strHash())(outputId.outputName) + .exec(); + } }); - uint64_t myId = state->db.getLastInsertedRowId(); - for (auto& outputId : info.drvOutputDeps) { - state->stmts->AddRealisationRealisationReference - .use()(myId)(outputId.strHash())(outputId.outputName) - .exec(); - } } void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) @@ -1730,22 +1730,26 @@ std::optional LocalStore::queryRealisation( auto signatures = tokenizeString(useQueryRealisedOutput.getStr(2)); - std::set drvOutputDeps; + std::map dependentRealisations; auto useRealisationRefs( state->stmts->QueryRealisationRealisationReferences.use()( realisationDbId)); - while (useRealisationRefs.next()) - drvOutputDeps.insert(DrvOutput{ - Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), - useRealisationRefs.getStr(1), - } - ); + while (useRealisationRefs.next()) { + auto depHash = useRealisationRefs.getStr(0); + auto depOutputName = useRealisationRefs.getStr(1); + auto useQueryRealisedOutput( + state->stmts->QueryRealisedOutput.use()(depHash)(depOutputName)); + assert(useQueryRealisedOutput.next()); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto depId = DrvOutput { Hash::parseAnyPrefixed(depHash), depOutputName }; + dependentRealisations.insert({depId, outputPath}); + } return Ret{Realisation{ .id = id, .outPath = outputPath, .signatures = signatures, - .drvOutputDeps = drvOutputDeps, + .dependentRealisations = dependentRealisations, }}; }); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 8793f9f4a..80ee15c49 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -254,25 +254,22 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } -std::set drvOutputReferences( +std::map drvOutputReferences( const std::set inputRealisations, const StorePathSet pathReferences) { - std::set res; + std::map res; - std::map> inputsByOutputPath; - for (const auto & input : inputRealisations) - inputsByOutputPath[input.outPath].insert(input.id); - - for (const auto & path : pathReferences) { - auto theseInputs = inputsByOutputPath[path]; - res.insert(theseInputs.begin(), theseInputs.end()); + for (const auto & input : inputRealisations) { + if (pathReferences.count(input.outPath)) { + res.insert({input.id, input.outPath}); + } } return res; } -std::set drvOutputReferences( +std::map drvOutputReferences( Store & store, const Derivation & drv, const StorePath & outputPath) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index fab10f68c..d2d306476 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -33,7 +33,7 @@ void Realisation::closure(Store & store, std::set startOutputs, std { auto getDeps = [&](const Realisation& current) -> std::set { std::set res; - for (auto& currentDep : current.drvOutputDeps) { + for (auto& [currentDep, _] : current.dependentRealisations) { if (auto currentRealisation = store.queryRealisation(currentDep)) res.insert(*currentRealisation); else @@ -60,14 +60,14 @@ void Realisation::closure(Store & store, std::set startOutputs, std } nlohmann::json Realisation::toJSON() const { - nlohmann::json jsonDrvOutputDeps; - for (auto & dep : drvOutputDeps) - jsonDrvOutputDeps.push_back(dep.to_string()); + auto jsonDependentRealisations = nlohmann::json::object(); + for (auto & [depId, depOutPath] : dependentRealisations) + jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string()); return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, {"signatures", signatures}, - {"drvOutputDeps", jsonDrvOutputDeps}, + {"dependentRealisations", jsonDependentRealisations}, }; } @@ -93,16 +93,16 @@ Realisation Realisation::fromJSON( if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); - std::set drvOutputDeps; - if (auto jsonDependencies = json.find("drvOutputDeps"); jsonDependencies != json.end()) - for (auto & jsonDep : *jsonDependencies) - drvOutputDeps.insert(DrvOutput::parse(jsonDep.get())); + std::map dependentRealisations; + if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) + for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get>()) + dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)}); return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), .signatures = signatures, - .drvOutputDeps = drvOutputDeps, + .dependentRealisations = dependentRealisations, }; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 776ab606c..8cda5a752 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -28,7 +28,13 @@ struct Realisation { StringSet signatures; - std::set drvOutputDeps; + /** + * The realisations that are required for the current one to be valid. + * + * When importing this realisation, the store will first check that all its + * dependencies exist, and map to the correct output path + */ + std::map dependentRealisations; nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 08573f709..6f29c430a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -798,7 +798,7 @@ std::map copyPaths(ref srcStore, ref dstStor pool, Realisation::closure(*srcStore, toplevelRealisations), [&](const Realisation& current) -> std::set { std::set children; - for (const auto& drvOutput : current.drvOutputDeps) { + for (const auto& [drvOutput, _] : current.dependentRealisations) { auto currentChild = srcStore->queryRealisation(drvOutput); if (!currentChild) throw Error( diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 29af0f495..9a0027641 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -864,7 +864,7 @@ std::pair splitUriAndParams(const std::string & uri) std::optional getDerivationCA(const BasicDerivation & drv); -std::set drvOutputReferences( +std::map drvOutputReferences( Store & store, const Derivation & drv, const StorePath & outputPath);