From 79c169d1c615211af69c1cbc6218f7465a4f81ed Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 09:49:09 -0500 Subject: [PATCH] Allow substituting from different storeDir Substituters can substitute from one store dir to another with a little bit of help. The store api just needs to have a CA so it can recompute the store path based on the new store dir. We can only do this for fixed output derivations with no references, though. --- src/libstore/local-store.cc | 16 ++++++++++++---- src/libstore/local-store.hh | 3 ++- src/libstore/misc.cc | 27 ++++++++++++++++++++++++++- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 7 +++++++ src/libstore/store-api.hh | 2 +- 7 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..6385453cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -855,16 +855,24 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; - for (auto & path : paths) { - if (infos.count(path)) continue; + for (auto & path_ : paths) { + auto path(path_.clone()); debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { auto info = sub->queryPathInfo(path); + + auto ca = pathsCA.find(printStorePath(path)); + if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) { + if (!hasPrefix(ca->second, "fixed:")) + continue; + // recompute store path so that we can use a fixed output ca + path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name()); + } else continue; + auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c..5a976e3e1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -142,7 +142,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, + std::map pathsCA = {}) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..1b0a055d3 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,6 +108,27 @@ void Store::computeFSClosure(const StorePath & startPath, } +static std::optional getDerivationCA(ref drv) +{ + auto outputHashMode = drv->env.find("outputHashMode"); + auto outputHashAlgo = drv->env.find("outputHashAlgo"); + auto outputHash = drv->env.find("outputHash"); + if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) { + auto ht = parseHashType(outputHashAlgo->second); + auto h = Hash(outputHash->second, ht); + FileIngestionMethod ingestionMethod; + if (outputHashMode->second == "recursive") + ingestionMethod = FileIngestionMethod::Recursive; + else if (outputHashMode->second == "flat") + ingestionMethod = FileIngestionMethod::Flat; + else + throw Error("unknown ingestion method: '%s'", outputHashMode->second); + return makeFixedOutputCA(ingestionMethod, h); + } + + return std::nullopt; +} + void Store::queryMissing(const std::vector & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) @@ -159,7 +180,11 @@ void Store::queryMissing(const std::vector & targets, SubstitutablePathInfos infos; StorePathSet paths; // FIXME paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + + std::map pathsCA = {}; + if (auto ca = getDerivationCA(drv)) + pathsCA.insert({outPathS, *ca}); + querySubstitutablePathInfos(paths, infos, pathsCA); if (infos.empty()) { drvState_->lock()->done = true; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..956980f49 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (paths.empty()) return; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524..867d650bc 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -58,7 +58,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, std::map pathsCA = {}) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..40000da01 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -580,6 +580,13 @@ void copyStorePath(ref srcStore, ref dstStore, uint64_t total = 0; + // recompute store path on the chance dstStore does it differently + if (hasPrefix(info->ca, "fixed:") && info->references.empty()) { + auto info2 = make_ref(*info); + info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name()); + info = info2; + } + if (!info->narHash) { StringSink sink; srcStore->narFromPath({storePath}, sink); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d..6f961329c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -445,7 +445,7 @@ public: sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos, std::map pathsCA = {}) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource,