Allow substituting from different storeDir

Substituters can substitute from one store dir to another with a
little bit of help. The store api just needs to have a CA so it can
recompute the store path based on the new store dir. We can only do
this for fixed output derivations with no references, though.
This commit is contained in:
Matthew Bauer 2020-06-12 09:49:09 -05:00
parent d558fb98f6
commit 79c169d1c6
7 changed files with 50 additions and 9 deletions

View file

@ -855,16 +855,24 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
{ {
if (!settings.useSubstitutes) return; if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) { for (auto & sub : getDefaultSubstituters()) {
if (sub->storeDir != storeDir) continue; for (auto & path_ : paths) {
for (auto & path : paths) { auto path(path_.clone());
if (infos.count(path)) continue;
debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
try { try {
auto info = sub->queryPathInfo(path); auto info = sub->queryPathInfo(path);
auto ca = pathsCA.find(printStorePath(path));
if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) {
if (!hasPrefix(ca->second, "fixed:"))
continue;
// recompute store path so that we can use a fixed output ca
path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name());
} else continue;
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info)); std::shared_ptr<const ValidPathInfo>(info));
infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ infos.insert_or_assign(path.clone(), SubstitutablePathInfo{

View file

@ -142,7 +142,8 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathSet & paths, void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos,
std::map<std::string, std::string> pathsCA = {}) override;
void addToStore(const ValidPathInfo & info, Source & source, void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,

View file

@ -108,6 +108,27 @@ void Store::computeFSClosure(const StorePath & startPath,
} }
static std::optional<std::string> getDerivationCA(ref<Derivation> drv)
{
auto outputHashMode = drv->env.find("outputHashMode");
auto outputHashAlgo = drv->env.find("outputHashAlgo");
auto outputHash = drv->env.find("outputHash");
if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) {
auto ht = parseHashType(outputHashAlgo->second);
auto h = Hash(outputHash->second, ht);
FileIngestionMethod ingestionMethod;
if (outputHashMode->second == "recursive")
ingestionMethod = FileIngestionMethod::Recursive;
else if (outputHashMode->second == "flat")
ingestionMethod = FileIngestionMethod::Flat;
else
throw Error("unknown ingestion method: '%s'", outputHashMode->second);
return makeFixedOutputCA(ingestionMethod, h);
}
return std::nullopt;
}
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
unsigned long long & downloadSize_, unsigned long long & narSize_) unsigned long long & downloadSize_, unsigned long long & narSize_)
@ -159,7 +180,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
SubstitutablePathInfos infos; SubstitutablePathInfos infos;
StorePathSet paths; // FIXME StorePathSet paths; // FIXME
paths.insert(outPath.clone()); paths.insert(outPath.clone());
querySubstitutablePathInfos(paths, infos);
std::map<std::string, std::string> pathsCA = {};
if (auto ca = getDerivationCA(drv))
pathsCA.insert({outPathS, *ca});
querySubstitutablePathInfos(paths, infos, pathsCA);
if (infos.empty()) { if (infos.empty()) {
drvState_->lock()->done = true; drvState_->lock()->done = true;

View file

@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
{ {
if (paths.empty()) return; if (paths.empty()) return;

View file

@ -58,7 +58,7 @@ public:
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
void querySubstitutablePathInfos(const StorePathSet & paths, void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) override; SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) override;
void addToStore(const ValidPathInfo & info, Source & nar, void addToStore(const ValidPathInfo & info, Source & nar,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,

View file

@ -580,6 +580,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0; uint64_t total = 0;
// recompute store path on the chance dstStore does it differently
if (hasPrefix(info->ca, "fixed:") && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info);
info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name());
info = info2;
}
if (!info->narHash) { if (!info->narHash) {
StringSink sink; StringSink sink;
srcStore->narFromPath({storePath}, sink); srcStore->narFromPath({storePath}, sink);

View file

@ -445,7 +445,7 @@ public:
sizes) of a set of paths. If a path does not have substitute sizes) of a set of paths. If a path does not have substitute
info, it's omitted from the resulting infos map. */ info, it's omitted from the resulting infos map. */
virtual void querySubstitutablePathInfos(const StorePathSet & paths, virtual void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) { return; }; SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) { return; };
/* Import a path into the store. */ /* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource, virtual void addToStore(const ValidPathInfo & info, Source & narSource,