Merge pull request #6229 from obsidiansystems/refactor-hash-modulo

Overhaul derivation hash modulo somewhat
This commit is contained in:
Théophane Hufschmitt 2022-03-15 21:23:44 +01:00 committed by GitHub
commit 516a7ac4de
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 131 additions and 72 deletions

View file

@ -1207,32 +1207,37 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
// hash per output. // hash per output.
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
std::visit(overloaded { std::visit(overloaded {
[&](Hash & h) { [&](const DrvHash & drvHash) {
for (auto & i : outputs) { auto & h = drvHash.hash;
auto outPath = state.store->makeOutputPath(i, h, drvName); switch (drvHash.kind) {
drv.env[i] = state.store->printStorePath(outPath); case DrvHash::Kind::Deferred:
drv.outputs.insert_or_assign(i, for (auto & i : outputs) {
DerivationOutput { drv.outputs.insert_or_assign(i,
.output = DerivationOutputInputAddressed { DerivationOutput {
.path = std::move(outPath), .output = DerivationOutputDeferred{},
}, });
}); }
break;
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath),
},
});
}
break;
} }
}, },
[&](CaOutputHashes &) { [&](const CaOutputHashes &) {
// Shouldn't happen as the toplevel derivation is not CA. // Shouldn't happen as the toplevel derivation is not CA.
assert(false); assert(false);
}, },
[&](DeferredHash &) {
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputDeferred{},
});
}
},
}, },
hashModulo); hashModulo.raw());
} }

View file

@ -510,7 +510,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
*/ */
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{ {
bool isDeferred = false; auto kind = DrvHash::Kind::Regular;
/* Return a fixed hash for fixed-output derivations. */ /* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) { switch (drv.type()) {
case DerivationType::CAFixed: { case DerivationType::CAFixed: {
@ -526,7 +526,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
return outputHashes; return outputHashes;
} }
case DerivationType::CAFloating: case DerivationType::CAFloating:
isDeferred = true; kind = DrvHash::Kind::Deferred;
break; break;
case DerivationType::InputAddressed: case DerivationType::InputAddressed:
break; break;
@ -537,21 +537,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
/* For other derivations, replace the inputs paths with recursive /* For other derivations, replace the inputs paths with recursive
calls to this function. */ calls to this function. */
std::map<std::string, StringSet> inputs2; std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) { for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
const auto & res = pathDerivationModulo(store, i.first); // Avoid lambda capture restriction with standard / Clang
auto & inputOutputs = inputOutputs0;
const auto & res = pathDerivationModulo(store, drvPath);
std::visit(overloaded { std::visit(overloaded {
// Regular non-CA derivation, replace derivation // Regular non-CA derivation, replace derivation
[&](const Hash & drvHash) { [&](const DrvHash & drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); kind |= drvHash.kind;
}, inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
[&](const DeferredHash & deferredHash) {
isDeferred = true;
inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
}, },
// CA derivation's output hashes // CA derivation's output hashes
[&](const CaOutputHashes & outputHashes) { [&](const CaOutputHashes & outputHashes) {
std::set<std::string> justOut = { "out" }; std::set<std::string> justOut = { "out" };
for (auto & output : i.second) { for (auto & output : inputOutputs) {
/* Put each one in with a single "out" output.. */ /* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output); const auto h = outputHashes.at(output);
inputs2.insert_or_assign( inputs2.insert_or_assign(
@ -559,15 +558,24 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
justOut); justOut);
} }
}, },
}, res); }, res.raw());
} }
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
if (isDeferred) return DrvHash { .hash = hash, .kind = kind };
return DeferredHash { hash }; }
else
return hash;
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
{
switch (other) {
case DrvHash::Kind::Regular:
break;
case DrvHash::Kind::Deferred:
self = other;
break;
}
} }
@ -575,20 +583,15 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
{ {
std::map<std::string, Hash> res; std::map<std::string, Hash> res;
std::visit(overloaded { std::visit(overloaded {
[&](const Hash & drvHash) { [&](const DrvHash & drvHash) {
for (auto & outputName : drv.outputNames()) { for (auto & outputName : drv.outputNames()) {
res.insert({outputName, drvHash}); res.insert({outputName, drvHash.hash});
}
},
[&](const DeferredHash & deferredHash) {
for (auto & outputName : drv.outputNames()) {
res.insert({outputName, deferredHash.hash});
} }
}, },
[&](const CaOutputHashes & outputHashes) { [&](const CaOutputHashes & outputHashes) {
res = outputHashes; res = outputHashes;
}, },
}, hashDerivationModulo(store, drv, true)); }, hashDerivationModulo(store, drv, true).raw());
return res; return res;
} }
@ -738,7 +741,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) { for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutputDeferred>(output.output)) { if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
Hash h = std::get<Hash>(hashModulo); auto & h = hashModulo.requireNoFixedNonDeferred();
auto outPath = store.makeOutputPath(outputName, h, drv.name); auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath); drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput { output = DerivationOutput {
@ -751,31 +754,51 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
} }
const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
auto * drvHashOpt = std::get_if<DrvHash>(&raw());
assert(drvHashOpt);
assert(drvHashOpt->kind == DrvHash::Kind::Regular);
return drvHashOpt->hash;
}
static bool tryResolveInput(
Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
const StorePath & inputDrv, const StringSet & inputOutputs)
{
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
auto getOutput = [&](const std::string & outputName) {
auto & actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt)
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(inputDrv)
);
return actualPathOpt;
};
for (auto & outputName : inputOutputs) {
auto actualPathOpt = getOutput(outputName);
if (!actualPathOpt) return false;
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, inputDrv, outputName),
store.printStorePath(actualPath));
inputSrcs.insert(std::move(actualPath));
}
return true;
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this }; BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation // Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites; StringMap inputRewrites;
for (auto & input : inputDrvs) { for (auto & [inputDrv, inputOutputs] : inputDrvs)
auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
StringSet newOutputNames; return std::nullopt;
for (auto & outputName : input.second) {
auto actualPathOpt = inputDrvOutputs.at(outputName);
if (!actualPathOpt) {
warn("output %s of input %s missing, aborting the resolving",
outputName,
store.printStorePath(input.first)
);
return std::nullopt;
}
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, input.first, outputName),
store.printStorePath(actualPath));
resolved.inputSrcs.insert(std::move(actualPath));
}
}
rewriteDerivation(store, resolved, inputRewrites); rewriteDerivation(store, resolved, inputRewrites);

View file

@ -175,13 +175,43 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front. // whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes; typedef std::map<std::string, Hash> CaOutputHashes;
struct DeferredHash { Hash hash; }; struct DrvHash {
Hash hash;
enum struct Kind {
// Statically determined derivations.
// This hash will be directly used to compute the output paths
Regular,
// Floating-output derivations (and their dependencies).
Deferred,
};
Kind kind;
};
void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
typedef std::variant< typedef std::variant<
Hash, // regular DRV normalized hash // Regular normalized derivation hash, and whether it was deferred (because
CaOutputHashes, // Fixed-output derivation hashes // an ancestor derivation is a floating content addressed derivation).
DeferredHash // Deferred hashes for floating outputs drvs and their dependencies DrvHash,
> DrvHashModulo; // Fixed-output derivation hashes
CaOutputHashes
> DrvHashModuloRaw;
struct DrvHashModulo : DrvHashModuloRaw {
using Raw = DrvHashModuloRaw;
using Raw::Raw;
/* Get hash, throwing if it is per-output CA hashes or a
deferred Drv hash.
*/
const Hash & requireNoFixedNonDeferred() const;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
};
/* Returns hashes with the details of fixed-output subderivations /* Returns hashes with the details of fixed-output subderivations
expunged. expunged.

View file

@ -701,8 +701,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](const DerivationOutputInputAddressed & doia) { [&](const DerivationOutputInputAddressed & doia) {
if (!h) { if (!h) {
// somewhat expensive so we do lazily // somewhat expensive so we do lazily
auto temp = hashDerivationModulo(*this, drv, true); auto h0 = hashDerivationModulo(*this, drv, true);
h = std::get<Hash>(temp); h = h0.requireNoFixedNonDeferred();
} }
StorePath recomputed = makeOutputPath(i.first, *h, drvName); StorePath recomputed = makeOutputPath(i.first, *h, drvName);
if (doia.path != recomputed) if (doia.path != recomputed)

View file

@ -206,7 +206,8 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
drv.env[output.first] = ""; drv.env[output.first] = "";
} }
Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); auto h0 = hashDerivationModulo(*evalStore, drv, true);
const Hash & h = h0.requireNoFixedNonDeferred();
for (auto & output : drv.outputs) { for (auto & output : drv.outputs) {
auto outPath = store->makeOutputPath(output.first, h, drv.name); auto outPath = store->makeOutputPath(output.first, h, drv.name);