lix-project/lix-releng-staging
0
0
Fork 0
Code Packages Activity

Merge pull request #4378 from NixOS/fix-master-build

Browse source 
Fix the detection of already built drv outputs
This commit is contained in:
Eelco Dolstra 2020-12-17 14:12:28 +01:00 committed by GitHub
parent ae3c3e3bb2 4d45839499
commit 26e189c3f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 78 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
src/libstore/derivations.cc
View file

@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
std::optional<BasicDerivation> Derivation::tryResolveUncached(Store & store) {
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
@ -771,4 +771,34 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
return resolved;
}
std::optional<BasicDerivation> Derivation::tryResolve(Store& store)
{
auto drvPath = writeDerivation(store, *this, NoRepair, false);
return Derivation::tryResolve(store, drvPath);
}
std::optional<BasicDerivation> Derivation::tryResolve(Store& store, const StorePath& drvPath)
{
// This is quite dirty and leaky, but will disappear once #4340 is merged
static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
{
auto resolutions = resolutionsCache.lock();
auto resolvedDrvIter = resolutions->find(drvPath);
if (resolvedDrvIter != resolutions->end()) {
auto & [_, resolvedDrv] = *resolvedDrvIter;
return *resolvedDrv;
}
}
/* Try resolve drv and use that path instead. */
auto drv = store.readDerivation(drvPath);
auto attempt = drv.tryResolveUncached(store);
if (!attempt)
return std::nullopt;
/* Store in memo table. */
resolutionsCache.lock()->insert_or_assign(drvPath, *attempt);
return *attempt;
}
}

4
src/libstore/derivations.hh
View file

@ -138,10 +138,14 @@ struct Derivation : BasicDerivation
2. Input placeholders are replaced with realized input store paths. */
std::optional<BasicDerivation> tryResolve(Store & store);
static std::optional<BasicDerivation> tryResolve(Store & store, const StorePath & drvPath);
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
private:
std::optional<BasicDerivation> tryResolveUncached(Store & store);
};

45
src/libstore/local-store.cc
View file

@ -877,35 +877,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
});
}
// Try to resolve the derivation at path `original`, with a caching layer
// to make it more efficient
std::optional<Derivation> cachedResolve(
LocalStore& store,
const StorePath& original)
{
// This is quite dirty and leaky, but will disappear once #4340 is merged
static Sync<std::map<StorePath, std::optional<Derivation>>> resolutionsCache;
{
auto resolutions = resolutionsCache.lock();
auto resolvedDrvIter = resolutions->find(original);
if (resolvedDrvIter != resolutions->end()) {
auto & [_, resolvedDrv] = *resolvedDrvIter;
return *resolvedDrv;
}
}
/* Try resolve drv and use that path instead. */
auto drv = store.readDerivation(original);
auto attempt = drv.tryResolve(store);
if (!attempt)
return std::nullopt;
/* Store in memo table. */
resolutionsCache.lock()->insert_or_assign(original, *attempt);
return *attempt;
}
std::map<std::string, std::optional<StorePath>>
LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_)
{
auto path = path_;
auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
@ -924,20 +898,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_)
if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
return outputs;
auto drv = readDerivation(path);
auto resolvedDrv = cachedResolve(*this, path);
if (!resolvedDrv) {
for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) {
if (!outputs.count(outputName))
outputs.emplace(outputName, std::nullopt);
}
return outputs;
}
auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv);
for (auto& [outputName, hash] : resolvedDrvHashes) {
auto drv = readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto& [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation)
outputs.insert_or_assign(outputName, realisation->outPath);

2
src/libstore/local-store.hh
View file

@ -127,7 +127,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;

39
src/libstore/store-api.cc
View file

@ -366,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
std::map<std::string, std::optional<StorePath>> Store::queryDerivationOutputMapNoResolve(const StorePath & path)
{
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = readInvalidDerivation(path);
for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) {
outputs.emplace(outputName, output.second);
}
return outputs;
}
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
{
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
auto resolvedDrv = Derivation::tryResolve(*this, path);
if (resolvedDrv) {
auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true);
if (isValidPath(resolvedDrvPath))
return queryDerivationOutputMapNoResolve(resolvedDrvPath);
}
}
return queryDerivationOutputMapNoResolve(path);
}
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
auto resp = queryPartialDerivationOutputMap(path);
OutputPathMap result;
@ -730,14 +753,14 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod
for (auto & path : paths) {
if (path.path.isDerivation()) {
if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
for (auto & outputName : path.outputs) {
if (!queryRealisation({path.path, outputName}))
unsupported("buildPaths");
}
} else
unsupported("buildPaths");
auto outPaths = queryPartialDerivationOutputMap(path.path);
for (auto & outputName : path.outputs) {
auto currentOutputPathIter = outPaths.find(outputName);
if (currentOutputPathIter == outPaths.end() ||
!currentOutputPathIter->second ||
!isValidPath(*currentOutputPathIter->second))
unsupported("buildPaths");
}
} else
paths2.insert(path.path);
}

9
src/libstore/store-api.hh
View file

@ -416,8 +416,13 @@ public:
/* Query the mapping outputName => outputPath for the given derivation. All
outputs are mentioned so ones mising the mapping are mapped to
`std::nullopt`. */
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path)
{ unsupported("queryPartialDerivationOutputMap"); }
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
/*
* Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve
* the derivation
*/
virtual std::map<std::string, std::optional<StorePath>> queryDerivationOutputMapNoResolve(const StorePath & path);
/* Query the mapping outputName=>outputPath for the given derivation.
Assume every output has a mapping and throw an exception otherwise. */