Don't require --fallback to recover from disappeared binary cache NARs

This commit is contained in:
Eelco Dolstra 2018-06-05 16:04:41 +02:00
parent 691b7582c7
commit 4ac4f675df
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
4 changed files with 41 additions and 17 deletions

View file

@ -218,7 +218,11 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink)
auto info = queryPathInfo(storePath).cast<const NarInfo>(); auto info = queryPathInfo(storePath).cast<const NarInfo>();
auto source = sinkToSource([this, url{info->url}](Sink & sink) { auto source = sinkToSource([this, url{info->url}](Sink & sink) {
try {
getFile(url, sink); getFile(url, sink);
} catch (NoSuchBinaryCacheFile & e) {
throw SubstituteGone(e.what());
}
}); });
stats.narRead++; stats.narRead++;

View file

@ -733,7 +733,7 @@ private:
/* Whether to retry substituting the outputs after building the /* Whether to retry substituting the outputs after building the
inputs. */ inputs. */
bool retrySubstitution = false; bool retrySubstitution;
/* The derivation stored at drvPath. */ /* The derivation stored at drvPath. */
std::unique_ptr<BasicDerivation> drv; std::unique_ptr<BasicDerivation> drv;
@ -1123,6 +1123,8 @@ void DerivationGoal::haveDerivation()
{ {
trace("have derivation"); trace("have derivation");
retrySubstitution = false;
for (auto & i : drv->outputs) for (auto & i : drv->outputs)
worker.store.addTempRoot(i.second.path); worker.store.addTempRoot(i.second.path);
@ -1161,7 +1163,7 @@ void DerivationGoal::outputsSubstituted()
/* If the substitutes form an incomplete closure, then we should /* If the substitutes form an incomplete closure, then we should
build the dependencies of this derivation, but after that, we build the dependencies of this derivation, but after that, we
can still use the substitutes for this derivation itself. */ can still use the substitutes for this derivation itself. */
if (nrIncompleteClosure > 0 && !retrySubstitution) retrySubstitution = true; if (nrIncompleteClosure > 0) retrySubstitution = true;
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
@ -3524,8 +3526,8 @@ private:
/* The current substituter. */ /* The current substituter. */
std::shared_ptr<Store> sub; std::shared_ptr<Store> sub;
/* Whether any substituter can realise this path. */ /* Whether a substituter failed. */
bool hasSubstitute; bool substituterFailed = false;
/* Path info returned by the substituter's query info operation. */ /* Path info returned by the substituter's query info operation. */
std::shared_ptr<const ValidPathInfo> info; std::shared_ptr<const ValidPathInfo> info;
@ -3589,7 +3591,6 @@ public:
SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair)
: Goal(worker) : Goal(worker)
, hasSubstitute(false)
, repair(repair) , repair(repair)
{ {
this->storePath = storePath; this->storePath = storePath;
@ -3653,9 +3654,9 @@ void SubstitutionGoal::tryNext()
/* Hack: don't indicate failure if there were no substituters. /* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a In that case the calling derivation should just do a
build. */ build. */
amDone(hasSubstitute ? ecFailed : ecNoSubstituters); amDone(substituterFailed ? ecFailed : ecNoSubstituters);
if (hasSubstitute) { if (substituterFailed) {
worker.failedSubstitutions++; worker.failedSubstitutions++;
worker.updateProgress(); worker.updateProgress();
} }
@ -3691,8 +3692,6 @@ void SubstitutionGoal::tryNext()
worker.updateProgress(); worker.updateProgress();
hasSubstitute = true;
/* Bail out early if this substituter lacks a valid /* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */ only after we've downloaded the path. */
@ -3807,8 +3806,19 @@ void SubstitutionGoal::finished()
state = &SubstitutionGoal::init; state = &SubstitutionGoal::init;
worker.waitForAWhile(shared_from_this()); worker.waitForAWhile(shared_from_this());
return; return;
} catch (Error & e) { } catch (std::exception & e) {
printError(e.msg()); printError(e.what());
/* Cause the parent build to fail unless --fallback is given,
or the substitute has disappeared. The latter case behaves
the same as the substitute never having existed in the
first place. */
try {
throw;
} catch (SubstituteGone &) {
} catch (...) {
substituterFailed = true;
}
/* Try the next substitute. */ /* Try the next substitute. */
state = &SubstitutionGoal::tryNext; state = &SubstitutionGoal::tryNext;

View file

@ -22,6 +22,7 @@ MakeError(SubstError, Error)
MakeError(BuildError, Error) /* denotes a permanent build failure */ MakeError(BuildError, Error) /* denotes a permanent build failure */
MakeError(InvalidPath, Error) MakeError(InvalidPath, Error)
MakeError(Unsupported, Error) MakeError(Unsupported, Error)
MakeError(SubstituteGone, Error)
struct BasicDerivation; struct BasicDerivation;

View file

@ -76,19 +76,28 @@ if nix-store --substituters "file://$cacheDir" -r $outPath; then
fi fi
# Test whether fallback works if we have cached info but the # Test whether fallback works if a NAR has disappeared. This does not require --fallback.
# corresponding NAR has disappeared.
clearStore clearStore
nix-build --substituters "file://$cacheDir" dependencies.nix --dry-run # get info
mkdir $cacheDir/tmp
mv $cacheDir/nar $cacheDir/nar2 mv $cacheDir/nar $cacheDir/nar2
nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result
mv $cacheDir/nar2 $cacheDir/nar
# Test whether fallback works if a NAR is corrupted. This does require --fallback.
clearStore
mv $cacheDir/nar $cacheDir/nar2
mkdir $cacheDir/nar
for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done
(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) (! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result)
nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback
rm -rf $cacheDir/nar
mv $cacheDir/nar2 $cacheDir/nar mv $cacheDir/nar2 $cacheDir/nar