From 175c78591b1877c280936cc663b76e47f207dd77 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 23:09:43 +0100 Subject: [PATCH 1/6] Random cleanup --- src/libstore/build/goal.hh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 07c752bb9..35121c5d9 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -40,21 +40,21 @@ struct Goal : public std::enable_shared_from_this WeakGoals waiters; /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; + size_t nrFailed = 0; /* Number of substitution goals we are/were waiting for that failed because there are no substituters. */ - unsigned int nrNoSubstituters; + size_t nrNoSubstituters = 0; /* Number of substitution goals we are/were waiting for that failed because they had unsubstitutable references. */ - unsigned int nrIncompleteClosure; + size_t nrIncompleteClosure = 0; /* Name of this goal for debugging purposes. */ std::string name; /* Whether the goal is finished. */ - ExitCode exitCode; + ExitCode exitCode = ecBusy; /* Build result. */ BuildResult buildResult; @@ -65,10 +65,7 @@ struct Goal : public std::enable_shared_from_this Goal(Worker & worker, DerivedPath path) : worker(worker) , buildResult { .path = std::move(path) } - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } + { } virtual ~Goal() { From 09796c026376b49a8bc5bb3a2865db015d3dfb06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 23:24:10 +0100 Subject: [PATCH 2/6] Random cleanup --- src/libstore/build/goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index d2420b107..58e805f55 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee) void Goal::waiteeDone(GoalPtr waitee, ExitCode result) { - assert(waitees.find(waitee) != waitees.end()); + assert(waitees.count(waitee)); waitees.erase(waitee); trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); From fe5509df9a58359a67da8c72ed8721cb3a33371d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 23:24:48 +0100 Subject: [PATCH 3/6] Only return wanted outputs --- src/libstore/build/derivation-goal.cc | 5 +---- src/libstore/build/local-derivation-goal.cc | 3 ++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 40c445836..7dc5737fb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -311,14 +311,11 @@ void DerivationGoal::outputsSubstitutionTried() gaveUpOnSubstitution(); } + /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - /* The inputs must be built before we can build this goal. */ if (useDerivation) for (auto & i : dynamic_cast(drv.get())->inputDrvs) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 75e7e6ca3..ed83c4770 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2613,7 +2613,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - builtOutputs.emplace(thisRealisation.id, thisRealisation); + if (wantedOutputs.empty() || wantedOutputs.count(outputName)) + builtOutputs.emplace(thisRealisation.id, thisRealisation); } return builtOutputs; From 540d7e33d879e21d34de0935292a65bf27f5e312 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 23:25:12 +0100 Subject: [PATCH 4/6] Retry substitution after an incomplete closure only once This avoids an infinite loop in the final test in tests/binary-cache.sh. I think this was only not triggered previously by accident (because we were clearing wantedOutputs in between). --- src/libstore/build/derivation-goal.cc | 5 ++--- src/libstore/build/derivation-goal.hh | 8 ++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7dc5737fb..3d1c4fbc1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -207,8 +207,6 @@ void DerivationGoal::haveDerivation() if (!drv->type().hasKnownOutputPaths()) settings.requireExperimentalFeature(Xp::CaDerivations); - retrySubstitution = false; - for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) worker.store.addTempRoot(*i.second.second); @@ -423,7 +421,8 @@ void DerivationGoal::inputsRealised() return; } - if (retrySubstitution) { + if (retrySubstitution && !retriedSubstitution) { + retriedSubstitution = true; haveDerivation(); return; } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index ea2db89b2..f556b6f25 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -61,8 +61,12 @@ struct DerivationGoal : public Goal bool needRestart = false; /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; + inputs. This is done in case of an incomplete closure. */ + bool retrySubstitution = false; + + /* Whether we've retried substitution, in which case we won't try + again. */ + bool retriedSubstitution = false; /* The derivation stored at drvPath. */ std::unique_ptr drv; From 187dc080a2ab3403c1b1b47b3face205a4a67fb4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 24 Mar 2022 23:36:14 +0100 Subject: [PATCH 5/6] tests/build.sh: Test that 'nix build' only prints wanted outputs --- tests/build.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/build.sh b/tests/build.sh index c77f620f7..13a0f42be 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -1,15 +1,27 @@ source common.sh -expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]' +clearStore + +# Make sure that 'nix build' only returns the outputs we asked for. +nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys | length == 1) and + (.outputs.first | match(".*multiple-outputs-a-first"))) +' + nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status ' (.[0] | (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys | length == 2) and (.outputs.first | match(".*multiple-outputs-a-first")) and (.outputs.second | match(".*multiple-outputs-a-second"))) and (.[1] | (.drvPath | match(".*multiple-outputs-b.drv")) and + (.outputs | keys | length == 1) and (.outputs.out | match(".*multiple-outputs-b"))) ' + testNormalization () { clearStore outPath=$(nix-build ./simple.nix --no-out-link) From 50c229ad9aeece3b4728fedb741d8ef7adb2b2c1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 25 Mar 2022 08:02:49 +0100 Subject: [PATCH 6/6] Use wantOutput Co-authored-by: John Ericson --- src/libstore/build/local-derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ed83c4770..b176f318b 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2613,7 +2613,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - if (wantedOutputs.empty() || wantedOutputs.count(outputName)) + if (wantOutput(outputName, wantedOutputs)) builtOutputs.emplace(thisRealisation.id, thisRealisation); }