From 7e162c69fe6cbfb929b5356a7df9de5c25c22565 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Nov 2022 16:26:20 +0100 Subject: [PATCH 1/2] derivation-goal: Fix `requires non-existing output` error It occurred when a output of the dependency was already available, so it didn't need rebuilding and didn't get added to the inputDrvOutputs. This process-related info wasn't suitable for the purpose of finding the actual input paths for the builder. It is better to do this in absolute terms by querying the store. --- src/libstore/build/derivation-goal.cc | 30 +++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 41d2e2a1c..00e375fe9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + for (auto & j : wantedDepOutputs) { + /* TODO (impure derivations-induced tech debt): + Tracking input derivation outputs statefully through the + goals is error prone and has led to bugs. + For a robust nix, we need to move towards the `else` branch, + which does not rely on goal state to match up with the + reality of the store, which is our real source of truth. + However, the impure derivations feature still relies on this + fragile way of doing things, because its builds do not have + a representation in the store, which is a usability problem + in itself */ + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { worker.store.computeFSClosure(*outPath, inputPaths); - else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + else { + auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); + auto outMapPath = outMap.find(j); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + worker.store.computeFSClosure(outMapPath->second, inputPaths); + } + } } } From c279ddb18cf3a34b0f6d4e3adcf9455da5397ad7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 9 Nov 2022 15:21:13 +0100 Subject: [PATCH 2/2] tests: Reproduce #6572 --- tests/build.sh | 51 ++++++++++++++++++++++++++++++++++++++ tests/multiple-outputs.nix | 30 ++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/tests/build.sh b/tests/build.sh index fc6825e25..c7db039b4 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -70,3 +70,54 @@ testNormalization () { } testNormalization + +# https://github.com/NixOS/nix/issues/6572 +issue_6572_independent_outputs() { + nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json + + # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" + p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) + cmp $p < $TEST_ROOT/a.json + + # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + nix-store --delete "$p" # Clean up for next test + + # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" + p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) + cmp $p <$out + ''; + }; + b = mkDerivation { defaultOutput = assert a.second.helloString == "Hello, world!"; a; firstOutput = assert a.outputName == "first"; a.first.first; @@ -87,4 +96,25 @@ rec { buildCommand = "mkdir $a $b $c"; }; + independent = mkDerivation { + name = "multiple-outputs-independent"; + outputs = [ "first" "second" ]; + builder = builtins.toFile "builder.sh" + '' + mkdir $first $second + test -z $all + echo "first" > $first/file + echo "second" > $second/file + ''; + }; + + use-independent = mkDerivation { + name = "use-independent"; + inherit (a) first second; + builder = builtins.toFile "builder.sh" + '' + cat $first/file $second/file >$out + ''; + }; + }