From 98df735b5149bc1e39ce6b0bae13fbf7cebcdc05 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Feb 2005 12:57:13 +0000 Subject: [PATCH] * Propagate the deriver of a path through the substitute mechanism. * Removed some dead code (successor stuff) from nix-push. * Updated terminology in the tests (store expr -> drv path). * Check that the deriver is set properly in the tests. --- scripts/nix-pull.in | 1 + scripts/nix-push.in | 29 ++++++++++------------------- scripts/readmanifest.pm.in | 12 ++++++------ src/libstore/build.cc | 3 ++- src/libstore/store.cc | 17 ++++++++++++----- src/libstore/store.hh | 5 ++++- src/nix-store/main.cc | 1 + tests/build-hook.sh | 6 +++--- tests/dependencies.sh | 12 ++++++++---- tests/fallback.sh | 10 +++++----- tests/gc-concurrent.sh | 14 +++++++------- tests/gc.sh | 6 +++--- tests/locking.sh | 8 ++++---- tests/nix-pull.sh | 4 ++++ tests/parallel.sh | 6 +++--- tests/simple.sh | 6 +++--- tests/substitutes.sh | 10 +++++----- tests/substitutes2.sh | 10 +++++----- 18 files changed, 86 insertions(+), 74 deletions(-) diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index 7f46236b8..13a2199fb 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -81,6 +81,7 @@ foreach my $storePath (keys %narFiles) { my $narFileList = $narFiles{$storePath}; foreach my $narFile (@{$narFileList}) { print WRITE "$storePath\n"; + print WRITE "$narFile->{deriver}\n"; print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; print WRITE "0\n"; my @references = split " ", $narFile->{references}; diff --git a/scripts/nix-push.in b/scripts/nix-push.in index 60ccce4ed..52f4a3012 100644 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -138,7 +138,6 @@ print STDERR "creating manifest...\n"; my %narFiles; my %patches; -my %successors; my @nararchives; for (my $n = 0; $n < scalar @storePaths; $n++) { @@ -169,7 +168,14 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { my $narbz2Size = (stat $narfile)[7]; - my $references = join(" ", split(" ", `$binDir/nix-store --query --references '$storePath'`)); + my $references = `$binDir/nix-store --query --references '$storePath'`; + die "cannot query references for `$storePath'" if $? != 0; + $references = join(" ", split(" ", $references)); + + my $deriver = `$binDir/nix-store --query --deriver '$storePath'`; + die "cannot query deriver for `$storePath'" if $? != 0; + chomp $deriver; + $deriver = "" if $deriver eq "unknown-deriver"; my $url; if ($localCopy) { @@ -184,27 +190,12 @@ for (my $n = 0; $n < scalar @storePaths; $n++) { , narHash => $narHash , hashAlgo => "sha1" , references => $references + , deriver => $deriver } ]; - - if ($storePath =~ /\.store$/) { - open PREDS, "$binDir/nix-store --query --predecessors $storePath |" or die "cannot run nix"; - while () { - chomp; - die unless (/^\//); - my $pred = $_; - # Only include predecessors that are themselves being - # pushed. - if (defined $storePaths{$pred}) { - $successors{$pred} = $storePath; - } - } - close PREDS; - } - } -writeManifest $manifest, \%narFiles, \%patches, \%successors; +writeManifest $manifest, \%narFiles, \%patches; sub copyFile { diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index ea1c5a447..0d14ffd1b 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -51,6 +51,7 @@ sub readManifest { my $patchType; my $narHash; my $references; + my $deriver; while () { chomp; @@ -73,6 +74,7 @@ sub readManifest { undef $baseHash; undef $patchType; $references = ""; + $deriver = ""; } } else { @@ -102,6 +104,7 @@ sub readManifest { push @{$narFileList}, { url => $url, hash => $hash, size => $size , narHash => $narHash, references => $references + , deriver => $deriver }; } @@ -131,6 +134,7 @@ sub readManifest { elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } + elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } # Compatibility; elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } @@ -148,7 +152,6 @@ sub writeManifest my $manifest = shift; my $narFiles = shift; my $patches = shift; - my $successors = shift; open MANIFEST, ">$manifest.tmp"; # !!! check exclusive @@ -164,11 +167,8 @@ sub writeManifest print MANIFEST " Size: $narFile->{size}\n"; print MANIFEST " References: $narFile->{references}\n" if defined $narFile->{references} && $narFile->{references} ne ""; - foreach my $p (keys %{$successors}) { # !!! quadratic - if ($$successors{$p} eq $storePath) { - print MANIFEST " SuccOf: $p\n"; - } - } + print MANIFEST " Deriver: $narFile->{deriver}\n" + if defined $narFile->{deriver} && $narFile->{deriver} ne ""; print MANIFEST "}\n"; } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3785c7da7..e71201785 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1502,7 +1502,8 @@ void SubstitutionGoal::finished() Transaction txn; createStoreTransaction(txn); - registerValidPath(txn, storePath, contentHash, references, ""); + registerValidPath(txn, storePath, contentHash, + references, sub.deriver); txn.commit(); outputLock->setDeletion(true); diff --git a/src/libstore/store.cc b/src/libstore/store.cc index 83577e4fb..5516dc801 100644 --- a/src/libstore/store.cc +++ b/src/libstore/store.cc @@ -391,6 +391,9 @@ Path queryDeriver(const Transaction & txn, const Path & storePath) } +const int substituteVersion = 2; + + static Substitutes readSubstitutes(const Transaction & txn, const Path & srcPath) { @@ -406,13 +409,15 @@ static Substitutes readSubstitutes(const Transaction & txn, break; } Strings ss2 = unpackStrings(*i); - if (ss2.size() == 3) { - /* Another old-style substitute. */ - continue; - } - if (ss2.size() != 2) throw Error("malformed substitute"); + if (ss2.size() == 0) continue; + int version; + if (!string2Int(ss2.front(), version)) continue; + if (version != substituteVersion) continue; + if (ss2.size() != 4) throw Error("malformed substitute"); Strings::iterator j = ss2.begin(); + j++; Substitute sub; + sub.deriver = *j++; sub.program = *j++; sub.args = unpackStrings(*j++); subs.push_back(sub); @@ -431,6 +436,8 @@ static void writeSubstitutes(const Transaction & txn, i != subs.end(); ++i) { Strings ss2; + ss2.push_back((format("%1%") % substituteVersion).str()); + ss2.push_back(i->deriver); ss2.push_back(i->program); ss2.push_back(packStrings(i->args)); ss.push_back(packStrings(ss2)); diff --git a/src/libstore/store.hh b/src/libstore/store.hh index 082fc9de4..3b18664de 100644 --- a/src/libstore/store.hh +++ b/src/libstore/store.hh @@ -16,7 +16,10 @@ const int nixSchemaVersion = 2; path (typically by fetching it from somewhere, e.g., from the network). */ struct Substitute -{ +{ + /* The derivation that built this store path (empty if none). */ + Path deriver; + /* Program to be executed to create the store path. Must be in the output path of `storeExpr'. */ Path program; diff --git a/src/nix-store/main.cc b/src/nix-store/main.cc index a124543fb..fa7960c38 100644 --- a/src/nix-store/main.cc +++ b/src/nix-store/main.cc @@ -280,6 +280,7 @@ static void opSubstitute(Strings opFlags, Strings opArgs) PathSet references; getline(cin, srcPath); if (cin.eof()) break; + getline(cin, sub.deriver); getline(cin, sub.program); string s; int n; diff --git a/tests/build-hook.sh b/tests/build-hook.sh index 402ed83c9..f11cd15b8 100644 --- a/tests/build-hook.sh +++ b/tests/build-hook.sh @@ -1,10 +1,10 @@ export NIX_BUILD_HOOK="build-hook.hook.sh" -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate build-hook.nix) +drvPath=$($TOP/src/nix-instantiate/nix-instantiate build-hook.nix) -echo "store expr is $storeExpr" +echo "derivation is $drvPath" -outPath=$($TOP/src/nix-store/nix-store -quf "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -quf "$drvPath") echo "output path is $outPath" diff --git a/tests/dependencies.sh b/tests/dependencies.sh index 56cb1e4c3..87f0682ac 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -1,15 +1,15 @@ -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix) +drvPath=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix) -echo "store expr is $storeExpr" +echo "derivation is $drvPath" -outPath=$($TOP/src/nix-store/nix-store -rvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -rvv "$drvPath") echo "output path is $outPath" text=$(cat "$outPath"/foobar) if test "$text" != "FOOBAR"; then exit 1; fi -deps=$($TOP/src/nix-store/nix-store -quR "$storeExpr") +deps=$($TOP/src/nix-store/nix-store -quR "$drvPath") echo "output closure contains $deps" @@ -24,3 +24,7 @@ input2OutPath=$(echo "$deps" | grep "dependencies-input-2") # The referers closure of input-2 should include outPath. $TOP/src/nix-store/nix-store -q --referers-closure "$input2OutPath" | grep "$outPath" + +# Check that the derivers are set properly. +test $($TOP/src/nix-store/nix-store -q --deriver "$outPath") = "$drvPath" +$TOP/src/nix-store/nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" diff --git a/tests/fallback.sh b/tests/fallback.sh index ac4370c61..24e578410 100644 --- a/tests/fallback.sh +++ b/tests/fallback.sh @@ -1,14 +1,14 @@ -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate fallback.nix) -echo "store expr is $storeExpr" +drvPath=$($TOP/src/nix-instantiate/nix-instantiate fallback.nix) +echo "derivation is $drvPath" -outPath=$($TOP/src/nix-store/nix-store -q --fallback "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -q --fallback "$drvPath") echo "output path is $outPath" # Register a non-existant substitute -(echo $outPath && echo $TOP/no-such-program && echo 0 && echo 0) | $TOP/src/nix-store/nix-store --substitute +(echo $outPath && echo "" && echo $TOP/no-such-program && echo 0 && echo 0) | $TOP/src/nix-store/nix-store --substitute # Build the derivation -$TOP/src/nix-store/nix-store -r --fallback "$storeExpr" +$TOP/src/nix-store/nix-store -r --fallback "$drvPath" text=$(cat "$outPath"/hello) if test "$text" != "Hello World!"; then exit 1; fi diff --git a/tests/gc-concurrent.sh b/tests/gc-concurrent.sh index b3fb06535..8cc24565d 100644 --- a/tests/gc-concurrent.sh +++ b/tests/gc-concurrent.sh @@ -1,17 +1,17 @@ -storeExpr1=$($TOP/src/nix-instantiate/nix-instantiate gc-concurrent.nix) -outPath1=$($TOP/src/nix-store/nix-store -q $storeExpr1) +drvPath1=$($TOP/src/nix-instantiate/nix-instantiate gc-concurrent.nix) +outPath1=$($TOP/src/nix-store/nix-store -q $drvPath1) -storeExpr2=$($TOP/src/nix-instantiate/nix-instantiate gc-concurrent2.nix) -outPath2=$($TOP/src/nix-store/nix-store -q $storeExpr2) +drvPath2=$($TOP/src/nix-instantiate/nix-instantiate gc-concurrent2.nix) +outPath2=$($TOP/src/nix-store/nix-store -q $drvPath2) -ln -s $storeExpr2 "$NIX_STATE_DIR"/gcroots/foo2 +ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo2 # Start build #1 in the background. It starts immediately. -$TOP/src/nix-store/nix-store -rvv "$storeExpr1" & +$TOP/src/nix-store/nix-store -rvv "$drvPath1" & pid1=$! # Start build #2 in the background after 3 seconds. -(sleep 3 && $TOP/src/nix-store/nix-store -rvv "$storeExpr2") & +(sleep 3 && $TOP/src/nix-store/nix-store -rvv "$drvPath2") & pid2=$! # Run the garbage collector while the build is running. Note: the GC diff --git a/tests/gc.sh b/tests/gc.sh index e45e35a8c..9fe0068b3 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -1,5 +1,5 @@ -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix) -outPath=$($TOP/src/nix-store/nix-store -rvv "$storeExpr") +drvPath=$($TOP/src/nix-instantiate/nix-instantiate dependencies.nix) +outPath=$($TOP/src/nix-store/nix-store -rvv "$drvPath") # Set a GC root. ln -s $outPath "$NIX_STATE_DIR"/gcroots/foo @@ -11,4 +11,4 @@ cat $outPath/foobar cat $outPath/input-2/bar # Check that the derivation has been GC'd. -if cat $storeExpr > /dev/null; then false; fi +if cat $drvPath > /dev/null; then false; fi diff --git a/tests/locking.sh b/tests/locking.sh index 5cafa2910..eeac569b8 100644 --- a/tests/locking.sh +++ b/tests/locking.sh @@ -1,15 +1,15 @@ -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate locking.nix) +drvPath=$($TOP/src/nix-instantiate/nix-instantiate locking.nix) -echo "store expr is $storeExpr" +echo "derivation is $drvPath" for i in 1 2 3 4 5; do echo "WORKER $i" - $TOP/src/nix-store/nix-store -rvv "$storeExpr" & + $TOP/src/nix-store/nix-store -rvv "$drvPath" & done sleep 5 -outPath=$($TOP/src/nix-store/nix-store -qvvf "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qvvf "$drvPath") echo "output path is $outPath" diff --git a/tests/nix-pull.sh b/tests/nix-pull.sh index 9a17f5bf6..b4a5e3371 100644 --- a/tests/nix-pull.sh +++ b/tests/nix-pull.sh @@ -31,3 +31,7 @@ echo "building $drvPath using substitutes..." $TOP/src/nix-store/nix-store -r $drvPath cat $outPath/input-2/bar + +# Check that the derivers are set properly. +test $($TOP/src/nix-store/nix-store -q --deriver "$outPath") = "$drvPath" +$TOP/src/nix-store/nix-store -q --deriver $(readlink $outPath/input-2) | grep -q -- "-input-2.drv" diff --git a/tests/parallel.sh b/tests/parallel.sh index 0044aaea6..0cc4ea9c3 100644 --- a/tests/parallel.sh +++ b/tests/parallel.sh @@ -1,8 +1,8 @@ -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate parallel.nix) +drvPath=$($TOP/src/nix-instantiate/nix-instantiate parallel.nix) -echo "store expr is $storeExpr" +echo "derivation is $drvPath" -outPath=$($TOP/src/nix-store/nix-store -qfvv -j10000 "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qfvv -j10000 "$drvPath") echo "output path is $outPath" diff --git a/tests/simple.sh b/tests/simple.sh index a5435a53b..996ef4e78 100644 --- a/tests/simple.sh +++ b/tests/simple.sh @@ -1,8 +1,8 @@ -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate simple.nix) +drvPath=$($TOP/src/nix-instantiate/nix-instantiate simple.nix) -echo "store expr is $storeExpr" +echo "derivation is $drvPath" -outPath=$($TOP/src/nix-store/nix-store -rvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -rvv "$drvPath") echo "output path is $outPath" diff --git a/tests/substitutes.sh b/tests/substitutes.sh index 061f5f32a..cfc3fa71a 100644 --- a/tests/substitutes.sh +++ b/tests/substitutes.sh @@ -1,20 +1,20 @@ # Instantiate. -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate substitutes.nix) -echo "store expr is $storeExpr" +drvPath=$($TOP/src/nix-instantiate/nix-instantiate substitutes.nix) +echo "derivation is $drvPath" # Find the output path. -outPath=$($TOP/src/nix-store/nix-store -qvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qvv "$drvPath") echo "output path is $outPath" regSub() { - (echo $1 && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld && echo 0) | $TOP/src/nix-store/nix-store --substitute + (echo $1 && echo "" && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld && echo 0) | $TOP/src/nix-store/nix-store --substitute } # Register a substitute for the output path. regSub $outPath $(pwd)/substituter.sh -$TOP/src/nix-store/nix-store -rvv "$storeExpr" +$TOP/src/nix-store/nix-store -rvv "$drvPath" text=$(cat "$outPath"/hello) if test "$text" != "Hallo Wereld"; then exit 1; fi diff --git a/tests/substitutes2.sh b/tests/substitutes2.sh index e48e12a8f..1a7b848ca 100644 --- a/tests/substitutes2.sh +++ b/tests/substitutes2.sh @@ -1,13 +1,13 @@ # Instantiate. -storeExpr=$($TOP/src/nix-instantiate/nix-instantiate substitutes2.nix) -echo "store expr is $storeExpr" +drvPath=$($TOP/src/nix-instantiate/nix-instantiate substitutes2.nix) +echo "derivation is $drvPath" # Find the output path. -outPath=$($TOP/src/nix-store/nix-store -qvvvvv "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qvvvvv "$drvPath") echo "output path is $outPath" regSub() { - (echo $1 && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld && echo 0) | $TOP/src/nix-store/nix-store --substitute + (echo $1 && echo "" && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld && echo 0) | $TOP/src/nix-store/nix-store --substitute } # Register a substitute for the output path. @@ -17,7 +17,7 @@ regSub $outPath $(pwd)/substituter.sh # precedence over the previous one. It will fail. regSub $outPath $(pwd)/substituter2.sh -$TOP/src/nix-store/nix-store -rvv "$storeExpr" +$TOP/src/nix-store/nix-store -rvv "$drvPath" text=$(cat "$outPath"/hello) if test "$text" != "Hallo Wereld"; then exit 1; fi