From 5116214343ecce70a3cb7037f223313314a0a614 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 1 Jul 2013 21:02:36 +0200 Subject: [PATCH] Add support for uncompressed NARs in binary caches Issue NixOS/hydra#102. --- corepkgs/nar.nix | 19 +++++++++++-------- doc/manual/nix-push.xml | 7 +++++++ scripts/download-from-binary-cache.pl.in | 7 ++++--- scripts/download-using-manifests.pl.in | 9 +++++---- scripts/nix-push.in | 4 +++- tests/binary-cache.sh | 2 +- 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/corepkgs/nar.nix b/corepkgs/nar.nix index fc9687af7..73009047e 100644 --- a/corepkgs/nar.nix +++ b/corepkgs/nar.nix @@ -6,25 +6,28 @@ let '' export PATH=${nixBinDir}:${coreutils} - if [ $compressionType = "xz" ]; then - ext=xz - compressor="${xz} -9" + if [ $compressionType = xz ]; then + ext=.xz + compressor="| ${xz} -9" + elif [ $compressionType = bzip2 ]; then + ext=.bz2 + compressor="| ${bzip2}" else - ext=bz2 - compressor="${bzip2}" + ext= + compressor= fi echo "packing ‘$storePath’..." mkdir $out - dst=$out/tmp.nar.$ext + dst=$out/tmp.nar$ext set -o pipefail - nix-store --dump "$storePath" | $compressor > $dst + eval "nix-store --dump \"$storePath\" $compressor > $dst" hash=$(nix-hash --flat --type $hashAlgo --base32 $dst) echo -n $hash > $out/nar-compressed-hash - mv $dst $out/$hash.nar.$ext + mv $dst $out/$hash.nar$ext ''; in diff --git a/doc/manual/nix-push.xml b/doc/manual/nix-push.xml index c4a6b8080..e789bbf7d 100644 --- a/doc/manual/nix-push.xml +++ b/doc/manual/nix-push.xml @@ -20,6 +20,7 @@ nix-push dest-dir + @@ -106,6 +107,12 @@ automatically. + + + Do not compress NARs. + + + Overwrite .narinfo files if diff --git a/scripts/download-from-binary-cache.pl.in b/scripts/download-from-binary-cache.pl.in index abd1f7b71..ab72e83e8 100644 --- a/scripts/download-from-binary-cache.pl.in +++ b/scripts/download-from-binary-cache.pl.in @@ -513,8 +513,9 @@ sub downloadBinary { next unless defined $info; my $decompressor; - if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; } - elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; } + if ($info->{compression} eq "bzip2") { $decompressor = "| $Nix::Config::bzip2 -d"; } + elsif ($info->{compression} eq "xz") { $decompressor = "| $Nix::Config::xz -d"; } + elsif ($info->{compression} eq "none") { $decompressor = ""; } else { print STDERR "unknown compression method ‘$info->{compression}’\n"; next; @@ -522,7 +523,7 @@ sub downloadBinary { my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs print STDERR "\n*** Downloading ‘$url’ to ‘$storePath’...\n"; checkURL $url; - if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { + if (system("$Nix::Config::curl --fail --location --insecure '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { warn "download of `$url' failed" . ($! ? ": $!" : "") . "\n"; next; } diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 0471a9e1f..9d4b89bac 100755 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -344,17 +344,18 @@ while (scalar @path > 0) { checkURL $narFile->{url}; my $decompressor = - $narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" : - $narFile->{compressionType} eq "xz" ? "$Nix::Config::xz -d" : + $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" : + $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" : + $narFile->{compressionType} eq "none" ? "" : die "unknown compression type `$narFile->{compressionType}'"; if ($curStep < $maxStep) { # The archive will be used a base to a patch. - system("$curl '$narFile->{url}' | $decompressor > $tmpNar") == 0 + system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0 or die "cannot download and unpack `$narFile->{url}' to `$v'\n"; } else { # Unpack the archive to the target path. - system("$curl '$narFile->{url}' | $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0 + system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0 or die "cannot download and unpack `$narFile->{url}' to `$v'\n"; } diff --git a/scripts/nix-push.in b/scripts/nix-push.in index 7a6670f67..2c392c415 100755 --- a/scripts/nix-push.in +++ b/scripts/nix-push.in @@ -34,6 +34,8 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { exec "man nix-push" or die; } elsif ($arg eq "--bzip2") { $compressionType = "bzip2"; + } elsif ($arg eq "--none") { + $compressionType = "none"; } elsif ($arg eq "--force") { $force = 1; } elsif ($arg eq "--dest") { @@ -202,7 +204,7 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) { $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash"; close HASH; - my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2"); + my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : ""); my $narFile = "$narDir/$narName"; (-f $narFile) or die "NAR file for $storePath not found"; diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 878c4669b..eb2ebbff8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -47,7 +47,7 @@ clearStore nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info mkdir $cacheDir/tmp -mv $cacheDir/*.nar.xz $cacheDir/tmp/ +mv $cacheDir/*.nar* $cacheDir/tmp/ NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback