Add support for uncompressed NARs in binary caches

Issue NixOS/hydra#102.
This commit is contained in:
Eelco Dolstra 2013-07-01 21:02:36 +02:00
parent 7986711632
commit 5116214343
6 changed files with 31 additions and 17 deletions

View file

@ -6,25 +6,28 @@ let
'' ''
export PATH=${nixBinDir}:${coreutils} export PATH=${nixBinDir}:${coreutils}
if [ $compressionType = "xz" ]; then if [ $compressionType = xz ]; then
ext=xz ext=.xz
compressor="${xz} -9" compressor="| ${xz} -9"
elif [ $compressionType = bzip2 ]; then
ext=.bz2
compressor="| ${bzip2}"
else else
ext=bz2 ext=
compressor="${bzip2}" compressor=
fi fi
echo "packing $storePath..." echo "packing $storePath..."
mkdir $out mkdir $out
dst=$out/tmp.nar.$ext dst=$out/tmp.nar$ext
set -o pipefail set -o pipefail
nix-store --dump "$storePath" | $compressor > $dst eval "nix-store --dump \"$storePath\" $compressor > $dst"
hash=$(nix-hash --flat --type $hashAlgo --base32 $dst) hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
echo -n $hash > $out/nar-compressed-hash echo -n $hash > $out/nar-compressed-hash
mv $dst $out/$hash.nar.$ext mv $dst $out/$hash.nar$ext
''; '';
in in

View file

@ -20,6 +20,7 @@
<command>nix-push</command> <command>nix-push</command>
<arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg> <arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg>
<arg><option>--bzip2</option></arg> <arg><option>--bzip2</option></arg>
<arg><option>--none</option></arg>
<arg><option>--force</option></arg> <arg><option>--force</option></arg>
<arg><option>--link</option></arg> <arg><option>--link</option></arg>
<arg><option>--manifest</option></arg> <arg><option>--manifest</option></arg>
@ -106,6 +107,12 @@ automatically.</para>
</varlistentry> </varlistentry>
<varlistentry><term><option>--none</option></term>
<listitem><para>Do not compress NARs.</para></listitem>
</varlistentry>
<varlistentry><term><option>--force</option></term> <varlistentry><term><option>--force</option></term>
<listitem><para>Overwrite <filename>.narinfo</filename> files if <listitem><para>Overwrite <filename>.narinfo</filename> files if

View file

@ -513,8 +513,9 @@ sub downloadBinary {
next unless defined $info; next unless defined $info;
my $decompressor; my $decompressor;
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; } if ($info->{compression} eq "bzip2") { $decompressor = "| $Nix::Config::bzip2 -d"; }
elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; } elsif ($info->{compression} eq "xz") { $decompressor = "| $Nix::Config::xz -d"; }
elsif ($info->{compression} eq "none") { $decompressor = ""; }
else { else {
print STDERR "unknown compression method $info->{compression}\n"; print STDERR "unknown compression method $info->{compression}\n";
next; next;
@ -522,7 +523,7 @@ sub downloadBinary {
my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
print STDERR "\n*** Downloading $url to $storePath...\n"; print STDERR "\n*** Downloading $url to $storePath...\n";
checkURL $url; checkURL $url;
if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) { if (system("$Nix::Config::curl --fail --location --insecure '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
warn "download of `$url' failed" . ($! ? ": $!" : "") . "\n"; warn "download of `$url' failed" . ($! ? ": $!" : "") . "\n";
next; next;
} }

View file

@ -344,17 +344,18 @@ while (scalar @path > 0) {
checkURL $narFile->{url}; checkURL $narFile->{url};
my $decompressor = my $decompressor =
$narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" : $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" :
$narFile->{compressionType} eq "xz" ? "$Nix::Config::xz -d" : $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" :
$narFile->{compressionType} eq "none" ? "" :
die "unknown compression type `$narFile->{compressionType}'"; die "unknown compression type `$narFile->{compressionType}'";
if ($curStep < $maxStep) { if ($curStep < $maxStep) {
# The archive will be used a base to a patch. # The archive will be used a base to a patch.
system("$curl '$narFile->{url}' | $decompressor > $tmpNar") == 0 system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0
or die "cannot download and unpack `$narFile->{url}' to `$v'\n"; or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
} else { } else {
# Unpack the archive to the target path. # Unpack the archive to the target path.
system("$curl '$narFile->{url}' | $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0 system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
or die "cannot download and unpack `$narFile->{url}' to `$v'\n"; or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
} }

View file

@ -34,6 +34,8 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
exec "man nix-push" or die; exec "man nix-push" or die;
} elsif ($arg eq "--bzip2") { } elsif ($arg eq "--bzip2") {
$compressionType = "bzip2"; $compressionType = "bzip2";
} elsif ($arg eq "--none") {
$compressionType = "none";
} elsif ($arg eq "--force") { } elsif ($arg eq "--force") {
$force = 1; $force = 1;
} elsif ($arg eq "--dest") { } elsif ($arg eq "--dest") {
@ -202,7 +204,7 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) {
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash"; $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH; close HASH;
my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2"); my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");
my $narFile = "$narDir/$narName"; my $narFile = "$narDir/$narName";
(-f $narFile) or die "NAR file for $storePath not found"; (-f $narFile) or die "NAR file for $storePath not found";

View file

@ -47,7 +47,7 @@ clearStore
nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info
mkdir $cacheDir/tmp mkdir $cacheDir/tmp
mv $cacheDir/*.nar.xz $cacheDir/tmp/ mv $cacheDir/*.nar* $cacheDir/tmp/
NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback