Add support for uncompressed NARs in binary caches

Issue NixOS/hydra#102.

corepkgs/nar.nix

@@ -6,25 +6,28 @@ let
     ''
       export PATH=${nixBinDir}:${coreutils}
 
-      if [ $compressionType = "xz" ]; then
+      if [ $compressionType = xz ]; then
-        ext=xz
+        ext=.xz
-        compressor="${xz} -9"
+        compressor="| ${xz} -9"
+      elif [ $compressionType = bzip2 ]; then
+        ext=.bz2
+        compressor="| ${bzip2}"
       else
-        ext=bz2
+        ext=
-        compressor="${bzip2}"
+        compressor=
       fi
 
       echo "packing ‘$storePath’..."
       mkdir $out
-      dst=$out/tmp.nar.$ext
+      dst=$out/tmp.nar$ext
 
       set -o pipefail
-      nix-store --dump "$storePath" | $compressor > $dst
+      eval "nix-store --dump \"$storePath\" $compressor > $dst"
 
       hash=$(nix-hash --flat --type $hashAlgo --base32 $dst)
       echo -n $hash > $out/nar-compressed-hash
 
-      mv $dst $out/$hash.nar.$ext
+      mv $dst $out/$hash.nar$ext
     '';
 
 in

doc/manual/nix-push.xml

@@ -20,6 +20,7 @@
     <command>nix-push</command>
     <arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg>
     <arg><option>--bzip2</option></arg>
+    <arg><option>--none</option></arg>
     <arg><option>--force</option></arg>
     <arg><option>--link</option></arg>
     <arg><option>--manifest</option></arg>
@@ -106,6 +107,12 @@ automatically.</para>
 
   </varlistentry>
 
+  <varlistentry><term><option>--none</option></term>
+
+    <listitem><para>Do not compress NARs.</para></listitem>
+
+  </varlistentry>
+
   <varlistentry><term><option>--force</option></term>
 
     <listitem><para>Overwrite <filename>.narinfo</filename> files if

scripts/download-from-binary-cache.pl.in

@@ -513,8 +513,9 @@ sub downloadBinary {
         next unless defined $info;
 
         my $decompressor;
-        if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
+        if ($info->{compression} eq "bzip2") { $decompressor = "| $Nix::Config::bzip2 -d"; }
-        elsif ($info->{compression} eq "xz") { $decompressor = "$Nix::Config::xz -d"; }
+        elsif ($info->{compression} eq "xz") { $decompressor = "| $Nix::Config::xz -d"; }
+        elsif ($info->{compression} eq "none") { $decompressor = ""; }
         else {
             print STDERR "unknown compression method ‘$info->{compression}’\n";
             next;
@@ -522,7 +523,7 @@ sub downloadBinary {
         my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
         print STDERR "\n*** Downloading ‘$url’ to ‘$storePath’...\n";
         checkURL $url;
-        if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
+        if (system("$Nix::Config::curl --fail --location --insecure '$url' $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
             warn "download of `$url' failed" . ($! ? ": $!" : "") . "\n";
             next;
         }

scripts/download-using-manifests.pl.in

@@ -344,17 +344,18 @@ while (scalar @path > 0) {
         checkURL $narFile->{url};
 
         my $decompressor =
-            $narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" :
+            $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" :
-            $narFile->{compressionType} eq "xz" ? "$Nix::Config::xz -d" :
+            $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" :
+            $narFile->{compressionType} eq "none" ? "" :
             die "unknown compression type `$narFile->{compressionType}'";
 
         if ($curStep < $maxStep) {
             # The archive will be used a base to a patch.
-            system("$curl '$narFile->{url}' | $decompressor > $tmpNar") == 0
+            system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0
                 or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
         } else {
             # Unpack the archive to the target path.
-            system("$curl '$narFile->{url}' | $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
+            system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
                 or die "cannot download and unpack `$narFile->{url}' to `$v'\n";
         }
 

scripts/nix-push.in

@@ -34,6 +34,8 @@ for (my $n = 0; $n < scalar @ARGV; $n++) {
         exec "man nix-push" or die;
     } elsif ($arg eq "--bzip2") {
         $compressionType = "bzip2";
+    } elsif ($arg eq "--none") {
+        $compressionType = "none";
     } elsif ($arg eq "--force") {
         $force = 1;
     } elsif ($arg eq "--dest") {
@@ -202,7 +204,7 @@ for (my $n = 0; $n < scalar @storePaths2; $n++) {
     $compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
     close HASH;
 
-    my $narName = "$compressedHash.nar." . ($compressionType eq "xz" ? "xz" : "bz2");
+    my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");
 
     my $narFile = "$narDir/$narName";
     (-f $narFile) or die "NAR file for $storePath not found";

tests/binary-cache.sh

@@ -47,7 +47,7 @@ clearStore
 nix-build --option binary-caches "file://$cacheDir" dependencies.nix --dry-run # get info
 
 mkdir $cacheDir/tmp
-mv $cacheDir/*.nar.xz $cacheDir/tmp/
+mv $cacheDir/*.nar* $cacheDir/tmp/
 
 NIX_DEBUG_SUBST=1 nix-build --option binary-caches "file://$cacheDir" dependencies.nix -o $TEST_ROOT/result --fallback