nix-push: Only generate and copy a NAR if it doesn't already exist

This prevents unnecessary and slow rebuilds of NARs that already exist
in the binary cache.

perl/lib/Nix/Manifest.pm

@@ -9,7 +9,7 @@ use Fcntl ':flock';
 use Nix::Config;
 
 our @ISA = qw(Exporter);
-our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests);
+our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo);
 
 
 sub addNAR {
@@ -388,4 +388,42 @@ sub deleteOldManifests {
 }
 
 
+# Parse a NAR info file.
+sub parseNARInfo {
+    my ($storePath, $content) = @_;
+
+    my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system);
+    my $compression = "bzip2";
+    my @refs;
+
+    foreach my $line (split "\n", $content) {
+        return undef unless $line =~ /^(.*): (.*)$/;
+        if ($1 eq "StorePath") { $storePath2 = $2; }
+        elsif ($1 eq "URL") { $url = $2; }
+        elsif ($1 eq "Compression") { $compression = $2; }
+        elsif ($1 eq "FileHash") { $fileHash = $2; }
+        elsif ($1 eq "FileSize") { $fileSize = int($2); }
+        elsif ($1 eq "NarHash") { $narHash = $2; }
+        elsif ($1 eq "NarSize") { $narSize = int($2); }
+        elsif ($1 eq "References") { @refs = split / /, $2; }
+        elsif ($1 eq "Deriver") { $deriver = $2; }
+        elsif ($1 eq "System") { $system = $2; }
+    }
+
+    return undef if $storePath ne $storePath2 || !defined $url || !defined $narHash;
+
+    return
+        { url => $url
+        , compression => $compression
+        , fileHash => $fileHash
+        , fileSize => $fileSize
+        , narHash => $narHash
+        , narSize => $narSize
+        , refs => [ @refs ]
+        , deriver => $deriver
+        , system => $system
+        };
+}
+
+
 return 1;

perl/lib/Nix/Utils.pm

@@ -1,5 +1,8 @@
 package Nix::Utils;
 
+our @ISA = qw(Exporter);
+our @EXPORT = qw(checkURL uniq writeFile readFile);
+
 $urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )";
 
 sub checkURL {
@@ -17,3 +20,19 @@ sub uniq {
     }
     return @res;
 }
+
+sub writeFile {
+    my ($fn, $s) = @_;
+    open TMP, ">$fn" or die;
+    print TMP "$s" or die;
+    close TMP or die;
+}
+
+sub readFile {
+    local $/ = undef;
+    my ($fn) = @_;
+    open TMP, "<$fn" or die;
+    my $s = <TMP>;
+    close TMP or die;
+    return $s;
+}

scripts/download-from-binary-cache.pl.in

@@ -6,6 +6,7 @@ use IO::Select;
 use Nix::Config;
 use Nix::Store;
 use Nix::Utils;
+use Nix::Manifest;
 use WWW::Curl::Easy;
 use WWW::Curl::Multi;
 use strict;
@@ -199,7 +200,7 @@ sub getAvailableCaches {
     # denotes options passed by the client.
     if (defined $Nix::Config::config{"untrusted-binary-caches"}) {
         my @untrustedUrls = strToList $Nix::Config::config{"untrusted-binary-caches"};
-        my @trustedUrls = Nix::Utils::uniq(@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
+        my @trustedUrls = uniq(@urls, strToList($Nix::Config::config{"trusted-binary-caches"} // ""));
         @urls = ();
         foreach my $url (@untrustedUrls) {
             die "binary cache ‘$url’ is not trusted (please add it to ‘trusted-binary-caches’ [@trustedUrls] in $Nix::Config::confDir/nix.conf)\n"
@@ -208,7 +209,7 @@ sub getAvailableCaches {
         }
     }
 
-    foreach my $url (Nix::Utils::uniq @urls) {
+    foreach my $url (uniq @urls) {
 
         # FIXME: not atomic.
         $queryCache->execute($url);
@@ -265,48 +266,17 @@ sub processNARInfo {
         return undef;
     }
 
-    my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system);
+    my $narInfo = parseNARInfo($storePath, $request->{content});
-    my $compression = "bzip2";
+    return undef unless defined $narInfo;
-    my @refs;
-    foreach my $line (split "\n", $request->{content}) {
-        unless ($line =~ /^(.*): (.*)$/) {
-            print STDERR "bad NAR info file ‘$request->{url}’\n";
-            return undef;
-        }
-        if ($1 eq "StorePath") { $storePath2 = $2; }
-        elsif ($1 eq "URL") { $url = $2; }
-        elsif ($1 eq "Compression") { $compression = $2; }
-        elsif ($1 eq "FileHash") { $fileHash = $2; }
-        elsif ($1 eq "FileSize") { $fileSize = int($2); }
-        elsif ($1 eq "NarHash") { $narHash = $2; }
-        elsif ($1 eq "NarSize") { $narSize = int($2); }
-        elsif ($1 eq "References") { @refs = split / /, $2; }
-        elsif ($1 eq "Deriver") { $deriver = $2; }
-        elsif ($1 eq "System") { $system = $2; }
-    }
-    return undef if $storePath ne $storePath2;
-    if ($storePath ne $storePath2 || !defined $url || !defined $narHash) {
-        print STDERR "bad NAR info file ‘$request->{url}’\n";
-        return undef;
-    }
 
     # Cache the result.
     $insertNAR->execute(
-        $cache->{id}, basename($storePath), $url, $compression, $fileHash, $fileSize,
+        $cache->{id}, basename($storePath), $narInfo->{url}, $narInfo->{compression},
-        $narHash, $narSize, join(" ", @refs), $deriver, $system, time())
+        $narInfo->{fileHash}, $narInfo->{fileSize}, $narInfo->{narHash}, $narInfo->{narSize},
+        join(" ", @$narInfo->{refs}), $narInfo->{deriver}, $narInfo->{system}, time())
         unless $request->{url} =~ /^file:/;
 
-    return
+    return $narInfo;
-        { url => $url
-        , compression => $compression
-        , fileHash => $fileHash
-        , fileSize => $fileSize
-        , narHash => $narHash
-        , narSize => $narSize
-        , refs => [ @refs ]
-        , deriver => $deriver
-        , system => $system
-        };
 }
 
 
@@ -509,7 +479,7 @@ sub downloadBinary {
         }
         my $url = "$cache->{url}/$info->{url}"; # FIXME: handle non-relative URLs
         print STDERR "\n*** Downloading ‘$url’ to ‘$storePath’...\n";
-        Nix::Utils::checkURL $url;
+        checkURL $url;
         if (system("$Nix::Config::curl --fail --location --insecure '$url' | $decompressor | $Nix::Config::binDir/nix-store --restore $destPath") != 0) {
             die "download of `$info->{url}' failed" . ($! ? ": $!" : "") . "\n" unless $? == 0;
             next;

scripts/download-using-manifests.pl.in

@@ -308,7 +308,7 @@ while (scalar @path > 0) {
         # Download the patch.
         print STDERR "  downloading patch...\n";
         my $patchPath = "$tmpDir/patch";
-        Nix::Utils::checkURL $patch->{url};
+        checkURL $patch->{url};
         system("$curl '$patch->{url}' -o $patchPath") == 0
             or die "cannot download patch `$patch->{url}'\n";
 
@@ -339,7 +339,7 @@ while (scalar @path > 0) {
         my $size = $narFile->{size} || -1;
         print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
 
-        Nix::Utils::checkURL $narFile->{url};
+        checkURL $narFile->{url};
 
         my $decompressor =
             $narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" :

scripts/nix-prefetch-url.in

@@ -6,6 +6,7 @@ use File::Temp qw(tempdir);
 use File::stat;
 use Nix::Store;
 use Nix::Config;
+use Nix::Utils;
 
 my $url = shift;
 my $expHash = shift;
@@ -20,22 +21,6 @@ EOF
     exit 1;
 }
 
-sub writeFile {
-    my ($fn, $s) = @_;
-    open TMP, ">$fn" or die;
-    print TMP "$s" or die;
-    close TMP or die;
-}
-
-sub readFile {
-    local $/ = undef;
-    my ($fn) = @_;
-    open TMP, "<$fn" or die;
-    my $s = <TMP>;
-    close TMP or die;
-    return $s;
-}
-
 my $tmpDir = tempdir("nix-prefetch-url.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 

scripts/nix-push.in

@@ -9,6 +9,7 @@ use File::Copy;
 use Nix::Config;
 use Nix::Store;
 use Nix::Manifest;
+use Nix::Utils;
 
 my $tmpDir = tempdir("nix-push.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
@@ -81,12 +82,43 @@ foreach my $path (@roots) {
 my @storePaths = keys %storePaths;
 
 
+# Don't create archives for files that are already in the binary cache.
+my @storePaths2;
+my %narFiles;
+foreach my $storePath (@storePaths) {
+    my $pathHash = substr(basename($storePath), 0, 32);
+    my $narInfoFile = "$destDir/$pathHash.narinfo";
+    if (-e $narInfoFile) {
+        my $narInfo = parseNARInfo($storePath, readFile($narInfoFile));
+        my $narFile = "$destDir/$narInfo->{url}";
+        if (-e $narFile) {
+            print STDERR "skipping existing $storePath\n";
+            # Add the NAR info to $narFiles if we're writing a
+            # manifest.
+            $narFiles{$storePath} = [
+                { url => ("$archivesURL/" . basename $narInfo->{url})
+                  , hash => $narInfo->{fileHash}
+                  , size => $narInfo->{fileSize}
+                  , compressionType => $narInfo->{compression}
+                  , narHash => $narInfo->{narHash}
+                  , narSize => $narInfo->{narSize}
+                  , references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}})
+                  , deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef
+                  }
+            ] if $writeManifest;
+            next;
+        }
+    }
+    push @storePaths2, $storePath;
+}
+
+
 # Create a list of Nix derivations that turn each path into a Nix
 # archive.
 open NIX, ">$nixExpr";
 print NIX "[";
 
-foreach my $storePath (@storePaths) {
+foreach my $storePath (@storePaths2) {
     die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);
 
     # Construct a Nix expression that creates a Nix archive.
@@ -130,10 +162,8 @@ print STDERR "copying archives...\n";
 my $totalNarSize = 0;
 my $totalCompressedSize = 0;
 
-my %narFiles;
+for (my $n = 0; $n < scalar @storePaths2; $n++) {
-
+    my $storePath = $storePaths2[$n];
-for (my $n = 0; $n < scalar @storePaths; $n++) {
-    my $storePath = $storePaths[$n];
     my $narDir = $narPaths[$n];
     my $baseName = basename $storePath;
 
@@ -226,7 +256,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
 }
 
 printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
-    $totalCompressedSize / (1024 * 1024), $totalCompressedSize / $totalNarSize * 100;
+    $totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100;
 
 
 # Optionally write a manifest.