From 34d4018237ede6a2c27e5e3239121e06f2c71bd0 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date: Fri, 27 Aug 2010 09:01:45 +0000
Subject: [PATCH] * Speed up the mirror script by reusing the file size / hash
 info   from the previous manifest (rather than reading all those .hash  
 files, which is rather I/O intensive).

git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@23477 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb
---
 mirror-channel.pl | 53 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 38 insertions(+), 15 deletions(-)

diff --git a/mirror-channel.pl b/mirror-channel.pl
index f04796e..c50284e 100644
--- a/mirror-channel.pl
+++ b/mirror-channel.pl
@@ -27,6 +27,22 @@ die "$patchesPath doesn't exist\n" unless -d $patchesPath;
 open LOCK, ">$dstChannelPath/.lock" or die;
 flock LOCK, LOCK_EX;
 
+system("date");
+
+# Read the old manifest, if available.
+my %narFilesOld;
+my %localPathsOld;
+my %patchesOld;
+
+readManifest("$dstChannelPath/MANIFEST", \%narFilesOld, \%localPathsOld, \%patchesOld)
+    if -f "$dstChannelPath/MANIFEST";
+
+my %knownURLs;
+while (my ($storePath, $files) = each %narFilesOld) {
+    $knownURLs{$_->{url}} = $_ foreach @{$files};
+}
+
+# Fetch the new manifest.
 my ($fh, $tmpManifest) = tempfile(UNLINK => 1);
 system("$curl '$srcChannelURL/MANIFEST' > $tmpManifest") == 0 or die;
 
@@ -35,7 +51,7 @@ my %narFiles;
 my %localPaths;
 my %patches;
 
-my $version = readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
+readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
 
 %localPaths = ();
 %patches = (); # not supported yet
@@ -66,23 +82,30 @@ while (my ($storePath, $files) = each %narFiles) {
             system("$curl '$srcURL' > $dstFileTmp") == 0 or die "failed to download `$srcURL'";
             rename($dstFileTmp, $dstFile) or die "cannot rename $dstFileTmp";
         }
-        
-        $file->{size} = stat($dstFile)->size or die "cannot get size of $dstFile";
 
-        my $hashFile = "$narPath/.hash.$dstName";
-        my $hash;
-        if (-e $hashFile) {
-            open HASH, "<$hashFile" or die;
-            $hash = <HASH>;
-            close HASH;
+        my $old = $knownURLs{$dstURL};
+
+        if (defined $old) {
+            $file->{size} = $old->{size};
+            $file->{hash} = $old->{hash};
         } else {
-            $hash = `nix-hash --flat --type sha256 --base32 '$dstFile'` or die;
-            chomp $hash;
-            open HASH, ">$hashFile" or die;
-            print HASH $hash;
-            close HASH;
+            $file->{size} = stat($dstFile)->size or die "cannot get size of $dstFile";
+
+            my $hashFile = "$narPath/.hash.$dstName";
+            my $hash;
+            if (-e $hashFile) {
+                open HASH, "<$hashFile" or die;
+                $hash = <HASH>;
+                close HASH;
+            } else {
+                $hash = `nix-hash --flat --type sha256 --base32 '$dstFile'` or die;
+                chomp $hash;
+                open HASH, ">$hashFile" or die;
+                print HASH $hash;
+                close HASH;
+            }
+            $file->{hash} = "sha256:$hash";
         }
-        $file->{hash} = "sha256:$hash";
     }
 }