the-distro/channel-scripts
1
0
Fork 1
Code Issues 8 Pull requests Packages Projects Releases Wiki Activity

* Speed up the mirror script by reusing the file size / hash info

Browse source 
from the previous manifest (rather than reading all those .hash 
  files, which is rather I/O intensive).


git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@23477 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb
This commit is contained in:
Eelco Dolstra 2010-08-27 09:01:45 +00:00
parent 412010e2dc
commit 34d4018237
1 changed files with 38 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
mirror-channel.pl
View file

@ -27,6 +27,22 @@ die "$patchesPath doesn't exist\n" unless -d $patchesPath;
open LOCK, ">$dstChannelPath/.lock" or die; open LOCK, ">$dstChannelPath/.lock" or die;
flock LOCK, LOCK_EX; flock LOCK, LOCK_EX;
system("date");
# Read the old manifest, if available.
my %narFilesOld;
my %localPathsOld;
my %patchesOld;
readManifest("$dstChannelPath/MANIFEST", \%narFilesOld, \%localPathsOld, \%patchesOld)
if -f "$dstChannelPath/MANIFEST";
my %knownURLs;
while (my ($storePath, $files) = each %narFilesOld) {
$knownURLs{$_->{url}} = $_ foreach @{$files};
}
# Fetch the new manifest.
my ($fh, $tmpManifest) = tempfile(UNLINK => 1); my ($fh, $tmpManifest) = tempfile(UNLINK => 1);
system("$curl '$srcChannelURL/MANIFEST' > $tmpManifest") == 0 or die; system("$curl '$srcChannelURL/MANIFEST' > $tmpManifest") == 0 or die;
@ -35,7 +51,7 @@ my %narFiles;
my %localPaths; my %localPaths;
my %patches; my %patches;
my $version = readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches); readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
%localPaths = (); %localPaths = ();
%patches = (); # not supported yet %patches = (); # not supported yet
@ -67,6 +83,12 @@ while (my ($storePath, $files) = each %narFiles) {
rename($dstFileTmp, $dstFile) or die "cannot rename $dstFileTmp"; rename($dstFileTmp, $dstFile) or die "cannot rename $dstFileTmp";
} }
my $old = $knownURLs{$dstURL};
if (defined $old) {
$file->{size} = $old->{size};
$file->{hash} = $old->{hash};
} else {
$file->{size} = stat($dstFile)->size or die "cannot get size of $dstFile"; $file->{size} = stat($dstFile)->size or die "cannot get size of $dstFile";
my $hashFile = "$narPath/.hash.$dstName"; my $hashFile = "$narPath/.hash.$dstName";
@ -84,6 +106,7 @@ while (my ($storePath, $files) = each %narFiles) {
} }
$file->{hash} = "sha256:$hash"; $file->{hash} = "sha256:$hash";
} }
}
} }
# Write the new manifest. # Write the new manifest.