the-distro/channel-scripts
1
0
Fork 1
Code Issues 8 Pull requests Packages Projects Releases Wiki Activity

* Speed up the mirror script by reusing the file size / hash info

Browse source 
from the previous manifest (rather than reading all those .hash 
  files, which is rather I/O intensive).


git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@23477 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb
This commit is contained in:
Eelco Dolstra 2010-08-27 09:01:45 +00:00
parent 412010e2dc
commit 34d4018237
1 changed files with 38 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
mirror-channel.pl
View file

@ -27,6 +27,22 @@ die "$patchesPath doesn't exist\n" unless -d $patchesPath;
open LOCK, ">$dstChannelPath/.lock" or die;
flock LOCK, LOCK_EX;
system("date");
# Read the old manifest, if available.
my %narFilesOld;
my %localPathsOld;
my %patchesOld;
readManifest("$dstChannelPath/MANIFEST", \%narFilesOld, \%localPathsOld, \%patchesOld)
if -f "$dstChannelPath/MANIFEST";
my %knownURLs;
while (my ($storePath, $files) = each %narFilesOld) {
$knownURLs{$_->{url}} = $_ foreach @{$files};
}
# Fetch the new manifest.
my ($fh, $tmpManifest) = tempfile(UNLINK => 1);
system("$curl '$srcChannelURL/MANIFEST' > $tmpManifest") == 0 or die;
@ -35,7 +51,7 @@ my %narFiles;
my %localPaths;
my %patches;
my $version = readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
%localPaths = ();
%patches = (); # not supported yet
@ -66,23 +82,30 @@ while (my ($storePath, $files) = each %narFiles) {
system("$curl '$srcURL' > $dstFileTmp") == 0 or die "failed to download `$srcURL'";
rename($dstFileTmp, $dstFile) or die "cannot rename $dstFileTmp";
}
$file->{size} = stat($dstFile)->size or die "cannot get size of $dstFile";
my $hashFile = "$narPath/.hash.$dstName";
my $hash;
if (-e $hashFile) {
open HASH, "<$hashFile" or die;
$hash = <HASH>;
close HASH;
my $old = $knownURLs{$dstURL};
if (defined $old) {
$file->{size} = $old->{size};
$file->{hash} = $old->{hash};
} else {
$hash = `nix-hash --flat --type sha256 --base32 '$dstFile'` or die;
chomp $hash;
open HASH, ">$hashFile" or die;
print HASH $hash;
close HASH;
$file->{size} = stat($dstFile)->size or die "cannot get size of $dstFile";
my $hashFile = "$narPath/.hash.$dstName";
my $hash;
if (-e $hashFile) {
open HASH, "<$hashFile" or die;
$hash = <HASH>;
close HASH;
} else {
$hash = `nix-hash --flat --type sha256 --base32 '$dstFile'` or die;
chomp $hash;
open HASH, ">$hashFile" or die;
print HASH $hash;
close HASH;
}
$file->{hash} = "sha256:$hash";
}
$file->{hash} = "sha256:$hash";
}
}