danderson/channel-scripts
0
0
Fork 0
forked from the-distro/channel-scripts
Code Pull requests Activity

* Parallelize mirror-channel.pl by forking several worker processes to do

Browse source 
the compression / uploading.


git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@34738 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb
This commit is contained in:
Eelco Dolstra 2013-06-17 13:14:05 +00:00
parent 9ff5956af0
commit b6d6feaef3
1 changed files with 22 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
mirror-channel.pl
View file

@ -10,6 +10,8 @@ use Nix::Store;
use File::Basename;
use File::stat;
use Net::Amazon::S3;
use List::MoreUtils qw(part);
use Forks::Super;
if (scalar @ARGV < 4 || scalar @ARGV > 6) {
@ -19,6 +21,8 @@ if (scalar @ARGV < 4 || scalar @ARGV > 6) {
my $curl = "curl --location --silent --show-error --fail";
my $nrProcesses = 8;
my $srcChannelURL = $ARGV[0];
my $dstChannelPath = $ARGV[1];
my $bucketName = $ARGV[2];
@ -98,14 +102,15 @@ sub queryPathHash16 {
# to point to the mirror. Also fill in the size and hash fields in
# the manifest in order to be compatible with Nix < 0.13.
foreach my $storePath (permute(keys %narFiles)) {
sub mirrorStorePath {
my ($storePath) = @_;
my $nars = $narFiles{$storePath};
die if scalar @{$nars} != 1;
my $nar = $$nars[0];
my $pathHash = substr(basename($storePath), 0, 32);
my $narInfoFile = "$pathHash.narinfo";
print STDERR "checking $narInfoFile\n";
print STDERR "$$: checking $narInfoFile\n";
my $get = $bucket->get_key_filename("$pathHash.narinfo", "GET");
my $narInfo;
@ -117,7 +122,7 @@ foreach my $storePath (permute(keys %narFiles)) {
$nar->{narSize} = $narInfo->{narSize};
$nar->{url} = "$cacheURL/$narInfo->{url}";
} else {
my $dstFileTmp = "/tmp/nar";
my $dstFileTmp = "/tmp/nar.$$";
my $ext;
if (isValidPath($storePath) && queryPathHash16($storePath) eq $nar->{narHash}) {
@ -172,6 +177,20 @@ foreach my $storePath (permute(keys %narFiles)) {
}
# Spawn a bunch of children to mirror paths in parallel.
my $i = 0;
my @filesPerProcess = part { $i++ % $nrProcesses } permute(keys %narFiles);
for (my $n = 0; $n < $nrProcesses; $n++) {
my $pid = fork { sub => sub { mirrorStorePath($_) foreach @{$filesPerProcess[$n]}; } };
}
for (my $n = 0; $n < $nrProcesses; $n++) {
my $pid = wait;
die if $pid == -1;
die "worker process failed: $?" if $? != 0;
}
# Read all the old patches and propagate the useful ones. We use the
# file "all-patches" to keep track of all patches that have been
# generated in the past, so that patches are not lost if (for