the-distro/channel-scripts
1
0
Fork 1
Code Issues 8 Pull requests Packages Projects Releases Wiki Activity

* Fix manifest generation: changes to the NAR info structures in the

Browse source 
children need to be communicated back to the parent.


git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@34739 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb
This commit is contained in:
Eelco Dolstra 2013-06-17 19:28:59 +00:00
parent b6d6feaef3
commit 1720cb8877
1 changed files with 13 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
mirror-channel.pl
View file

@ -11,7 +11,7 @@ use File::Basename;
use File::stat; use File::stat;
use Net::Amazon::S3; use Net::Amazon::S3;
use List::MoreUtils qw(part); use List::MoreUtils qw(part);
use Forks::Super; use Forks::Super 'bg_eval';
if (scalar @ARGV < 4 || scalar @ARGV > 6) { if (scalar @ARGV < 4 || scalar @ARGV > 6) {
@ -45,7 +45,7 @@ my $s3 = Net::Amazon::S3->new(
retry => 1, retry => 1,
}); });
my $bucket = $s3->bucket("nix-cache") or die; my $bucket = $s3->bucket($bucketName) or die;
# Fetch the manifest. # Fetch the manifest.
@ -103,7 +103,7 @@ sub queryPathHash16 {
# the manifest in order to be compatible with Nix < 0.13. # the manifest in order to be compatible with Nix < 0.13.
sub mirrorStorePath { sub mirrorStorePath {
my ($storePath) = @_; my ($storePath, $res) = @_;
my $nars = $narFiles{$storePath}; my $nars = $narFiles{$storePath};
die if scalar @{$nars} != 1; die if scalar @{$nars} != 1;
my $nar = $$nars[0]; my $nar = $$nars[0];
@ -174,20 +174,25 @@ sub mirrorStorePath {
$bucket->add_key($narInfoFile, $info) or die "failed to upload $narInfoFile to S3\n"; $bucket->add_key($narInfoFile, $info) or die "failed to upload $narInfoFile to S3\n";
} }
$res->{$storePath} = $nar;
} }
# Spawn a bunch of children to mirror paths in parallel. # Spawn a bunch of children to mirror paths in parallel.
my $i = 0; my $i = 0;
my @filesPerProcess = part { $i++ % $nrProcesses } permute(keys %narFiles); my @filesPerProcess = part { $i++ % $nrProcesses } permute(keys %narFiles);
my @results;
for (my $n = 0; $n < $nrProcesses; $n++) { for (my $n = 0; $n < $nrProcesses; $n++) {
my $pid = fork { sub => sub { mirrorStorePath($_) foreach @{$filesPerProcess[$n]}; } }; push @results, bg_eval { my $res = {}; mirrorStorePath($_, $res) foreach @{$filesPerProcess[$n]}; return $res; }
} }
for (my $n = 0; $n < $nrProcesses; $n++) {
my $pid = wait; # Get the updated NAR info from the children so we can update the manifest.
die if $pid == -1; foreach my $r (@results) {
die "worker process failed: $?" if $? != 0; while (my ($storePath, $nar) = each %$r) {
$narFiles{$storePath} = [$nar];
}
} }