* Enable asynchronous patch generation (i.e. it doesn't block
mirroring of the channel). git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@24922 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb
This commit is contained in:
Eelco Dolstra
parent
bf23243d12
commit
d715593122
|
|
@ -1,9 +1,21 @@
|
|||
# This script mirrors a remote Nix channel in the local filesystem.
|
||||
# It downloads the remote manifest, then any NAR files that are not
|
||||
# already available in the target directory. If $ENABLE_PATCHES is
|
||||
# set, it also generates patches between the NAR files in the old
|
||||
# version of the manifest and the new version. Because this script
|
||||
# can take a long time to finish, it uses a lock to guard against
|
||||
# concurrent updates, allowing it to be run periodically from a cron
|
||||
# job.
|
||||
|
||||
use strict;
|
||||
use readmanifest;
|
||||
use GeneratePatches;
|
||||
use File::Basename;
|
||||
use File::stat;
|
||||
use File::Temp qw/tempfile/;
|
||||
use File::Temp qw/tempfile tempdir/;
|
||||
use Fcntl ':flock';
|
||||
use POSIX qw(strftime);
|
||||
|
||||
|
||||
if (scalar @ARGV != 6 && scalar @ARGV != 7) {
|
||||
print STDERR "Syntax: perl mirror-channel.pl <src-channel-url> <dst-channel-dir> <nar-dir> <nar-url> <patches-dir> <patches-url> [<nix-exprs-url>]\n";
|
||||
|
|
@ -19,15 +31,21 @@ my $narURL = $ARGV[3];
|
|||
my $patchesPath = $ARGV[4];
|
||||
my $patchesURL = $ARGV[5];
|
||||
my $nixexprsURL = $ARGV[6] || "$srcChannelURL/nixexprs.tar.bz2";
|
||||
my $enablePatches = defined $ENV{'ENABLE_PATCHES'} && -e "$dstChannelPath/MANIFEST";
|
||||
|
||||
die "$dstChannelPath doesn't exist\n" unless -d $dstChannelPath;
|
||||
die "$narPath doesn't exist\n" unless -d $narPath;
|
||||
die "$patchesPath doesn't exist\n" unless -d $patchesPath;
|
||||
|
||||
my $tmpDir = tempdir("nix-mirror-XXXXXXX", TMPDIR => 1, CLEANUP => 1);
|
||||
print STDERR "$tmpDir\n";
|
||||
|
||||
|
||||
open LOCK, ">$dstChannelPath/.lock" or die;
|
||||
flock LOCK, LOCK_EX;
|
||||
|
||||
system("date");
|
||||
print STDERR "started mirroring at ", strftime("%a %b %e %H:%M:%S %Y", localtime), "\n";
|
||||
|
||||
|
||||
# Read the old manifest, if available.
|
||||
my %narFilesOld;
|
||||
|
|
@ -42,16 +60,15 @@ while (my ($storePath, $files) = each %narFilesOld) {
|
|||
$knownURLs{$_->{url}} = $_ foreach @{$files};
|
||||
}
|
||||
|
||||
|
||||
# Fetch the new manifest.
|
||||
my ($fh, $tmpManifest) = tempfile(UNLINK => 1);
|
||||
system("$curl '$srcChannelURL/MANIFEST' > $tmpManifest") == 0 or die;
|
||||
my $srcManifest = "$tmpDir/MANIFEST.src";
|
||||
system("$curl '$srcChannelURL/MANIFEST' > $srcManifest") == 0 or die;
|
||||
|
||||
|
||||
# Read the manifest.
|
||||
my %narFiles;
|
||||
my %localPaths;
|
||||
my %patches;
|
||||
|
||||
readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
|
||||
my (%narFiles, %localPaths, %patches);
|
||||
readManifest($srcManifest, \%narFiles, \%localPaths, \%patches);
|
||||
|
||||
%localPaths = ();
|
||||
%patches = (); # not supported yet
|
||||
|
|
@ -59,9 +76,11 @@ readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
|
|||
my $size = scalar (keys %narFiles);
|
||||
print "$size store paths in manifest\n";
|
||||
|
||||
|
||||
# Protect against Hydra problems that leave the channel empty.
|
||||
die "cowardly refusing to mirror an empty channel" if $size == 0;
|
||||
|
||||
|
||||
# Download every file that we don't already have, and update every URL
|
||||
# to point to the mirror. Also fill in the size and hash fields in
|
||||
# the manifest in order to be compatible with Nix < 0.13.
|
||||
|
|
@ -110,28 +129,68 @@ while (my ($storePath, $files) = each %narFiles) {
|
|||
}
|
||||
}
|
||||
|
||||
# Write the new manifest.
|
||||
writeManifest("$dstChannelPath/MANIFEST.tmp", \%narFiles, \%patches);
|
||||
|
||||
# Generate patches.
|
||||
if (defined $ENV{'ENABLE_PATCHES'} && -e "$dstChannelPath/MANIFEST.tmp") {
|
||||
system("generate-patches.pl $narPath $patchesPath $patchesURL $dstChannelPath/MANIFEST $dstChannelPath/MANIFEST.tmp") == 0 or die;
|
||||
# Read all the old patches and propagate the useful ones. We use the
|
||||
# file "all-patches" to keep track of all patches that have been
|
||||
# generated in the past, so that patches are not lost if (for
|
||||
# instance) a package temporarily disappears from the source channel,
|
||||
# or if multiple instances of this script are running concurrently.
|
||||
my (%dummy1, %dummy2, %allPatches);
|
||||
|
||||
sub readAllPatches {
|
||||
readManifest("$patchesPath/all-patches", \%dummy1, \%dummy2, \%allPatches)
|
||||
if -f "$patchesPath/all-patches";
|
||||
}
|
||||
|
||||
readAllPatches;
|
||||
|
||||
propagatePatches \%allPatches, \%narFiles, \%patches;
|
||||
propagatePatches \%patchesOld, \%narFiles, \%patches; # not really needed
|
||||
|
||||
|
||||
# Make the temporary manifest available.
|
||||
writeManifest("$dstChannelPath/MANIFEST.tmp", \%narFiles, \%patches);
|
||||
|
||||
rename("$dstChannelPath/MANIFEST.tmp", "$dstChannelPath/MANIFEST") or die;
|
||||
rename("$dstChannelPath/MANIFEST.tmp.bz2", "$dstChannelPath/MANIFEST.bz2") or die;
|
||||
|
||||
# Mirror nixexprs.tar.bz2.
|
||||
|
||||
# Mirror nixexprs.tar.bz2. This should really be done atomically with updating the manifest.
|
||||
my $tmpFile = "$dstChannelPath/.tmp.$$.nixexprs.tar.bz2";
|
||||
system("$curl '$nixexprsURL' > $tmpFile") == 0 or die "cannot download `$nixexprsURL'";
|
||||
rename($tmpFile, "$dstChannelPath/nixexprs.tar.bz2") or die "cannot rename $tmpFile";
|
||||
|
||||
# Remove ".hash.*" files corresponding to NARs that have been removed.
|
||||
#foreach my $fn (glob "$narPath/.hash.*") {
|
||||
# my $fn2 = $fn;
|
||||
# $fn2 =~ s/\.hash\.//;
|
||||
# if (! -e "$fn2") {
|
||||
# print STDERR "removing hash $fn\n";
|
||||
# unlink "$fn";
|
||||
# }
|
||||
#}
|
||||
|
||||
# Release the lock on the manifest to allow the manifest to be updated
|
||||
# by other runs of this script while we're generating patches.
|
||||
flock LOCK, LOCK_UN;
|
||||
|
||||
|
||||
if ($enablePatches) {
|
||||
|
||||
# Generate patches asynchronously. This can take a long time.
|
||||
generatePatches(\%narFilesOld, \%narFiles, \%allPatches, \%patches,
|
||||
$narPath, $patchesPath, $patchesURL, $tmpDir);
|
||||
|
||||
# Lock all-patches.
|
||||
open PLOCK, ">$patchesPath/all-patches.lock" or die;
|
||||
flock PLOCK, LOCK_EX;
|
||||
|
||||
# Update the list of all patches. We need to reread all-patches
|
||||
# and merge in our new patches because the file may have changed
|
||||
# in the meantime.
|
||||
readAllPatches;
|
||||
copyPatches \%patches, \%allPatches;
|
||||
writeManifest("$patchesPath/all-patches", {}, \%allPatches, 0);
|
||||
|
||||
# Reacquire the manifest lock.
|
||||
flock LOCK, LOCK_EX;
|
||||
|
||||
# Rewrite the manifest. We have to reread it and propagate all
|
||||
# patches because it may have changed in the meantime.
|
||||
readManifest("$dstChannelPath/MANIFEST", \%narFiles, \%localPaths, \%patches);
|
||||
|
||||
propagatePatches \%allPatches, \%narFiles, \%patches;
|
||||
|
||||
writeManifest("$dstChannelPath/MANIFEST", \%narFiles, \%patches);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue