the-distro/channel-scripts
1
0
Fork 1
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity

Merge pull request #32 from garbas/push-channels-to-s3-bucket

Browse source 
Push channels to s3 bucket
This commit is contained in:
Eelco Dolstra 2020-03-05 21:57:09 +01:00 committed by GitHub
commit abb56de99f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
mirror-nixos-branch.pl
View file

@ -26,9 +26,9 @@ my $channelDirRel = $channelName eq "nixpkgs-unstable" ? "nixpkgs" : "$1/$2";
# Configuration. # Configuration.
my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; my $TMPDIR = $ENV{'TMPDIR'} // "/tmp";
my $channelsDir = "/home/hydra-mirror/channels";
my $filesCache = "${TMPDIR}/nixos-files.sqlite"; my $filesCache = "${TMPDIR}/nixos-files.sqlite";
my $bucketName = "nix-releases"; my $bucketReleasesName = "nix-releases";
my $bucketChannelsName = "nix-channels";
$ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels"; $ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels";
@ -44,7 +44,8 @@ my $s3 = Net::Amazon::S3->new(
host => "s3-eu-west-1.amazonaws.com", host => "s3-eu-west-1.amazonaws.com",
}); });
my $bucket = $s3->bucket($bucketName) or die; my $bucketReleases = $s3->bucket($bucketReleasesName) or die;
my $bucketChannels = $s3->bucket($bucketChannelsName) or die;
sub fetch { sub fetch {
@ -73,8 +74,7 @@ my $rev = $evalInfo->{jobsetevalinputs}->{nixpkgs}->{revision} or die;
print STDERR "release is ‘$releaseName’ (build $releaseId), eval is $evalId, prefix is $releasePrefix, Git commit is $rev\n"; print STDERR "release is ‘$releaseName’ (build $releaseId), eval is $evalId, prefix is $releasePrefix, Git commit is $rev\n";
# Guard against the channel going back in time. # Guard against the channel going back in time.
my @curReleaseUrl = split(/\//, read_file("$channelsDir/$channelName", err_mode => 'quiet') // ""); my $curRelease = $bucketChannels.get_key($channelName) or "";
my $curRelease = pop @curReleaseUrl;
$! = 0; # Clear errno to avoid reporting non-fork/exec-related issues $! = 0; # Clear errno to avoid reporting non-fork/exec-related issues
my $d = `NIX_PATH= nix-instantiate --eval -E "builtins.compareVersions (builtins.parseDrvName \\"$curRelease\\").version (builtins.parseDrvName \\"$releaseName\\").version"`; my $d = `NIX_PATH= nix-instantiate --eval -E "builtins.compareVersions (builtins.parseDrvName \\"$curRelease\\").version (builtins.parseDrvName \\"$releaseName\\").version"`;
if ($? != 0) { if ($? != 0) {
@ -89,7 +89,7 @@ if ($d == 1) {
exit if $d == 0; exit if $d == 0;
if ($bucket->head_key("$releasePrefix")) { if ($bucketReleases->head_key("$releasePrefix")) {
print STDERR "release already exists\n"; print STDERR "release already exists\n";
} else { } else {
my $tmpDir = "$TMPDIR/release-$channelName/$releaseName"; my $tmpDir = "$TMPDIR/release-$channelName/$releaseName";
@ -190,12 +190,12 @@ if ($bucket->head_key("$releasePrefix")) {
my $basename = basename $fn; my $basename = basename $fn;
my $key = "$releasePrefix/" . $basename; my $key = "$releasePrefix/" . $basename;
unless (defined $bucket->head_key($key)) { unless (defined $bucketReleases->head_key($key)) {
print STDERR "mirroring $fn to s3://$bucketName/$key...\n"; print STDERR "mirroring $fn to s3://$bucketReleasesName/$key...\n";
$bucket->add_key_filename( $bucketReleases->add_key_filename(
$key, $fn, $key, $fn,
{ content_type => $fn =~ /.sha256|src-url|binary-cache-url|git-revision/ ? "text/plain" : "application/octet-stream" }) { content_type => $fn =~ /.sha256|src-url|binary-cache-url|git-revision/ ? "text/plain" : "application/octet-stream" })
or die $bucket->err . ": " . $bucket->errstr; or die $bucketReleases->err . ": " . $bucketReleases->errstr;
} }
next if $basename =~ /.sha256$/; next if $basename =~ /.sha256$/;
@ -211,40 +211,18 @@ if ($bucket->head_key("$releasePrefix")) {
$html .= "</tbody></table></body></html>"; $html .= "</tbody></table></body></html>";
$bucket->add_key($releasePrefix, $html, $bucketReleases->add_key($releasePrefix, $html,
{ content_type => "text/html" }) { content_type => "text/html" })
or die $bucket->err . ": " . $bucket->errstr; or die $bucketReleases->err . ": " . $bucketReleases->errstr;
File::Path::remove_tree($tmpDir); File::Path::remove_tree($tmpDir);
} }
# Prevent concurrent writes to the channels directory.
open(my $lockfile, ">>", "$channelsDir/.htaccess.lock");
flock($lockfile, LOCK_EX) or die "cannot acquire channels lock\n";
# Update the channel.
my $htaccess = "$channelsDir/.htaccess-$channelName";
my $target = "https://releases.nixos.org/$releasePrefix";
write_file($htaccess,
"Redirect /channels/$channelName $target\n" .
"Redirect /releases/nixos/channels/$channelName $target\n");
my $channelLink = "$channelsDir/$channelName";
if ((read_file($channelLink, err_mode => 'quiet') // "") ne $target) {
write_file("$channelLink.tmp", "$target");
rename("$channelLink.tmp", $channelLink) or die;
}
system("cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess.tmp") == 0 or die;
rename("$channelsDir/.htaccess.tmp", "$channelsDir/.htaccess") or die;
# Update the nixos-* branch in the nixpkgs repo. Also update the # Update the nixos-* branch in the nixpkgs repo. Also update the
# nixpkgs-channels repo for compatibility. # nixpkgs-channels repo for compatibility.
system("git remote update origin >&2") == 0 or die; system("git remote update origin >&2") == 0 or die;
system("git push origin $rev:refs/heads/$channelName >&2") == 0 or die; system("git push origin $rev:refs/heads/$channelName >&2") == 0 or die;
system("git push channels $rev:refs/heads/$channelName >&2") == 0 or die; system("git push channels $rev:refs/heads/$channelName >&2") == 0 or die;
flock($lockfile, LOCK_UN) or die "cannot release channels lock\n"; # Update channel on channels.nixos.org
$bucketChannels->add_key($channelsDir, $target, { "x-amz-website-redirect-location header" => $target });
# Upload to nixos.org.
system("rsync -avx $channelsDir/ hydra-mirror\@nixos.org:/releases/channels/") == 0 or die;