diff --git a/mirror-nixos-branch.pl b/mirror-nixos-branch.pl index 682bbd7..862205c 100755 --- a/mirror-nixos-branch.pl +++ b/mirror-nixos-branch.pl @@ -10,16 +10,16 @@ use JSON::PP; use LWP::UserAgent; use List::MoreUtils qw(uniq); -my $branch = $ARGV[0]; -my $jobset = $ARGV[1]; +my $channelName = $ARGV[0]; +my $releaseUrl = $ARGV[1]; my $isMainRelease = ($ARGV[2] // 0) eq 1; -die "Usage: $0 BRANCH-NAME JOBSET_NAME [IS-MAIN-RELEASE]\n" unless defined $branch && defined $jobset; +die "Usage: $0 CHANNEL-NAME RELEASE-URL [IS-MAIN-RELEASE]\n" unless defined $channelName && defined $releaseUrl; -my $releaseUrl = "https://hydra.nixos.org/job/nixos/$jobset/tested/latest-finished"; -my $releasesDir = "/data/releases/nixos/$branch"; +$channelName =~ /^([a-z]+)-(.*)$/ or die; +my $channelDirRel = $channelName eq "nixpkgs-unstable" ? "nixpkgs" : "$1/$2"; +my $releasesDir = "/data/releases/$channelDirRel"; my $channelsDir = "/data/releases/channels"; -my $channelName = "nixos-$branch"; $ENV{'GIT_DIR'} = "/home/hydra-mirror/nixpkgs-channels"; @@ -74,11 +74,15 @@ if (-d $releaseDir) { } # Copy the manual. + my $manualJob = $channelName =~ /nixos/ ? "nixos.manual.x86_64-linux" : "manual"; + my $manualDir = $channelName =~ /nixos/ ? "nixos" : "nixpkgs"; if (! -e "$tmpDir/manual") { - my $manualInfo = decode_json(fetch("$evalUrl/job/nixos.manual.x86_64-linux", 'application/json')); + my $manualInfo = decode_json(fetch("$evalUrl/job/$manualJob", 'application/json')); my $manualPath = $manualInfo->{buildoutputs}->{out}->{path} or die; system("nix-store", "-r", $manualPath) == 0 or die "unable to fetch $manualPath\n"; - system("cp", "-rd", "$manualPath/share/doc/nixos", "$tmpDir/manual") == 0 or die "unable to copy manual from $manualPath"; + system("cp", "-rd", "$manualPath/share/doc/$manualDir", "$tmpDir/manual") == 0 or die "unable to copy manual from $manualPath"; + system("chmod", "-R", "u+w", "$tmpDir/manual"); + symlink("manual.html", "$tmpDir/manual/index.html") unless -e "$tmpDir/manual/index.html"; } sub downloadFile { @@ -108,21 +112,26 @@ if (-d $releaseDir) { write_file("$dstFile.sha256", $sha256_expected); } - downloadFile("nixos.channel", "nixexprs.tar.xz"); - downloadFile("nixos.iso_minimal.x86_64-linux"); + if ($channelName =~ /nixos/) { + downloadFile("nixos.channel", "nixexprs.tar.xz"); + downloadFile("nixos.iso_minimal.x86_64-linux"); - if ($branch !~ /-small/) { - downloadFile("nixos.iso_minimal.i686-linux"); - downloadFile("nixos.iso_graphical.x86_64-linux"); - downloadFile("nixos.iso_graphical.i686-linux"); - downloadFile("nixos.ova.x86_64-linux"); - downloadFile("nixos.ova.i686-linux"); + if ($channelName !~ /-small/) { + downloadFile("nixos.iso_minimal.i686-linux"); + downloadFile("nixos.iso_graphical.x86_64-linux"); + downloadFile("nixos.iso_graphical.i686-linux"); + downloadFile("nixos.ova.x86_64-linux"); + downloadFile("nixos.ova.i686-linux"); + } + + } else { + downloadFile("tarball", "nixexprs.tar.xz"); } # Make "github-link" a redirect to the GitHub history of this # release. write_file("$tmpDir/.htaccess", - "Redirect /releases/nixos/$branch/$releaseName/github-link https://github.com/NixOS/nixpkgs-channels/commits/$rev\n"); + "Redirect /releases/$channelDirRel/$releaseName/github-link https://github.com/NixOS/nixpkgs-channels/commits/$rev\n"); write_file("$tmpDir/github-link", ""); # FIXME: Generate the programs.sqlite database and put it in nixexprs.tar.xz. @@ -137,8 +146,8 @@ flock($lockfile, LOCK_EX) or die "cannot acquire channels lock\n"; # Update the channel. my $htaccess = "$channelsDir/.htaccess-$channelName"; write_file($htaccess, - "Redirect /channels/$channelName /releases/nixos/$branch/$releaseName\n" . - "Redirect /releases/nixos/channels/$channelName /releases/nixos/$branch/$releaseName\n"); + "Redirect /channels/$channelName /releases/$channelDirRel/$releaseName\n" . + "Redirect /releases/nixos/channels/$channelName /releases/$channelDirRel/$releaseName\n"); my $channelLink = "$channelsDir/$channelName"; unlink("$channelLink.tmp"); @@ -156,7 +165,7 @@ system("git push channels $rev:refs/heads/$channelName >&2") == 0 or die; if ($isMainRelease) { - my $baseURL = "/releases/nixos/$branch/$releaseName"; + my $baseURL = "/releases/$channelDirRel/$releaseName"; my $res = "Redirect /releases/nixos/latest $baseURL\n"; sub add { diff --git a/mirror-nixpkgs.sh b/mirror-nixpkgs.sh deleted file mode 100755 index c5cc0e9..0000000 --- a/mirror-nixpkgs.sh +++ /dev/null @@ -1,74 +0,0 @@ -#! /bin/sh -e - -releaseUrl=http://hydra.nixos.org/job/nixpkgs/trunk/unstable/latest-finished -releasesDir=/data/releases/nixpkgs -channelsDir=/data/releases/channels -channelName=nixpkgs-unstable -export GIT_DIR=/home/hydra-mirror/nixpkgs-channels -curl="curl --silent --show-error --fail" - -json=$($curl -L -H 'Accept: application/json' $releaseUrl) - -releaseId=$(echo "$json" | json id) -if [ -z "$releaseId" ]; then echo "Failed to get release id"; exit 1; fi - -release=$(echo "$json" | json nixname) -if [ -z "$release" ]; then echo "Failed to get release"; exit 1; fi - -url=$($curl --head http://hydra.nixos.org/build/$releaseId/eval | sed 's/Location: \(.*\)\r/\1/; t; d') -if [ -z "$url" ]; then exit 1; fi - -releaseDir=$releasesDir/$release - -echo "release is ‘$release’ (build $releaseId), eval is ‘$url’, dir is ‘$releaseDir’" >&2 - -# Figure out the Git revision from which this release was -# built. FIXME: get this from Hydra directly. -git remote update nixpkgs >&2 -shortRev=$(echo "$release" | sed 's/.*\.//') -rev=$(git rev-parse "$shortRev") -echo "revision is $rev" >&2 - -if [ -d $releaseDir ]; then - echo "release already exists" >&2 -else - tmpDir=$releasesDir/.tmp-$release-$$ - mkdir -p $tmpDir - - echo $url > $tmpDir/src-url - - perl -w ./mirror-channel.pl "$url/channel" "$tmpDir" \ - nix-cache https://cache.nixos.org \ - "$url/job/tarball/download/1" - - # Extract the manual. - $curl -L $url/job/manual/output/out | bzip2 -d | nix-store --restore $tmpDir/foo - mv $tmpDir/foo/share/doc/nixpkgs $tmpDir/manual - rm -rf $tmpDir/foo - ln -s manual.html $tmpDir/manual/index.html - - mv $tmpDir $releaseDir -fi - -# Prevent concurrent writes to the channels and the Git clone. -exec 10>$channelsDir/.htaccess.lock -flock 10 - -# Copy over to nixos.org. -cd "$releasesDir" -rsync -avR . hydra-mirror@nixos.org:"$releasesDir" --delete >&2 - -# Update the channel. -htaccess=$channelsDir/.htaccess-$channelName -echo "Redirect /channels/$channelName /releases/nixpkgs/$release" > $htaccess.tmp -echo "Redirect /releases/nixpkgs/channels/$channelName /releases/nixpkgs/$release" >> $htaccess.tmp # obsolete -mv $htaccess.tmp $htaccess -ln -sfn $releaseDir $channelsDir/$channelName # dummy symlink - -cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess - -cd "$channelsDir" -rsync -avR . hydra-mirror@nixos.org:"$channelsDir" --delete >&2 - -# Update the nixpkgs-channels repo. -git push nixpkgs-channels "$rev:refs/heads/nixpkgs-unstable" >&2