From ac8e5258354839fcb14e331c4879f5fd0ab4350d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2013 12:04:29 +0000 Subject: [PATCH] * Support multiple NixOS branches. git-svn-id: https://nixos.org/repos/nix/release/trunk/channels@34774 70bd8c7a-acb8-0310-9f0d-9cc1c95dcdbb --- mirror-nixos-branch.sh | 86 +++++++++++++++++++++++++++++++++++++++ mirror-nixos-unstable.sh | 88 ++++++---------------------------------- mirror-nixpkgs.sh | 28 +++++++------ 3 files changed, 115 insertions(+), 87 deletions(-) create mode 100755 mirror-nixos-branch.sh diff --git a/mirror-nixos-branch.sh b/mirror-nixos-branch.sh new file mode 100755 index 0000000..9f8f725 --- /dev/null +++ b/mirror-nixos-branch.sh @@ -0,0 +1,86 @@ +#! /bin/sh -e + +branch="$1" +jobset="$2" + +if [ -z "$branch" -o -z "$jobset" ]; then + echo "Usage: $0 BRANCH-NAME JOBSET-NAME" >&2 + exit 1 +fi + +releaseUrl="http://hydra.nixos.org/job/nixos/$jobset/tested/latest-finished" +releasesDir="/data/releases/nixos/$branch" +channelsDir=/data/releases/channels +channelName=nixos-"$branch" + +curl="curl --silent --show-error --fail" +wget="wget --no-verbose --content-disposition" + +json=$($curl -L -H 'Accept: application/json' $releaseUrl) + +releaseId=$(echo "$json" | json id) +if [ -z "$releaseId" ]; then echo "Failed to get release id"; exit 1; fi + +release=$(echo "$json" | json nixname) +if [ -z "$release" ]; then echo "Failed to get release"; exit 1; fi + +url=$($curl --head http://hydra.nixos.org/build/$releaseId/eval | sed 's/Location: \(.*\)\r/\1/; t; d') +if [ -z "$url" ]; then exit 1; fi + +releaseDir=$releasesDir/$release + +echo "release is ‘$release’ (build $releaseId), eval is ‘$url’, dir is ‘$releaseDir’" >&2 + +if [ -d $releaseDir ]; then + echo "release already exists" >&2 +else + tmpDir="$(dirname $releaseDir)/.tmp-$release-$$" + mkdir -p $tmpDir + + echo $url > $tmpDir/src-url + + # Copy the manual. + $curl -L $url/job/nixos.manual/output/out | bzip2 -d | nix-store --restore $tmpDir/foo + mv $tmpDir/foo/share/doc/nixos $tmpDir/manual + rm -rf $tmpDir/foo + ln -s manual.html $tmpDir/manual/index.html + + $wget --directory=$tmpDir $url/job/nixos.iso_minimal.i686-linux/download + $wget --directory=$tmpDir $url/job/nixos.iso_minimal.x86_64-linux/download + $wget --directory=$tmpDir $url/job/nixos.iso_graphical.i686-linux/download + $wget --directory=$tmpDir $url/job/nixos.iso_graphical.x86_64-linux/download + $wget --directory=$tmpDir $url/job/nixos.ova.i686-linux/download + $wget --directory=$tmpDir $url/job/nixos.ova.x86_64-linux/download + + perl -w ./mirror-channel.pl "$url/channel" "$tmpDir" \ + nix-cache http://cache.nixos.org \ + /data/releases/patches/all-patches "$url/job/nixos.channel/download/1" + + # Generate the programs.sqlite database and put it in nixexprs.tar.xz. + mkdir $tmpDir/unpack + tar xfJ $tmpDir/nixexprs.tar.xz -C $tmpDir/unpack + exprDir=$(echo $tmpDir/unpack/*) + ./generate-programs-index.pl "$exprDir" "$exprDir/programs.sqlite" + tar cfJ $tmpDir/nixexprs.tar.xz -C $tmpDir/unpack "$(basename "$exprDir")" + rm -rf $tmpDir/unpack + + mv $tmpDir $releaseDir +fi + +# Copy over to nixos.org. +cd "$releasesDir" +rsync -avR . hydra-mirror@nixos.org:"$releasesDir" --exclude .htaccess --exclude ".tmp.*" --delete >&2 + +# Update the channel. +htaccess=$channelsDir/.htaccess-$channelName +echo "Redirect /channels/$channelName http://nixos.org/releases/nixos/$branch/$release" > $htaccess.tmp +echo "Redirect /releases/nixos/channels/$channelName http://nixos.org/releases/nixos/$branch/$release" >> $htaccess.tmp # obsolete +mv $htaccess.tmp $htaccess +ln -sfn $releaseDir $channelsDir/$channelName # dummy symlink + +flock -x $channelsDir/.htaccess.lock -c "cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess" + +cd "$channelsDir" +rsync -avR . hydra-mirror@nixos.org:"$channelsDir" --delete >&2 + +echo "$releaseDir" diff --git a/mirror-nixos-unstable.sh b/mirror-nixos-unstable.sh index d2bf5b6..90d777e 100755 --- a/mirror-nixos-unstable.sh +++ b/mirror-nixos-unstable.sh @@ -1,90 +1,28 @@ #! /bin/sh -e -releaseUrl=http://hydra.nixos.org/job/nixos/trunk-combined/tested/latest-finished -releasesDir=/data/releases/nixos -channelsDir=/data/releases/channels -channelName=nixos-unstable -curl="curl --silent --show-error --fail" -wget="wget --no-verbose --content-disposition" +releaseDir=$(./mirror-nixos-branch.sh unstable trunk-combined) +echo $releaseDir -json=$($curl -L -H 'Accept: application/json' $releaseUrl) - -releaseId=$(echo "$json" | json id) -if [ -z "$releaseId" ]; then echo "Failed to get release id"; exit 1; fi - -release=$(echo "$json" | json nixname) -if [ -z "$release" ]; then echo "Failed to get release"; exit 1; fi - -url=$($curl --head http://hydra.nixos.org/build/$releaseId/eval | sed 's/Location: \(.*\)\r/\1/; t; d') -if [ -z "$url" ]; then exit 1; fi - -releaseDir=$releasesDir/$release - -echo "release is ‘$release’ (build $releaseId), eval is ‘$url’, dir is ‘$releaseDir’" - -if [ -d $releaseDir ]; then - echo "release already exists" -else - tmpDir=$releasesDir/.tmp-$release-$$ - mkdir -p $tmpDir - - echo $url > $tmpDir/src-url - - # Copy the manual. - $curl -L $url/job/nixos.manual/output/out | bzip2 -d | nix-store --restore $tmpDir/foo - mv $tmpDir/foo/share/doc/nixos $tmpDir/manual - rm -rf $tmpDir/foo - ln -s manual.html $tmpDir/manual/index.html - - $wget --directory=$tmpDir $url/job/nixos.iso_minimal.i686-linux/download - $wget --directory=$tmpDir $url/job/nixos.iso_minimal.x86_64-linux/download - $wget --directory=$tmpDir $url/job/nixos.iso_graphical.i686-linux/download - $wget --directory=$tmpDir $url/job/nixos.iso_graphical.x86_64-linux/download - $wget --directory=$tmpDir $url/job/nixos.ova.i686-linux/download - $wget --directory=$tmpDir $url/job/nixos.ova.x86_64-linux/download - - perl -w ./mirror-channel.pl "$url/channel" "$tmpDir" \ - nix-cache http://cache.nixos.org \ - /data/releases/patches/all-patches "$url/job/nixos.channel/download/1" - - # Generate the programs.sqlite database and put it in nixexprs.tar.xz. - mkdir $tmpDir/unpack - tar xfJ $tmpDir/nixexprs.tar.xz -C $tmpDir/unpack - exprDir=$(echo $tmpDir/unpack/*) - ./generate-programs-index.pl "$exprDir" "$exprDir/programs.sqlite" - tar cfJ $tmpDir/nixexprs.tar.xz -C $tmpDir/unpack "$(basename "$exprDir")" - rm -rf $tmpDir/unpack - - mv $tmpDir $releaseDir -fi - -htaccess=$channelsDir/.htaccess-nixos -echo "Redirect /channels/$channelName http://nixos.org/releases/nixos/$release" > $htaccess.tmp -echo "Redirect /releases/nixos/channels/$channelName http://nixos.org/releases/nixos/$release" >> $htaccess.tmp -ln -sfn $releaseDir $channelsDir/$channelName # dummy symlink -mv $htaccess.tmp $htaccess +release=$(basename $releaseDir) # Generate a .htaccess with some symbolic redirects to the latest ISOs. -htaccess=$releasesDir/.htaccess +htaccess=/data/releases/nixos/.htaccess -echo "Redirect /releases/nixos/latest http://nixos.org/releases/nixos/$release" > $htaccess.tmp +baseURL="http://nixos.org/releases/nixos/unstable/$release" +echo "Redirect /releases/nixos/latest $baseURL" > $htaccess.tmp fn=$(cd $releaseDir && echo nixos-minimal-*-i686-linux.iso) -echo "Redirect /releases/nixos/latest-iso-minimal-i686-linux http://nixos.org/releases/nixos/$release/$fn" >> $htaccess.tmp +echo "Redirect /releases/nixos/latest-iso-minimal-i686-linux $baseURL/$fn" >> $htaccess.tmp fn=$(cd $releaseDir && echo nixos-minimal-*-x86_64-linux.iso) -echo "Redirect /releases/nixos/latest-iso-minimal-x86_64-linux http://nixos.org/releases/nixos/$release/$fn" >> $htaccess.tmp +echo "Redirect /releases/nixos/latest-iso-minimal-x86_64-linux $baseURL/$fn" >> $htaccess.tmp fn=$(cd $releaseDir && echo nixos-graphical-*-i686-linux.iso) -echo "Redirect /releases/nixos/latest-iso-graphical-i686-linux http://nixos.org/releases/nixos/$release/$fn" >> $htaccess.tmp +echo "Redirect /releases/nixos/latest-iso-graphical-i686-linux $baseURL/$fn" >> $htaccess.tmp fn=$(cd $releaseDir && echo nixos-graphical-*-x86_64-linux.iso) -echo "Redirect /releases/nixos/latest-iso-graphical-x86_64-linux http://nixos.org/releases/nixos/$release/$fn" >> $htaccess.tmp +echo "Redirect /releases/nixos/latest-iso-graphical-x86_64-linux $baseURL/$fn" >> $htaccess.tmp fn=$(cd $releaseDir && echo nixos-*-i686-linux.ova) -echo "Redirect /releases/nixos/latest-ova-i686-linux http://nixos.org/releases/nixos/$release/$fn" >> $htaccess.tmp +echo "Redirect /releases/nixos/latest-ova-i686-linux $baseURL/$fn" >> $htaccess.tmp fn=$(cd $releaseDir && echo nixos-*-x86_64-linux.ova) -echo "Redirect /releases/nixos/latest-ova-x86_64-linux http://nixos.org/releases/nixos/$release/$fn" >> $htaccess.tmp +echo "Redirect /releases/nixos/latest-ova-x86_64-linux $baseURL/$fn" >> $htaccess.tmp mv $htaccess.tmp $htaccess -# Copy over to nixos.org. -cd /data/releases -rsync -avR nixos hydra-mirror@nixos.org:/data/releases --exclude nixos/.htaccess --delete -rsync -avR channels/.htaccess-nixos channels/nixos-unstable nixos/.htaccess hydra-mirror@nixos.org:/data/releases -ssh nixos.org "flock -x $channelsDir/.htaccess.lock -c \"cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess\"" +rsync -av $htaccess hydra-mirror@nixos.org:$htaccess >&2 diff --git a/mirror-nixpkgs.sh b/mirror-nixpkgs.sh index 4bbac28..52ca7d5 100755 --- a/mirror-nixpkgs.sh +++ b/mirror-nixpkgs.sh @@ -19,10 +19,10 @@ if [ -z "$url" ]; then exit 1; fi releaseDir=$releasesDir/$release -echo "release is ‘$release’ (build $releaseId), eval is ‘$url’, dir is ‘$releaseDir’" +echo "release is ‘$release’ (build $releaseId), eval is ‘$url’, dir is ‘$releaseDir’" >&2 if [ -d $releaseDir ]; then - echo "release already exists" + echo "release already exists" >&2 else tmpDir=$releasesDir/.tmp-$release-$$ mkdir -p $tmpDir @@ -42,14 +42,18 @@ else mv $tmpDir $releaseDir fi -htaccess=$channelsDir/.htaccess-nixpkgs -echo "Redirect /channels/$channelName http://nixos.org/releases/nixpkgs/$release" > $htaccess.tmp -echo "Redirect /releases/nixpkgs/channels/$channelName http://nixos.org/releases/nixpkgs/$release" >> $htaccess.tmp -ln -sfn $releaseDir $channelsDir/$channelName # dummy symlink -mv $htaccess.tmp $htaccess - # Copy over to nixos.org. -cd /data/releases -rsync -avR nixpkgs hydra-mirror@nixos.org:/data/releases --exclude nixpkgs/.htaccess --delete -rsync -avR channels/.htaccess-nixpkgs channels/nixpkgs-unstable hydra-mirror@nixos.org:/data/releases -ssh nixos.org "flock -x $channelsDir/.htaccess.lock -c \"cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess\"" +cd "$releasesDir" +rsync -avR . hydra-mirror@nixos.org:"$releasesDir" --delete >&2 + +# Update the channel. +htaccess=$channelsDir/.htaccess-$channelName +echo "Redirect /channels/$channelName http://nixos.org/releases/nixpkgs/$release" > $htaccess.tmp +echo "Redirect /releases/nixpkgs/channels/$channelName http://nixos.org/releases/nixpkgs/$release" >> $htaccess.tmp # obsolete +mv $htaccess.tmp $htaccess +ln -sfn $releaseDir $channelsDir/$channelName # dummy symlink + +flock -x $channelsDir/.htaccess.lock -c "cat $channelsDir/.htaccess-nix* > $channelsDir/.htaccess" + +cd "$channelsDir" +rsync -avR . hydra-mirror@nixos.org:"$channelsDir" --delete >&2