From 29846d7f2f22d4f2ae1b87b8b5c8f098c4941b91 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 4 Dec 2011 22:01:53 +0100 Subject: [PATCH] Do incremental SVN checkouts In hydra-evaluator, reuse an SVN working copy between runs (similar to what we do with Git and other input types). This reduces network traffic in the common case. Also, don't use nix-prefetch-svn. It doesn't do anything useful. --- src/lib/Hydra/Helper/AddBuilds.pm | 53 +++++++++++++-------- src/script/Makefile.am | 5 -- src/script/nix-prefetch-svn | 79 ------------------------------- 3 files changed, 33 insertions(+), 104 deletions(-) delete mode 100755 src/script/nix-prefetch-svn diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index cd5a93b9..40033d3b 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -11,6 +11,7 @@ use Digest::SHA qw(sha256_hex); use File::Basename; use File::stat; use File::Path; +use File::Temp; our @ISA = qw(Exporter); our @EXPORT = qw( @@ -182,6 +183,9 @@ sub fetchInputSVN { die unless $revision =~ /^\d+$/; + # Do we already have this revision in the store? + # !!! This needs to take $checkout into account! Otherwise "svn" + # and "svn-checkout" inputs can get mixed up. (my $cachedInput) = $db->resultset('CachedSubversionInputs')->search( {uri => $uri, revision => $revision}); @@ -189,21 +193,31 @@ sub fetchInputSVN { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; } else { - - # Then download this revision into the store. - print STDERR "checking out Subversion input ", $name, " from $uri revision $revision\n"; - $ENV{"NIX_HASH_ALGO"} = "sha256"; - $ENV{"PRINT_PATH"} = "1"; - $ENV{"NIX_PREFETCH_SVN_LEAVE_DOT_SVN"} = "$checkout"; - - (my $res, $stdout, $stderr) = captureStdoutStderr(600, - ("nix-prefetch-svn", $uri, $revision)); - die "Cannot check out Subversion repository `$uri':\n$stderr" unless $res; - ($sha256, $storePath) = split ' ', $stdout; + # No, do a checkout. The working copy is reused between + # invocations to speed things up. + mkpath(scmPath . "/svn"); + my $wcPath = scmPath . "/svn" . sha256_hex($uri) . "/svn-checkout"; + + print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n"; + + (my $res, $stdout, $stderr) = captureStdoutStderr(600, + ("svn", "checkout", $uri, "-r", $revision, $wcPath)); + + if ($checkout) { + $storePath = addToStore($wcPath, 1, "sha256"); + } else { + # Hm, if the Nix Perl bindings supported filters in + # addToStore(), then we wouldn't need to make a copy here. + my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die; + (system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed"; + $storePath = addToStore("$tmpDir/svn-export", 1, "sha256"); + } + + $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://; txn_do($db, sub { - $db->resultset('CachedSubversionInputs')->create( + $db->resultset('CachedSubversionInputs')->update_or_create( { uri => $uri , revision => $revision , sha256hash => $sha256 @@ -311,9 +325,8 @@ sub fetchInputGit { my $sha256; my $storePath; - my $clonePath; mkpath(scmPath); - $clonePath = scmPath . "/" . sha256_hex($uri); + my $clonePath = scmPath . "/" . sha256_hex($uri); my $stdout; my $stderr; if (! -d $clonePath) { @@ -411,9 +424,9 @@ sub fetchInputBazaar { my $storePath; my $stdout; my $stderr; - my $clonePath; + mkpath(scmPath); - $clonePath = scmPath . "/" . sha256_hex($uri); + my $clonePath = scmPath . "/" . sha256_hex($uri); if (! -d $clonePath) { (my $res, $stdout, $stderr) = captureStdoutStderr(600, @@ -484,9 +497,9 @@ sub fetchInputHg { # init local hg clone my $stdout; my $stderr; - my $clonePath; + mkpath(scmPath); - $clonePath = scmPath . "/" . sha256_hex($uri); + my $clonePath = scmPath . "/" . sha256_hex($uri); if (! -d $clonePath) { (my $res, $stdout, $stderr) = captureStdoutStderr(600, @@ -627,7 +640,7 @@ sub inputsToArgs { sub captureStdoutStderr { - (my $timeout, my @cmd) = @_; + my ($timeout, @cmd) = @_; my $res; my $stdin = ""; my $stdout; @@ -649,7 +662,7 @@ sub captureStdoutStderr { } } - + sub evalJobs { my ($inputInfo, $nixExprInputName, $nixExprPath) = @_; diff --git a/src/script/Makefile.am b/src/script/Makefile.am index 14f01d93..68ad388a 100644 --- a/src/script/Makefile.am +++ b/src/script/Makefile.am @@ -1,9 +1,5 @@ EXTRA_DIST = \ hydra-control \ - nix-prefetch-svn \ - nix-prefetch-git \ - nix-prefetch-bzr \ - nix-prefetch-hg \ $(bin_SCRIPTS) bin_SCRIPTS = \ @@ -13,7 +9,6 @@ bin_SCRIPTS = \ hydra-server \ hydra-update-gc-roots \ hydra-create \ - nix-prefetch-svn \ nix-prefetch-git \ nix-prefetch-bzr \ nix-prefetch-hg diff --git a/src/script/nix-prefetch-svn b/src/script/nix-prefetch-svn deleted file mode 100755 index 2858a0b0..00000000 --- a/src/script/nix-prefetch-svn +++ /dev/null @@ -1,79 +0,0 @@ -#! /bin/sh -e - -url=$1 -rev=$2 -expHash=$3 - -hashType=$NIX_HASH_ALGO -if test -z "$hashType"; then - hashType=sha256 -fi -if test -z "$hashFormat"; then - hashFormat=--base32 -fi - -if test -z "$url"; then - echo "syntax: nix-prefetch-svn URL [REVISION [EXPECTED-HASH]]" >&2 - exit 1 -fi - -test -n "$rev" || rev="HEAD" - -repoName=$(echo $url | sed ' - s,.*/\([^/]\+\)/trunk/*$,\1,;t - s,.*/\([^/]\+\)/branches/\([^/]\+\)/*$,\1-\2,;t - s,.*/\([^/]\+\)/tags/\([^/]\+\)/*$,\1-\2,;t - s,.*/\([^/]\+\)/*$,\1,;t -') -dstFile=$repoName-r$rev - -# If the hash was given, a file with that hash may already be in the -# store. -if test -n "$expHash"; then - finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" $dstFile) - if ! nix-store --check-validity "$finalPath" 2> /dev/null; then - finalPath= - fi - hash=$expHash -fi - - -# If we don't know the hash or a path with that hash doesn't exist, -# download the file and add it to the store. -if test -z "$finalPath"; then - tmpPath=/tmp/svn-checkout-tmp-$$ - tmpFile=$tmpPath/$dstFile - mkdir $tmpPath - - trap "rm -rf $tmpPath" EXIT - - # Perform the checkout. - if test "$NIX_PREFETCH_SVN_LEAVE_DOT_SVN" != 1 - then - command="export" - else - command="checkout" - fi - - echo p | svn "$command" --quiet -r "$rev" "$url" "$tmpFile" >&2 - - # Compute the hash. - hash=$(nix-hash --type $hashType $hashFormat $tmpFile) - if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi - - # Add the downloaded file to the Nix store. - finalPath=$(nix-store --add-fixed --recursive "$hashType" $tmpFile) - - if test -n "$expHash" -a "$expHash" != "$hash"; then - echo "hash mismatch for URL \`$url'" - exit 1 - fi -fi - -if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi - -echo $hash - -if test -n "$PRINT_PATH"; then - echo $finalPath -fi