Do incremental SVN checkouts

In hydra-evaluator, reuse an SVN working copy between runs (similar to
what we do with Git and other input types).  This reduces network
traffic in the common case.

Also, don't use nix-prefetch-svn.  It doesn't do anything useful.
This commit is contained in:
Eelco Dolstra 2011-12-04 22:01:53 +01:00
parent 692a11fd3b
commit 29846d7f2f
3 changed files with 33 additions and 104 deletions

View file

@ -11,6 +11,7 @@ use Digest::SHA qw(sha256_hex);
use File::Basename;
use File::stat;
use File::Path;
use File::Temp;
our @ISA = qw(Exporter);
our @EXPORT = qw(
@ -182,6 +183,9 @@ sub fetchInputSVN {
die unless $revision =~ /^\d+$/;
# Do we already have this revision in the store?
# !!! This needs to take $checkout into account! Otherwise "svn"
# and "svn-checkout" inputs can get mixed up.
(my $cachedInput) = $db->resultset('CachedSubversionInputs')->search(
{uri => $uri, revision => $revision});
@ -190,20 +194,30 @@ sub fetchInputSVN {
$sha256 = $cachedInput->sha256hash;
} else {
# Then download this revision into the store.
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision\n";
$ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_SVN_LEAVE_DOT_SVN"} = "$checkout";
# No, do a checkout. The working copy is reused between
# invocations to speed things up.
mkpath(scmPath . "/svn");
my $wcPath = scmPath . "/svn" . sha256_hex($uri) . "/svn-checkout";
print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n";
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
("nix-prefetch-svn", $uri, $revision));
die "Cannot check out Subversion repository `$uri':\n$stderr" unless $res;
("svn", "checkout", $uri, "-r", $revision, $wcPath));
($sha256, $storePath) = split ' ', $stdout;
if ($checkout) {
$storePath = addToStore($wcPath, 1, "sha256");
} else {
# Hm, if the Nix Perl bindings supported filters in
# addToStore(), then we wouldn't need to make a copy here.
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed";
$storePath = addToStore("$tmpDir/svn-export", 1, "sha256");
}
$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
txn_do($db, sub {
$db->resultset('CachedSubversionInputs')->create(
$db->resultset('CachedSubversionInputs')->update_or_create(
{ uri => $uri
, revision => $revision
, sha256hash => $sha256
@ -311,9 +325,8 @@ sub fetchInputGit {
my $sha256;
my $storePath;
my $clonePath;
mkpath(scmPath);
$clonePath = scmPath . "/" . sha256_hex($uri);
my $clonePath = scmPath . "/" . sha256_hex($uri);
my $stdout; my $stderr;
if (! -d $clonePath) {
@ -411,9 +424,9 @@ sub fetchInputBazaar {
my $storePath;
my $stdout; my $stderr;
my $clonePath;
mkpath(scmPath);
$clonePath = scmPath . "/" . sha256_hex($uri);
my $clonePath = scmPath . "/" . sha256_hex($uri);
if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
@ -484,9 +497,9 @@ sub fetchInputHg {
# init local hg clone
my $stdout; my $stderr;
my $clonePath;
mkpath(scmPath);
$clonePath = scmPath . "/" . sha256_hex($uri);
my $clonePath = scmPath . "/" . sha256_hex($uri);
if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
@ -627,7 +640,7 @@ sub inputsToArgs {
sub captureStdoutStderr {
(my $timeout, my @cmd) = @_;
my ($timeout, @cmd) = @_;
my $res;
my $stdin = "";
my $stdout;

View file

@ -1,9 +1,5 @@
EXTRA_DIST = \
hydra-control \
nix-prefetch-svn \
nix-prefetch-git \
nix-prefetch-bzr \
nix-prefetch-hg \
$(bin_SCRIPTS)
bin_SCRIPTS = \
@ -13,7 +9,6 @@ bin_SCRIPTS = \
hydra-server \
hydra-update-gc-roots \
hydra-create \
nix-prefetch-svn \
nix-prefetch-git \
nix-prefetch-bzr \
nix-prefetch-hg

View file

@ -1,79 +0,0 @@
#! /bin/sh -e
url=$1
rev=$2
expHash=$3
hashType=$NIX_HASH_ALGO
if test -z "$hashType"; then
hashType=sha256
fi
if test -z "$hashFormat"; then
hashFormat=--base32
fi
if test -z "$url"; then
echo "syntax: nix-prefetch-svn URL [REVISION [EXPECTED-HASH]]" >&2
exit 1
fi
test -n "$rev" || rev="HEAD"
repoName=$(echo $url | sed '
s,.*/\([^/]\+\)/trunk/*$,\1,;t
s,.*/\([^/]\+\)/branches/\([^/]\+\)/*$,\1-\2,;t
s,.*/\([^/]\+\)/tags/\([^/]\+\)/*$,\1-\2,;t
s,.*/\([^/]\+\)/*$,\1,;t
')
dstFile=$repoName-r$rev
# If the hash was given, a file with that hash may already be in the
# store.
if test -n "$expHash"; then
finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" $dstFile)
if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
finalPath=
fi
hash=$expHash
fi
# If we don't know the hash or a path with that hash doesn't exist,
# download the file and add it to the store.
if test -z "$finalPath"; then
tmpPath=/tmp/svn-checkout-tmp-$$
tmpFile=$tmpPath/$dstFile
mkdir $tmpPath
trap "rm -rf $tmpPath" EXIT
# Perform the checkout.
if test "$NIX_PREFETCH_SVN_LEAVE_DOT_SVN" != 1
then
command="export"
else
command="checkout"
fi
echo p | svn "$command" --quiet -r "$rev" "$url" "$tmpFile" >&2
# Compute the hash.
hash=$(nix-hash --type $hashType $hashFormat $tmpFile)
if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
# Add the downloaded file to the Nix store.
finalPath=$(nix-store --add-fixed --recursive "$hashType" $tmpFile)
if test -n "$expHash" -a "$expHash" != "$hash"; then
echo "hash mismatch for URL \`$url'"
exit 1
fi
fi
if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
echo $hash
if test -n "$PRINT_PATH"; then
echo $finalPath
fi