Optimize fetch-git.

This commit is contained in:
Nicolas Pierron 2012-04-14 18:17:35 -07:00
parent ccc5d38976
commit 19d9955e89
11 changed files with 139 additions and 58 deletions

View file

@ -315,7 +315,7 @@ sub fetchInputSystemBuild {
sub fetchInputGit { sub fetchInputGit {
my ($db, $project, $jobset, $name, $value) = @_; my ($db, $project, $jobset, $name, $value) = @_;
(my $uri, my $branch) = split ' ', $value; (my $uri, my $branch, my $deepClone) = split ' ', $value;
$branch = defined $branch ? $branch : "master"; $branch = defined $branch ? $branch : "master";
my $timestamp = time; my $timestamp = time;
@ -325,42 +325,64 @@ sub fetchInputGit {
mkpath(scmPath); mkpath(scmPath);
my $clonePath = scmPath . "/" . sha256_hex($uri); my $clonePath = scmPath . "/" . sha256_hex($uri);
my $stdout; my $stderr; my $stdout = ""; my $stderr = ""; my $res;
if (! -d $clonePath) { if (! -d $clonePath) {
(my $res, $stdout, $stderr) = captureStdoutStderr(600, # Clone everything and fetch the branch.
# TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri".
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "clone", "--branch", $branch, $uri, $clonePath)); ("git", "clone", "--branch", $branch, $uri, $clonePath));
die "Error cloning git repo at `$uri':\n$stderr" unless $res; die "Error cloning git repo at `$uri':\n$stderr" unless $res;
} }
# git pull + check rev
chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "pull", "--all"));
die "Error pulling latest change git repo at `$uri':\n$stderr" unless $res;
(my $res1, $stdout, $stderr) = captureStdoutStderr(600, if (defined $deepClone) {
("git", "ls-remote", $clonePath, $branch)); # This fetch every branches from the remote repository and create a
# local branch for each heads of the remote repository. This is
die "Cannot get head revision of Git branch '$branch' at `$uri':\n$stderr" unless $res1 ; # necessary to provide a working git-describe.
($res, $stdout, $stderr) = captureStdoutStderr(600,
# Take the first commit ID returned by `ls-remote'. The ("git", "pull", "--ff-only", "-fu", "--all", "origin"));
# assumption is that `ls-remote' returned both `refs/heads/BRANCH' die "Error pulling latest change from git repo at `$uri':\n$stderr" unless $res;
# and `refs/remotes/origin/BRANCH', and that both point at the } else {
# same commit. # This command force the update of the local branch to be in the same as
my ($first) = split /\n/, $stdout; # the remote branch for whatever the repository state is. This command mirror
(my $revision, my $ref) = split ' ', $first; # only one branch of the remote repository.
die unless $revision =~ /^[0-9a-fA-F]+$/; ($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "fetch", "-fu", "origin", "+$branch:$branch"));
if (-f ".topdeps") { die "Error fetching latest change from git repo at `$uri':\n$stderr" unless $res;
# This is a TopGit branch. Fetch all the topic branches so
# that builders can run "tg patch" and similar.
(my $res, $stdout, $stderr) = captureStdoutStderr(600,
("tg", "remote", "--populate", "origin"));
print STDERR "Warning: `tg remote --populate origin' failed:\n$stderr" unless $res;
} }
# Some simple caching: don't check a uri/branch more than once every hour, but prefer exact match on uri/branch/revision. ($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "rev-parse", "$branch"));
die "Error getting revision number of Git branch '$branch' at `$uri':\n$stderr" unless $res;
my ($revision) = split /\n/, $stdout;
die unless $revision =~ /^[0-9a-fA-F]+$/;
die "Error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout" unless $res;
my $ref = "refs/heads/$branch";
# If deepClone is defined, then we look at the content of the repository
# to determine if this is a top-git branch.
if (defined $deepClone) {
# Checkout the branch to look at its content.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("git", "checkout", "$branch"));
die "Error checking out Git branch '$branch' at `$uri':\n$stderr" unless $res;
if (-f ".topdeps") {
# This is a TopGit branch. Fetch all the topic branches so
# that builders can run "tg patch" and similar.
($res, $stdout, $stderr) = captureStdoutStderr(600,
("tg", "remote", "--populate", "origin"));
print STDERR "Warning: `tg remote --populate origin' failed:\n$stderr" unless $res;
}
}
# Some simple caching: don't check a uri/branch/revision more than once.
# TODO: Fix case where the branch is reset to a previous commit.
my $cachedInput ; my $cachedInput ;
($cachedInput) = $db->resultset('CachedGitInputs')->search( ($cachedInput) = $db->resultset('CachedGitInputs')->search(
{uri => $uri, branch => $branch, revision => $revision}, {uri => $uri, branch => $branch, revision => $revision},
@ -371,25 +393,28 @@ sub fetchInputGit {
$sha256 = $cachedInput->sha256hash; $sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision; $revision = $cachedInput->revision;
} else { } else {
# Then download this revision into the store. # Then download this revision into the store.
print STDERR "checking out Git input from $uri\n"; print STDERR "checking out Git branch $branch from $uri\n";
$ENV{"NIX_HASH_ALGO"} = "sha256"; $ENV{"NIX_HASH_ALGO"} = "sha256";
$ENV{"PRINT_PATH"} = "1"; $ENV{"PRINT_PATH"} = "1";
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0";
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "";
# Checked out code often wants to be able to run `git if (defined $deepClone) {
# describe', e.g., code that uses Gnulib's `git-version-gen' # Checked out code often wants to be able to run `git
# script. Thus, we leave `.git' in there. Same for # describe', e.g., code that uses Gnulib's `git-version-gen'
# Subversion (e.g., libgcrypt's build system uses that.) # script. Thus, we leave `.git' in there. Same for
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1"; # Subversion (e.g., libgcrypt's build system uses that.)
$ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1";
# Ask for a "deep clone" to allow "git describe" and similar # Ask for a "deep clone" to allow "git describe" and similar
# tools to work. See # tools to work. See
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569 # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
# for a discussion. # for a discussion.
$ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1"; $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1";
}
(my $res, $stdout, $stderr) = captureStdoutStderr(600, ($res, $stdout, $stderr) = captureStdoutStderr(600,
("nix-prefetch-git", $clonePath, $revision)); ("nix-prefetch-git", $clonePath, $revision));
die "Cannot check out Git repository branch '$branch' at `$uri':\n$stderr" unless $res; die "Cannot check out Git repository branch '$branch' at `$uri':\n$stderr" unless $res;
@ -509,7 +534,7 @@ sub fetchInputHg {
# init local hg clone # init local hg clone
my $stdout; my $stderr; my $stdout = ""; my $stderr = "";
mkpath(scmPath); mkpath(scmPath);
my $clonePath = scmPath . "/" . sha256_hex($uri); my $clonePath = scmPath . "/" . sha256_hex($uri);
@ -681,7 +706,7 @@ sub captureStdoutStderr {
if ($@) { if ($@) {
die unless $@ eq "timeout\n"; # propagate unexpected errors die unless $@ eq "timeout\n"; # propagate unexpected errors
return (undef, undef, undef); return (undef, "", "timeout\n");
} else { } else {
return ($res, $stdout, $stderr); return ($res, $stdout, $stderr);
} }

View file

@ -63,8 +63,8 @@ Options:
--url url Any url understand by 'git clone'. --url url Any url understand by 'git clone'.
--rev ref Any sha1 or references (such as refs/heads/master) --rev ref Any sha1 or references (such as refs/heads/master)
--hash h Expected hash. --hash h Expected hash.
--deepClone Clone submodules recursively. --deepClone Clone history until a tag is found as parent.
--no-deepClone Do not clone submodules. --no-deepClone Clone the minimum history.
--leave-dotGit Keep the .git directories. --leave-dotGit Keep the .git directories.
--builder Clone as fetchgit does, but url, rev, and out option are mandatory. --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
" "
@ -117,7 +117,14 @@ checkout_ref(){
# allow "git describe" and similar tools to work. See # allow "git describe" and similar tools to work. See
# http://thread.gmane.org/gmane.linux.distributions.nixos/3569 # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
# for a discussion. # for a discussion.
return 1
# To make git describe works, we need to fetch all tags.
if git fetch -t ${builder:+--progress} --depth 1 origin; then
return 1
else
# There is no tag, don't try to recover git-describe mechanism.
deepClone=false
fi
fi fi
if test -z "$ref"; then if test -z "$ref"; then
@ -209,7 +216,7 @@ clone_user_rev() {
# Allow doing additional processing before .git removal # Allow doing additional processing before .git removal
eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK" eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
if test -z "$leaveDotGit"; then if test -z "$leaveDotGit"; then
echo "removing \`.git'..." >&2 test -n "$QUIET" || echo "removing \`.git'..." >&2
find $dir -name .git\* | xargs rm -rf find $dir -name .git\* | xargs rm -rf
fi fi
} }
@ -248,7 +255,7 @@ else
# Compute the hash. # Compute the hash.
hash=$(nix-hash --type $hashType $hashFormat $tmpFile) hash=$(nix-hash --type $hashType $hashFormat $tmpFile)
if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi test -n "$QUIET" || echo "hash is $hash" >&2;
# Add the downloaded file to the Nix store. # Add the downloaded file to the Nix store.
finalPath=$(nix-store --add-fixed --recursive "$hashType" $tmpFile) finalPath=$(nix-store --add-fixed --recursive "$hashType" $tmpFile)
@ -259,7 +266,7 @@ else
fi fi
fi fi
if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi test -n "$QUIET" || echo "path is $finalPath" >&2
echo $hash echo $hash

View file

@ -8,7 +8,7 @@ use Hydra::Helper::AddBuilds;
use Cwd; use Cwd;
our @ISA = qw(Exporter); our @ISA = qw(Exporter);
our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild); our @EXPORT = qw(hydra_setup nrBuildsForJobset queuedBuildsForJobset nrQueuedBuildsForJobset createBaseJobset createJobsetWithOneInput evalSucceeds runBuild updateRepository);
sub hydra_setup { sub hydra_setup {
my ($db) = @_; my ($db) = @_;
@ -74,4 +74,12 @@ sub runBuild {
return captureStdoutStderr(60, ("../src/script/hydra-build", $build->id)); return captureStdoutStderr(60, ("../src/script/hydra-build", $build->id));
} }
sub updateRepository {
my ($scm, $update, $repo) = @_;
my ($res, $stdout, $stderr) = captureStdoutStderr(60, ($update, $repo));
die "Unexpected update error with $scm: $stderr\n" unless $res;
print STDOUT "Update $scm repository: $stdout" if $stdout ne "";
return $stdout ne "";
}
1; 1;

View file

@ -7,7 +7,7 @@ use Setup;
my $db = Hydra::Model::DB->new; my $db = Hydra::Model::DB->new;
use Test::Simple tests => 28; use Test::Simple tests => 48;
hydra_setup($db); hydra_setup($db);
@ -57,7 +57,17 @@ my @scminputs = ("svn", "svn-checkout", "git", "bzr", "bzr-checkout", "hg");
foreach my $scm (@scminputs) { foreach my $scm (@scminputs) {
$jobset = createJobsetWithOneInput($scm, "$scm-input.nix", "src", $scm, "$jobsBaseUri/$scm-repo"); $jobset = createJobsetWithOneInput($scm, "$scm-input.nix", "src", $scm, "$jobsBaseUri/$scm-repo");
ok(evalSucceeds($jobset), "Evaluating jobs/$scm-input.nix should exit with return code 0."); my $c = 1;
ok(nrQueuedBuildsForJobset($jobset) == 1, "Evaluating jobs/$scm-input.nix should result in 1 build in queue"); my $q = 1;
} do {
# Verify that it can be fetched and queued.
ok(evalSucceeds($jobset), "$c Evaluating jobs/$scm-input.nix should exit with return code 0."); $c++;
ok(nrQueuedBuildsForJobset($jobset) == $q, "$c Evaluating jobs/$scm-input.nix should result in 1 build in queue"); $c++;
# Verify that it is deterministic and not queued again.
ok(evalSucceeds($jobset), "$c Evaluating jobs/$scm-input.nix should exit with return code 0."); $c++;
ok(nrQueuedBuildsForJobset($jobset) == $q, "$c Evaluating jobs/$scm-input.nix should result in $q build in queue"); $c++;
$q++;
} while(updateRepository($scm, getcwd . "/jobs/$scm-update.sh", getcwd . "/$scm-repo/"));
}

View file

@ -0,0 +1 @@
#! /bin/sh

1
tests/jobs/bzr-update.sh Executable file
View file

@ -0,0 +1 @@
#! /bin/sh

26
tests/jobs/git-update.sh Executable file
View file

@ -0,0 +1,26 @@
#! /bin/sh
cd "$1"
STATE_FILE=.state
if test -e $STATE_FILE; then
state=$(cat $STATE_FILE)
else
state=0;
fi
case $state in
(0)
echo "Add new file."
touch git-file-2
git add git-file-2 >&2
git commit -m "add git file 2" git-file-2 >&2
;;
(1)
echo "Rewrite commit."
echo 1 > git-file-2
git add git-file-2 >&2
git commit --amend -m "add git file 2" git-file-2 >&2
;;
esac
echo $(($state + 1)) > $STATE_FILE

1
tests/jobs/hg-update.sh Executable file
View file

@ -0,0 +1 @@
#! /bin/sh

View file

@ -0,0 +1 @@
#! /bin/sh

1
tests/jobs/svn-update.sh Executable file
View file

@ -0,0 +1 @@
#! /bin/sh