From 1f1615e80b89350c0143c2c0a0c8007934f9f7f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 25 May 2013 15:36:58 -0400 Subject: [PATCH] Support revision control systems via plugins --- src/lib/Hydra.pm | 13 + src/lib/Hydra/Controller/API.pm | 6 +- src/lib/Hydra/Controller/Build.pm | 2 +- src/lib/Hydra/Controller/Root.pm | 9 + src/lib/Hydra/Helper/AddBuilds.pm | 511 +----------------------- src/lib/Hydra/Helper/Nix.pm | 31 +- src/lib/Hydra/Plugin.pm | 14 + src/lib/Hydra/Plugin/BazaarInput.pm | 87 ++++ src/lib/Hydra/Plugin/GitInput.pm | 148 +++++++ src/lib/Hydra/Plugin/MercurialInput.pm | 88 ++++ src/lib/Hydra/Plugin/PathInput.pm | 77 ++++ src/lib/Hydra/Plugin/SubversionInput.pm | 90 +++++ src/root/common.tt | 14 - src/root/edit-jobset.tt | 2 - src/script/hydra-evaluator | 9 +- 15 files changed, 588 insertions(+), 513 deletions(-) create mode 100644 src/lib/Hydra/Plugin/BazaarInput.pm create mode 100644 src/lib/Hydra/Plugin/GitInput.pm create mode 100644 src/lib/Hydra/Plugin/MercurialInput.pm create mode 100644 src/lib/Hydra/Plugin/PathInput.pm create mode 100644 src/lib/Hydra/Plugin/SubversionInput.pm diff --git a/src/lib/Hydra.pm b/src/lib/Hydra.pm index 732d56d3..56165418 100644 --- a/src/lib/Hydra.pm +++ b/src/lib/Hydra.pm @@ -4,6 +4,7 @@ use strict; use warnings; use parent 'Catalyst'; use Moose; +use Hydra::Plugin; use Hydra::Model::DB; use Catalyst::Runtime '5.70'; use Catalyst qw/ConfigLoader @@ -82,6 +83,18 @@ __PACKAGE__->config( __PACKAGE__->apply_request_class_roles(qw/Catalyst::TraitFor::Request::ProxyBase/); +my $plugins; + +has 'hydra_plugins' => ( + is => 'ro', + default => sub { return $plugins; } +); + +after setup_finalize => sub { + my $class = shift; + $plugins = [Hydra::Plugin->plugins(db => $class->model('DB'), config => $class->config)]; +}; + __PACKAGE__->setup(); 1; diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 04a6e54b..7a086b4b 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -219,15 +219,17 @@ sub scmdiff : Chained('api') PathPart('scmdiff') Args(0) { die("invalid revisions: [$rev1] [$rev2]") if $rev1 !~ m/^[a-zA-Z0-9_.]+$/ || $rev2 !~ m/^[a-zA-Z0-9_.]+$/; + # FIXME: injection danger. + my $diff = ""; if ($type eq "hg") { - my $clonePath = scmPath . "/" . sha256_hex($uri); + my $clonePath = getSCMCacheDir . "/hg/" . sha256_hex($uri); die if ! -d $clonePath; $branch = `(cd $clonePath; hg log --template '{branch}' -r $rev2)`; $diff .= `(cd $clonePath; hg log -r $rev1 -r $rev2 -b $branch)`; $diff .= `(cd $clonePath; hg diff -r $rev1:$rev2)`; } elsif ($type eq "git") { - my $clonePath = scmPath . "/" . sha256_hex($uri); + my $clonePath = getSCMCacheDir . "/git/" . sha256_hex($uri); die if ! -d $clonePath; $diff .= `(cd $clonePath; git log $rev1..$rev2)`; $diff .= `(cd $clonePath; git diff $rev1..$rev2)`; diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 0c5fa0bf..511cae8f 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -533,7 +533,7 @@ sub clone_submit : Chained('build') PathPart('clone/submit') Args(0) { # should be done asynchronously. But then error reporting # becomes harder. my $info = fetchInput( - $c->model('DB'), $build->project, $build->jobset, + $c->hydra_plugins, $c->model('DB'), $build->project, $build->jobset, $inputName, $inputType, $inputValue); push @{$$inputInfo{$inputName}}, $info if defined $info; }; diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index f49d2723..5d4861e3 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -28,6 +28,15 @@ sub begin :Private { $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count(); $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count(); } + + # Gather the supported input types. + $c->stash->{inputTypes} = { + 'string' => 'String value', + 'boolean' => 'Boolean', + 'build' => 'Build output', + 'sysbuild' => 'Build output (same system)' + }; + $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins}; } diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index f43bc2e0..67d6d505 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -3,7 +3,6 @@ package Hydra::Helper::AddBuilds; use strict; use feature 'switch'; use XML::Simple; -use POSIX qw(strftime); use IPC::Run; use Nix::Store; use Nix::Config; @@ -19,31 +18,12 @@ use File::Slurp; our @ISA = qw(Exporter); our @EXPORT = qw( - fetchInput evalJobs checkBuild inputsToArgs captureStdoutStderr - getReleaseName addBuildProducts restartBuild scmPath + fetchInput evalJobs checkBuild inputsToArgs + getReleaseName addBuildProducts restartBuild getPrevJobsetEval ); -sub scmPath { - return Hydra::Model::DB::getHydraPath . "/scm" ; -} - - -sub getStorePathHash { - my ($storePath) = @_; - my $hash = `nix-store --query --hash $storePath` - or die "cannot get hash of $storePath"; - chomp $hash; - die unless $hash =~ /^sha256:(.*)$/; - $hash = $1; - $hash = `nix-hash --to-base16 --type sha256 $hash` - or die "cannot convert hash"; - chomp $hash; - return $hash; -} - - sub getReleaseName { my ($outPath) = @_; return undef unless -f "$outPath/nix-support/hydra-release-name"; @@ -92,140 +72,6 @@ sub attrsToSQL { } -sub fetchInputPath { - my ($db, $project, $jobset, $name, $value) = @_; - - my $uri = $value; - - my $timestamp = time; - my $sha256; - my $storePath; - - # Some simple caching: don't check a path more than once every N seconds. - (my $cachedInput) = $db->resultset('CachedPathInputs')->search( - {srcpath => $uri, lastseen => {">", $timestamp - 30}}, - {rows => 1, order_by => "lastseen DESC"}); - - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { - $storePath = $cachedInput->storepath; - $sha256 = $cachedInput->sha256hash; - $timestamp = $cachedInput->timestamp; - } else { - - print STDERR "copying input ", $name, " from $uri\n"; - $storePath = `nix-store --add "$uri"` - or die "cannot copy path $uri to the Nix store.\n"; - chomp $storePath; - - $sha256 = getStorePathHash $storePath; - - ($cachedInput) = $db->resultset('CachedPathInputs')->search( - {srcpath => $uri, sha256hash => $sha256}); - - # Path inputs don't have a natural notion of a "revision", so - # we simulate it by using the timestamp that we first saw this - # path have this SHA-256 hash. So if the contents of the path - # changes, we get a new "revision", but if it doesn't change - # (or changes back), we don't get a new "revision". - if (!defined $cachedInput) { - txn_do($db, sub { - $db->resultset('CachedPathInputs')->update_or_create( - { srcpath => $uri - , timestamp => $timestamp - , lastseen => $timestamp - , sha256hash => $sha256 - , storepath => $storePath - }); - }); - } else { - $timestamp = $cachedInput->timestamp; - txn_do($db, sub { - $cachedInput->update({lastseen => time}); - }); - } - } - - return - { uri => $uri - , storePath => $storePath - , sha256hash => $sha256 - , revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) - }; -} - - -sub fetchInputSVN { - my ($db, $project, $jobset, $name, $value, $checkout) = @_; - - # Allow users to specify a revision number next to the URI. - my ($uri, $revision) = split ' ', $value; - - my $sha256; - my $storePath; - my $stdout; my $stderr; - - unless (defined $revision) { - # First figure out the last-modified revision of the URI. - my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri], - "|", ["sed", 's/^ *\([0-9]*\).*/\1/']); - IPC::Run::run(@cmd, \$stdout, \$stderr); - die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?; - $revision = $stdout; $revision =~ s/\s*([0-9]+)\s*/$1/sm; - } - - die unless $revision =~ /^\d+$/; - - # Do we already have this revision in the store? - # !!! This needs to take $checkout into account! Otherwise "svn" - # and "svn-checkout" inputs can get mixed up. - (my $cachedInput) = $db->resultset('CachedSubversionInputs')->search( - {uri => $uri, revision => $revision}); - - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { - $storePath = $cachedInput->storepath; - $sha256 = $cachedInput->sha256hash; - } else { - - # No, do a checkout. The working copy is reused between - # invocations to speed things up. - my $wcPath = scmPath . "/svn/" . sha256_hex($uri) . "/svn-checkout"; - - print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n"; - - (my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath); - die "error checking out Subversion repo at `$uri':\n$stderr" if $res; - - if ($checkout) { - $storePath = addToStore($wcPath, 1, "sha256"); - } else { - # Hm, if the Nix Perl bindings supported filters in - # addToStore(), then we wouldn't need to make a copy here. - my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die; - (system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed"; - $storePath = addToStore("$tmpDir/svn-export", 1, "sha256"); - } - - $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://; - - txn_do($db, sub { - $db->resultset('CachedSubversionInputs')->update_or_create( - { uri => $uri - , revision => $revision - , sha256hash => $sha256 - , storepath => $storePath - }); - }); - } - - return - { uri => $uri - , storePath => $storePath - , sha256hash => $sha256 - , revision => $revision - }; -} - - sub fetchInputBuild { my ($db, $project, $jobset, $name, $value) = @_; @@ -278,7 +124,7 @@ sub fetchInputSystemBuild { if (scalar(@validBuilds) == 0) { print STDERR "input `", $name, "': no previous build available\n"; - return undef; + return (); } my @inputs = (); @@ -303,323 +149,37 @@ sub fetchInputSystemBuild { } -sub fetchInputGit { - my ($db, $project, $jobset, $name, $value) = @_; - - (my $uri, my $branch, my $deepClone) = split ' ', $value; - $branch = defined $branch ? $branch : "master"; - - my $timestamp = time; - my $sha256; - my $storePath; - - mkpath(scmPath); - my $clonePath = scmPath . "/" . sha256_hex($uri); - - my $stdout = ""; my $stderr = ""; my $res; - if (! -d $clonePath) { - # Clone everything and fetch the branch. - # TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri". - ($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath); - die "error cloning git repo at `$uri':\n$stderr" if $res; - } - - chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir - - # This command force the update of the local branch to be in the same as - # the remote branch for whatever the repository state is. This command mirror - # only one branch of the remote repository. - ($res, $stdout, $stderr) = captureStdoutStderr(600, - "git", "fetch", "-fu", "origin", "+$branch:$branch"); - ($res, $stdout, $stderr) = captureStdoutStderr(600, - "git", "fetch", "-fu", "origin") if $res; - die "error fetching latest change from git repo at `$uri':\n$stderr" if $res; - - ($res, $stdout, $stderr) = captureStdoutStderr(600, - ("git", "rev-parse", "$branch")); - die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res; - - my ($revision) = split /\n/, $stdout; - die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout" - unless $revision =~ /^[0-9a-fA-F]+$/; - - my $ref = "refs/heads/$branch"; - - # If deepClone is defined, then we look at the content of the repository - # to determine if this is a top-git branch. - if (defined $deepClone) { - - # Checkout the branch to look at its content. - ($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch"); - die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res; - - if (-f ".topdeps") { - # This is a TopGit branch. Fetch all the topic branches so - # that builders can run "tg patch" and similar. - ($res, $stdout, $stderr) = captureStdoutStderr(600, - "tg", "remote", "--populate", "origin"); - print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res; - } - } - - # Some simple caching: don't check a uri/branch/revision more than once. - # TODO: Fix case where the branch is reset to a previous commit. - my $cachedInput ; - ($cachedInput) = $db->resultset('CachedGitInputs')->search( - {uri => $uri, branch => $branch, revision => $revision}, - {rows => 1}); - - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { - $storePath = $cachedInput->storepath; - $sha256 = $cachedInput->sha256hash; - $revision = $cachedInput->revision; - } else { - # Then download this revision into the store. - print STDERR "checking out Git branch $branch from $uri\n"; - $ENV{"NIX_HASH_ALGO"} = "sha256"; - $ENV{"PRINT_PATH"} = "1"; - $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0"; - $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = ""; - - if (defined $deepClone) { - # Checked out code often wants to be able to run `git - # describe', e.g., code that uses Gnulib's `git-version-gen' - # script. Thus, we leave `.git' in there. Same for - # Subversion (e.g., libgcrypt's build system uses that.) - $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1"; - - # Ask for a "deep clone" to allow "git describe" and similar - # tools to work. See - # http://thread.gmane.org/gmane.linux.distributions.nixos/3569 - # for a discussion. - $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1"; - } - - ($res, $stdout, $stderr) = captureStdoutStderr(600, "nix-prefetch-git", $clonePath, $revision); - die "cannot check out Git repository branch '$branch' at `$uri':\n$stderr" if $res; - - ($sha256, $storePath) = split ' ', $stdout; - - txn_do($db, sub { - $db->resultset('CachedGitInputs')->update_or_create( - { uri => $uri - , branch => $branch - , revision => $revision - , sha256hash => $sha256 - , storepath => $storePath - }); - }); - } - - # For convenience in producing readable version names, pass the - # number of commits in the history of this revision (‘revCount’) - # the output of git-describe (‘gitTag’), and the abbreviated - # revision (‘shortRev’). - my $revCount = `git rev-list $revision | wc -l`; chomp $revCount; - die "git rev-list failed" if $? != 0; - my $gitTag = `git describe --always $revision`; chomp $gitTag; - die "git describe failed" if $? != 0; - my $shortRev = `git rev-parse --short $revision`; chomp $shortRev; - die "git rev-parse failed" if $? != 0; - - return - { uri => $uri - , storePath => $storePath - , sha256hash => $sha256 - , revision => $revision - , revCount => int($revCount) - , gitTag => $gitTag - , shortRev => $shortRev - }; -} - - -sub fetchInputBazaar { - my ($db, $project, $jobset, $name, $value, $checkout) = @_; - - my $uri = $value; - - my $sha256; - my $storePath; - - my $stdout; my $stderr; - - mkpath(scmPath); - my $clonePath = scmPath . "/" . sha256_hex($uri); - - if (! -d $clonePath) { - (my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath); - die "error cloning bazaar branch at `$uri':\n$stderr" if $res; - } - - chdir $clonePath or die $!; - (my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull"); - die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res; - - # First figure out the last-modified revision of the URI. - my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']); - - IPC::Run::run(@cmd, \$stdout, \$stderr); - die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?; - my $revision = $stdout; chomp $revision; - die unless $revision =~ /^\d+$/; - - (my $cachedInput) = $db->resultset('CachedBazaarInputs')->search( - {uri => $uri, revision => $revision}); - - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { - $storePath = $cachedInput->storepath; - $sha256 = $cachedInput->sha256hash; - } else { - - # Then download this revision into the store. - print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n"; - $ENV{"NIX_HASH_ALGO"} = "sha256"; - $ENV{"PRINT_PATH"} = "1"; - $ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = "$checkout"; - - (my $res, $stdout, $stderr) = captureStdoutStderr(600, - "nix-prefetch-bzr", $clonePath, $revision); - die "cannot check out Bazaar branch `$uri':\n$stderr" if $res; - - ($sha256, $storePath) = split ' ', $stdout; - - txn_do($db, sub { - $db->resultset('CachedBazaarInputs')->create( - { uri => $uri - , revision => $revision - , sha256hash => $sha256 - , storepath => $storePath - }); - }); - } - - return - { uri => $uri - , storePath => $storePath - , sha256hash => $sha256 - , revision => $revision - }; -} - - -sub fetchInputHg { - my ($db, $project, $jobset, $name, $value) = @_; - - (my $uri, my $id) = split ' ', $value; - $id = defined $id ? $id : "default"; - - # init local hg clone - - my $stdout = ""; my $stderr = ""; - - mkpath(scmPath); - my $clonePath = scmPath . "/" . sha256_hex($uri); - - if (! -d $clonePath) { - (my $res, $stdout, $stderr) = captureStdoutStderr(600, - "hg", "clone", $uri, $clonePath); - die "error cloning mercurial repo at `$uri':\n$stderr" if $res; - } - - # hg pull + check rev - chdir $clonePath or die $!; - (my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull"); - die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res; - - (my $res1, $stdout, $stderr) = captureStdoutStderr(600, - "hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}"); - die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1; - - my ($revision, $revCount, $branch) = split ' ', $stdout; - - my $storePath; - my $sha256; - (my $cachedInput) = $db->resultset('CachedHgInputs')->search( - {uri => $uri, branch => $branch, revision => $revision}); - - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { - $storePath = $cachedInput->storepath; - $sha256 = $cachedInput->sha256hash; - } else { - print STDERR "checking out Mercurial input from $uri $branch revision $revision\n"; - $ENV{"NIX_HASH_ALGO"} = "sha256"; - $ENV{"PRINT_PATH"} = "1"; - - (my $res, $stdout, $stderr) = captureStdoutStderr(600, - "nix-prefetch-hg", $clonePath, $revision); - die "cannot check out Mercurial repository `$uri':\n$stderr" if $res; - - ($sha256, $storePath) = split ' ', $stdout; - - txn_do($db, sub { - $db->resultset('CachedHgInputs')->update_or_create( - { uri => $uri - , branch => $branch - , revision => $revision - , sha256hash => $sha256 - , storepath => $storePath - }); - }); - } - - return - { uri => $uri - , branch => $branch - , storePath => $storePath - , sha256hash => $sha256 - , revision => $revision - , revCount => int($revCount) - }; -} - - sub fetchInput { - my ($db, $project, $jobset, $name, $type, $value) = @_; + my ($plugins, $db, $project, $jobset, $name, $type, $value) = @_; my @inputs; - if ($type eq "path") { - push @inputs, fetchInputPath($db, $project, $jobset, $name, $value); - } - elsif ($type eq "svn") { - push @inputs, fetchInputSVN($db, $project, $jobset, $name, $value, 0); - } - elsif ($type eq "svn-checkout") { - push @inputs, fetchInputSVN($db, $project, $jobset, $name, $value, 1); - } - elsif ($type eq "build") { - push @inputs, fetchInputBuild($db, $project, $jobset, $name, $value); + if ($type eq "build") { + @inputs = fetchInputBuild($db, $project, $jobset, $name, $value); } elsif ($type eq "sysbuild") { - push @inputs, fetchInputSystemBuild($db, $project, $jobset, $name, $value); - } - elsif ($type eq "git") { - push @inputs, fetchInputGit($db, $project, $jobset, $name, $value); - } - elsif ($type eq "hg") { - push @inputs, fetchInputHg($db, $project, $jobset, $name, $value); - } - elsif ($type eq "bzr") { - push @inputs, fetchInputBazaar($db, $project, $jobset, $name, $value, 0); - } - elsif ($type eq "bzr-checkout") { - push @inputs, fetchInputBazaar($db, $project, $jobset, $name, $value, 1); + @inputs = fetchInputSystemBuild($db, $project, $jobset, $name, $value); } elsif ($type eq "string") { die unless defined $value; - push @inputs, { value => $value }; + @inputs = { value => $value }; } elsif ($type eq "boolean") { die unless defined $value && ($value eq "true" || $value eq "false"); - push @inputs, { value => $value }; + @inputs = { value => $value }; } else { - die "input `" . $name . "' has unknown type `$type'."; + my $found = 0; + foreach my $plugin (@{$plugins}) { + @inputs = $plugin->fetchInput($type, $name, $value); + if (defined $inputs[0]) { + $found = 1; + last; + } + } + die "input `$name' has unknown type `$type'." unless $found; } - foreach my $input (@inputs) { - $input->{type} = $type if defined $input; - } + $_->{type} = $type foreach @inputs; return @inputs; } @@ -641,6 +201,7 @@ sub booleanToString { return $result; } + sub buildInputToString { my ($exprType, $input) = @_; my $result; @@ -654,6 +215,7 @@ sub buildInputToString { ")"; } else { $result = "{ outPath = builtins.storePath " . $input->{storePath} . "" . + (defined $input->{revNumber} ? "; rev = " . $input->{revNumber} . "" : "") . (defined $input->{revision} ? "; rev = \"" . $input->{revision} . "\"" : "") . (defined $input->{revCount} ? "; revCount = " . $input->{revCount} . "" : "") . (defined $input->{gitTag} ? "; gitTag = \"" . $input->{gitTag} . "\"" : "") . @@ -681,16 +243,9 @@ sub inputsToArgs { when ("boolean") { push @res, "--arg", $input, booleanToString($exprType, $alt->{value}); } - when (["path", "build", "git", "hg", "sysbuild"]) { + default { push @res, "--arg", $input, buildInputToString($exprType, $alt); } - when (["svn", "svn-checkout", "bzr", "bzr-checkout"]) { - push @res, "--arg", $input, ( - "{ outPath = builtins.storePath " . $alt->{storePath} . "" . - (defined $alt->{revision} ? "; rev = " . $alt->{revision} . "" : "") . - ";}" - ); - } } } } @@ -699,28 +254,6 @@ sub inputsToArgs { } -sub captureStdoutStderr { - my ($timeout, @cmd) = @_; - my $stdin = ""; - my $stdout; - my $stderr; - - eval { - local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required - alarm $timeout; - IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr); - alarm 0; - }; - - if ($@) { - die unless $@ eq "timeout\n"; # propagate unexpected errors - return (-1, "", "timeout\n"); - } else { - return ($?, $stdout, $stderr); - } -} - - sub evalJobs { my ($inputInfo, $exprType, $nixExprInputName, $nixExprPath) = @_; diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index ababdfd8..571788b4 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -11,6 +11,7 @@ use Hydra::Model::DB; our @ISA = qw(Exporter); our @EXPORT = qw( getHydraHome getHydraConfig txn_do + getSCMCacheDir registerRoot getGCRootsDir gcRootFor getPrimaryBuildsForView getPrimaryBuildTotal @@ -18,7 +19,8 @@ our @EXPORT = qw( jobsetOverview removeAsciiEscapes getDrvLogPath logContents getMainOutput getEvals getMachines - pathIsInsidePrefix); + pathIsInsidePrefix + captureStdoutStderr); sub getHydraHome { @@ -50,6 +52,11 @@ sub txn_do { } +sub getSCMCacheDir { + return Hydra::Model::DB::getHydraPath . "/scm" ; +} + + sub getGCRootsDir { die unless defined $ENV{LOGNAME}; my $dir = ($ENV{NIX_STATE_DIR} || "/nix/var/nix" ) . "/gcroots/per-user/$ENV{LOGNAME}/hydra-roots"; @@ -443,4 +450,26 @@ sub pathIsInsidePrefix { } +sub captureStdoutStderr { + my ($timeout, @cmd) = @_; + my $stdin = ""; + my $stdout; + my $stderr; + + eval { + local $SIG{ALRM} = sub { die "timeout\n" }; # NB: \n required + alarm $timeout; + IPC::Run::run(\@cmd, \$stdin, \$stdout, \$stderr); + alarm 0; + }; + + if ($@) { + die unless $@ eq "timeout\n"; # propagate unexpected errors + return (-1, "", "timeout\n"); + } else { + return ($?, $stdout, $stderr); + } +} + + 1; diff --git a/src/lib/Hydra/Plugin.pm b/src/lib/Hydra/Plugin.pm index 6d07ab19..47a5faeb 100644 --- a/src/lib/Hydra/Plugin.pm +++ b/src/lib/Hydra/Plugin.pm @@ -20,4 +20,18 @@ sub buildFinished { my ($self, $build, $dependents) = @_; } +# Called to determine the set of supported input types. The plugin +# should add these to the $inputTypes hashref, e.g. $inputTypes{'svn'} +# = 'Subversion checkout'. +sub supportedInputTypes { + my ($self, $inputTypes) = @_; +} + +# Called to fetch an input of type ‘$type’. ‘$value’ is the input +# location, typically the repository URL. +sub fetchInput { + my ($self, $type, $name, $value) = @_; + return undef; +} + 1; diff --git a/src/lib/Hydra/Plugin/BazaarInput.pm b/src/lib/Hydra/Plugin/BazaarInput.pm new file mode 100644 index 00000000..5aa4b7a8 --- /dev/null +++ b/src/lib/Hydra/Plugin/BazaarInput.pm @@ -0,0 +1,87 @@ +package Hydra::Plugin::BazaarInput; + +use strict; +use parent 'Hydra::Plugin'; +use Digest::SHA qw(sha256_hex); +use File::Path; +use Hydra::Helper::Nix; +use Nix::Store; + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'bzr'} = 'Bazaar export'; + $inputTypes->{'bzr-checkout'} = 'Bazaar checkout'; +} + +sub fetchInput { + my ($self, $type, $name, $value) = @_; + + return undef if $type ne "bzr" && $type ne "bzr-checkout"; + + my $uri = $value; + + my $sha256; + my $storePath; + + my $stdout; my $stderr; + + my $cacheDir = getSCMCacheDir . "/bzr"; + mkpath($cacheDir); + my $clonePath = $cacheDir . "/" . sha256_hex($uri); + + if (! -d $clonePath) { + (my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "branch", $uri, $clonePath); + die "error cloning bazaar branch at `$uri':\n$stderr" if $res; + } + + chdir $clonePath or die $!; + (my $res, $stdout, $stderr) = captureStdoutStderr(600, "bzr", "pull"); + die "error pulling latest change bazaar branch at `$uri':\n$stderr" if $res; + + # First figure out the last-modified revision of the URI. + my @cmd = (["bzr", "revno"], "|", ["sed", 's/^ *\([0-9]*\).*/\1/']); + + IPC::Run::run(@cmd, \$stdout, \$stderr); + die "cannot get head revision of Bazaar branch at `$uri':\n$stderr" if $?; + my $revision = $stdout; chomp $revision; + die unless $revision =~ /^\d+$/; + + (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search( + {uri => $uri, revision => $revision}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + } else { + + # Then download this revision into the store. + print STDERR "checking out Bazaar input ", $name, " from $uri revision $revision\n"; + $ENV{"NIX_HASH_ALGO"} = "sha256"; + $ENV{"PRINT_PATH"} = "1"; + $ENV{"NIX_PREFETCH_BZR_LEAVE_DOT_BZR"} = $type eq "bzr-checkout" ? "1" : "0"; + + (my $res, $stdout, $stderr) = captureStdoutStderr(600, + "nix-prefetch-bzr", $clonePath, $revision); + die "cannot check out Bazaar branch `$uri':\n$stderr" if $res; + + ($sha256, $storePath) = split ' ', $stdout; + + txn_do($self->{db}, sub { + $self->{db}->resultset('CachedBazaarInputs')->create( + { uri => $uri + , revision => $revision + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } + + return + { uri => $uri + , storePath => $storePath + , sha256hash => $sha256 + , revision => $revision + }; +} + +1; diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm new file mode 100644 index 00000000..98c4c24a --- /dev/null +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -0,0 +1,148 @@ +package Hydra::Plugin::GitInput; + +use strict; +use parent 'Hydra::Plugin'; +use Digest::SHA qw(sha256_hex); +use File::Path; +use Hydra::Helper::Nix; +use Nix::Store; + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'git'} = 'Git checkout'; +} + +sub fetchInput { + my ($self, $type, $name, $value) = @_; + + return undef if $type ne "git"; + + (my $uri, my $branch, my $deepClone) = split ' ', $value; + $branch = defined $branch ? $branch : "master"; + + my $timestamp = time; + my $sha256; + my $storePath; + + my $cacheDir = getSCMCacheDir . "/git"; + mkpath($cacheDir); + my $clonePath = $cacheDir . "/" . sha256_hex($uri); + + my $stdout = ""; my $stderr = ""; my $res; + if (! -d $clonePath) { + # Clone everything and fetch the branch. + # TODO: Optimize the first clone by using "git init $clonePath" and "git remote add origin $uri". + ($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "clone", "--branch", $branch, $uri, $clonePath); + die "error cloning git repo at `$uri':\n$stderr" if $res; + } + + chdir $clonePath or die $!; # !!! urgh, shouldn't do a chdir + + # This command force the update of the local branch to be in the same as + # the remote branch for whatever the repository state is. This command mirror + # only one branch of the remote repository. + ($res, $stdout, $stderr) = captureStdoutStderr(600, + "git", "fetch", "-fu", "origin", "+$branch:$branch"); + ($res, $stdout, $stderr) = captureStdoutStderr(600, + "git", "fetch", "-fu", "origin") if $res; + die "error fetching latest change from git repo at `$uri':\n$stderr" if $res; + + ($res, $stdout, $stderr) = captureStdoutStderr(600, + ("git", "rev-parse", "$branch")); + die "error getting revision number of Git branch '$branch' at `$uri':\n$stderr" if $res; + + my ($revision) = split /\n/, $stdout; + die "error getting a well-formated revision number of Git branch '$branch' at `$uri':\n$stdout" + unless $revision =~ /^[0-9a-fA-F]+$/; + + my $ref = "refs/heads/$branch"; + + # If deepClone is defined, then we look at the content of the repository + # to determine if this is a top-git branch. + if (defined $deepClone) { + + # Checkout the branch to look at its content. + ($res, $stdout, $stderr) = captureStdoutStderr(600, "git", "checkout", "$branch"); + die "error checking out Git branch '$branch' at `$uri':\n$stderr" if $res; + + if (-f ".topdeps") { + # This is a TopGit branch. Fetch all the topic branches so + # that builders can run "tg patch" and similar. + ($res, $stdout, $stderr) = captureStdoutStderr(600, + "tg", "remote", "--populate", "origin"); + print STDERR "warning: `tg remote --populate origin' failed:\n$stderr" if $res; + } + } + + # Some simple caching: don't check a uri/branch/revision more than once. + # TODO: Fix case where the branch is reset to a previous commit. + my $cachedInput ; + ($cachedInput) = $self->{db}->resultset('CachedGitInputs')->search( + {uri => $uri, branch => $branch, revision => $revision}, + {rows => 1}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + $revision = $cachedInput->revision; + } else { + # Then download this revision into the store. + print STDERR "checking out Git branch $branch from $uri\n"; + $ENV{"NIX_HASH_ALGO"} = "sha256"; + $ENV{"PRINT_PATH"} = "1"; + $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "0"; + $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = ""; + + if (defined $deepClone) { + # Checked out code often wants to be able to run `git + # describe', e.g., code that uses Gnulib's `git-version-gen' + # script. Thus, we leave `.git' in there. Same for + # Subversion (e.g., libgcrypt's build system uses that.) + $ENV{"NIX_PREFETCH_GIT_LEAVE_DOT_GIT"} = "1"; + + # Ask for a "deep clone" to allow "git describe" and similar + # tools to work. See + # http://thread.gmane.org/gmane.linux.distributions.nixos/3569 + # for a discussion. + $ENV{"NIX_PREFETCH_GIT_DEEP_CLONE"} = "1"; + } + + ($res, $stdout, $stderr) = captureStdoutStderr(600, "nix-prefetch-git", $clonePath, $revision); + die "cannot check out Git repository branch '$branch' at `$uri':\n$stderr" if $res; + + ($sha256, $storePath) = split ' ', $stdout; + + txn_do($self->{db}, sub { + $self->{db}->resultset('CachedGitInputs')->update_or_create( + { uri => $uri + , branch => $branch + , revision => $revision + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } + + # For convenience in producing readable version names, pass the + # number of commits in the history of this revision (‘revCount’) + # the output of git-describe (‘gitTag’), and the abbreviated + # revision (‘shortRev’). + my $revCount = `git rev-list $revision | wc -l`; chomp $revCount; + die "git rev-list failed" if $? != 0; + my $gitTag = `git describe --always $revision`; chomp $gitTag; + die "git describe failed" if $? != 0; + my $shortRev = `git rev-parse --short $revision`; chomp $shortRev; + die "git rev-parse failed" if $? != 0; + + return + { uri => $uri + , storePath => $storePath + , sha256hash => $sha256 + , revision => $revision + , revCount => int($revCount) + , gitTag => $gitTag + , shortRev => $shortRev + }; +} + +1; diff --git a/src/lib/Hydra/Plugin/MercurialInput.pm b/src/lib/Hydra/Plugin/MercurialInput.pm new file mode 100644 index 00000000..02ea0ccc --- /dev/null +++ b/src/lib/Hydra/Plugin/MercurialInput.pm @@ -0,0 +1,88 @@ +package Hydra::Plugin::MercurialInput; + +use strict; +use parent 'Hydra::Plugin'; +use Digest::SHA qw(sha256_hex); +use File::Path; +use Hydra::Helper::Nix; +use Nix::Store; + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'hg'} = 'Mercurial checkout'; +} + +sub fetchInput { + my ($self, $type, $name, $value) = @_; + + return undef if $type ne "hg"; + + (my $uri, my $id) = split ' ', $value; + $id = defined $id ? $id : "default"; + + # init local hg clone + + my $stdout = ""; my $stderr = ""; + + my $cacheDir = getSCMCacheDir . "/hg"; + mkpath($cacheDir); + my $clonePath = $cacheDir . "/" . sha256_hex($uri); + + if (! -d $clonePath) { + (my $res, $stdout, $stderr) = captureStdoutStderr(600, + "hg", "clone", $uri, $clonePath); + die "error cloning mercurial repo at `$uri':\n$stderr" if $res; + } + + # hg pull + check rev + chdir $clonePath or die $!; + (my $res, $stdout, $stderr) = captureStdoutStderr(600, "hg", "pull"); + die "error pulling latest change mercurial repo at `$uri':\n$stderr" if $res; + + (my $res1, $stdout, $stderr) = captureStdoutStderr(600, + "hg", "log", "-r", $id, "--template", "{node|short} {rev} {branch}"); + die "error getting branch and revision of $id from `$uri':\n$stderr" if $res1; + + my ($revision, $revCount, $branch) = split ' ', $stdout; + + my $storePath; + my $sha256; + (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search( + {uri => $uri, branch => $branch, revision => $revision}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + } else { + print STDERR "checking out Mercurial input from $uri $branch revision $revision\n"; + $ENV{"NIX_HASH_ALGO"} = "sha256"; + $ENV{"PRINT_PATH"} = "1"; + + (my $res, $stdout, $stderr) = captureStdoutStderr(600, + "nix-prefetch-hg", $clonePath, $revision); + die "cannot check out Mercurial repository `$uri':\n$stderr" if $res; + + ($sha256, $storePath) = split ' ', $stdout; + + txn_do($self->{db}, sub { + $self->{db}->resultset('CachedHgInputs')->update_or_create( + { uri => $uri + , branch => $branch + , revision => $revision + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } + + return + { uri => $uri + , branch => $branch + , storePath => $storePath + , sha256hash => $sha256 + , revision => $revision + , revCount => int($revCount) + }; +} + +1; diff --git a/src/lib/Hydra/Plugin/PathInput.pm b/src/lib/Hydra/Plugin/PathInput.pm new file mode 100644 index 00000000..d913782d --- /dev/null +++ b/src/lib/Hydra/Plugin/PathInput.pm @@ -0,0 +1,77 @@ +package Hydra::Plugin::PathInput; + +use strict; +use parent 'Hydra::Plugin'; +use POSIX qw(strftime); +use Hydra::Helper::Nix; +use Nix::Store; + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'path'} = 'Local path'; +} + +sub fetchInput { + my ($self, $type, $name, $value) = @_; + + return undef if $type ne "path"; + + my $uri = $value; + + my $timestamp = time; + my $sha256; + my $storePath; + + # Some simple caching: don't check a path more than once every N seconds. + (my $cachedInput) = $self->{db}->resultset('CachedPathInputs')->search( + {srcpath => $uri, lastseen => {">", $timestamp - 30}}, + {rows => 1, order_by => "lastseen DESC"}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + $timestamp = $cachedInput->timestamp; + } else { + + print STDERR "copying input ", $name, " from $uri\n"; + $storePath = `nix-store --add "$uri"` + or die "cannot copy path $uri to the Nix store.\n"; + chomp $storePath; + + $sha256 = (queryPathInfo($storePath, 0))[1] or die; + + ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search( + {srcpath => $uri, sha256hash => $sha256}); + + # Path inputs don't have a natural notion of a "revision", so + # we simulate it by using the timestamp that we first saw this + # path have this SHA-256 hash. So if the contents of the path + # changes, we get a new "revision", but if it doesn't change + # (or changes back), we don't get a new "revision". + if (!defined $cachedInput) { + txn_do($self->{db}, sub { + $self->{db}->resultset('CachedPathInputs')->update_or_create( + { srcpath => $uri + , timestamp => $timestamp + , lastseen => $timestamp + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } else { + $timestamp = $cachedInput->timestamp; + txn_do($self->{db}, sub { + $cachedInput->update({lastseen => time}); + }); + } + } + + return + { uri => $uri + , storePath => $storePath + , sha256hash => $sha256 + , revision => strftime "%Y%m%d%H%M%S", gmtime($timestamp) + }; +} + +1; diff --git a/src/lib/Hydra/Plugin/SubversionInput.pm b/src/lib/Hydra/Plugin/SubversionInput.pm new file mode 100644 index 00000000..46b479de --- /dev/null +++ b/src/lib/Hydra/Plugin/SubversionInput.pm @@ -0,0 +1,90 @@ +package Hydra::Plugin::SubversionInput; + +use strict; +use parent 'Hydra::Plugin'; +use Digest::SHA qw(sha256_hex); +use Hydra::Helper::Nix; +use IPC::Run; +use Nix::Store; + +sub supportedInputTypes { + my ($self, $inputTypes) = @_; + $inputTypes->{'svn'} = 'Subversion export'; + $inputTypes->{'svn-checkout'} = 'Subversion checkout'; +} + +sub fetchInput { + my ($self, $type, $name, $value) = @_; + + return undef if $type ne "svn" && $type ne "svn-checkout"; + + # Allow users to specify a revision number next to the URI. + my ($uri, $revision) = split ' ', $value; + + my $sha256; + my $storePath; + my $stdout; my $stderr; + + unless (defined $revision) { + # First figure out the last-modified revision of the URI. + my @cmd = (["svn", "ls", "-v", "--depth", "empty", $uri], + "|", ["sed", 's/^ *\([0-9]*\).*/\1/']); + IPC::Run::run(@cmd, \$stdout, \$stderr); + die "cannot get head revision of Subversion repository at `$uri':\n$stderr" if $?; + $revision = int($stdout); $revision =~ s/\s*([0-9]+)\s*/$1/sm; + } + + die unless $revision =~ /^\d+$/; + $revision = int($revision); + + # Do we already have this revision in the store? + # !!! This needs to take $checkout into account! Otherwise "svn" + # and "svn-checkout" inputs can get mixed up. + (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search( + {uri => $uri, revision => $revision}); + + if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + $storePath = $cachedInput->storepath; + $sha256 = $cachedInput->sha256hash; + } else { + + # No, do a checkout. The working copy is reused between + # invocations to speed things up. + my $wcPath = getSCMCacheDir . "/svn/" . sha256_hex($uri) . "/svn-checkout"; + + print STDERR "checking out Subversion input ", $name, " from $uri revision $revision into $wcPath\n"; + + (my $res, $stdout, $stderr) = captureStdoutStderr(600, "svn", "checkout", $uri, "-r", $revision, $wcPath); + die "error checking out Subversion repo at `$uri':\n$stderr" if $res; + + if ($type eq "svn-checkout") { + $storePath = addToStore($wcPath, 1, "sha256"); + } else { + # Hm, if the Nix Perl bindings supported filters in + # addToStore(), then we wouldn't need to make a copy here. + my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die; + (system "svn", "export", $wcPath, "$tmpDir/svn-export", "--quiet") == 0 or die "svn export failed"; + $storePath = addToStore("$tmpDir/svn-export", 1, "sha256"); + } + + $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://; + + txn_do($self->{db}, sub { + $self->{db}->resultset('CachedSubversionInputs')->update_or_create( + { uri => $uri + , revision => $revision + , sha256hash => $sha256 + , storepath => $storePath + }); + }); + } + + return + { uri => $uri + , storePath => $storePath + , sha256hash => $sha256 + , revNumber => $revision + }; +} + +1; diff --git a/src/root/common.tt b/src/root/common.tt index 0fa0c233..280d3792 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -4,20 +4,6 @@ USE String; USE HTML; -inputTypes = - { "svn" = "Subversion export" - , "svn-checkout" = "Subversion checkout" - , "bzr" = "Bazaar export" - , "bzr-checkout" = "Bazaar checkout" - , "git" = "Git checkout" - , "hg" = "Mercurial checkout" - , "string" = "String value" - , "boolean" = "Boolean" - , "path" = "Local path" - , "build" = "Build output" - , "sysbuild" = "Build output (same system)" - }; - BLOCK renderDateTime; date.format(timestamp, '%Y-%m-%d %H:%M:%S'); END; diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index 734d70ab..efac4790 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -167,5 +167,3 @@ [% END %] - - diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index d2d61048..dd6ffa7e 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -3,6 +3,7 @@ use strict; use feature 'switch'; use Hydra::Schema; +use Hydra::Plugin; use Hydra::Helper::Nix; use Hydra::Helper::AddBuilds; use Hydra::Model::DB; @@ -20,6 +21,8 @@ STDOUT->autoflush(); my $db = Hydra::Model::DB->new(); my $config = getHydraConfig(); +my $plugins = [Hydra::Plugin->plugins(db => $db, config => $config)]; + # Don't check a jobset more than once every five minutes. my $minCheckInterval = 5 * 60; @@ -29,10 +32,8 @@ sub fetchInputs { my ($project, $jobset, $inputInfo) = @_; foreach my $input ($jobset->jobsetinputs->all) { foreach my $alt ($input->jobsetinputalts->all) { - my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value); - foreach my $info_el (@info) { - push @{$$inputInfo{$input->name}}, $info_el if defined $info_el; - } + push @{$$inputInfo{$input->name}}, $_ + foreach fetchInput($plugins, $db, $project, $jobset, $input->name, $input->type, $alt->value); } } }