From 8f104396ec7c8a8fb8165bdbde9a1348a35fd239 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Nov 2013 21:17:22 +0000 Subject: [PATCH] Support passing a jobset evaluation as an input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All successful, non-garbage-collected builds in the evaluation are passed in a attribute set. So if you declare a Hydra input named ‘foo’ of type ‘eval’, you get a set with members ‘foo.’. For instance, if you passed a Nixpkgs eval as an input named ‘nixpkgs’, then you could get the Firefox build for x86_64-linux as ‘nixpkgs.firefox.x86_64-linux’. Inputs of type ‘eval’ can be specified in three ways: * As the number of the evaluation. * As a jobset identifier (‘:’), which will yield the latest finished evaluation of that jobset. Note that there is no guarantee that any job in that evaluation has succeeded, so it might not be very useful. * As a job identifier (‘::’), which will yield the latest finished evaluation of that jobset in which succeeded. In conjunction with aggregate jobs, this allows you to make sure that the evaluation contains the desired builds. --- src/lib/Hydra/Controller/Jobset.pm | 14 ++++-- src/lib/Hydra/Controller/JobsetEval.pm | 2 +- src/lib/Hydra/Controller/Root.pm | 3 +- src/lib/Hydra/Helper/AddBuilds.pm | 59 ++++++++++++++++++++++++++ src/lib/Hydra/Helper/CatalystUtils.pm | 2 +- 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 35b29d86..fb82a584 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -173,10 +173,16 @@ sub nixExprPathFromParams { sub checkInputValue { - my ($c, $type, $value) = @_; + my ($c, $name, $type, $value) = @_; $value = trim $value; - error($c, "Invalid Boolean value ‘$value’.") if + error($c, "The value ‘$value’ of input ‘$name’ is not a Boolean (‘true’ or ‘false’).") if $type eq "boolean" && !($value eq "true" || $value eq "false"); + error($c, "The value ‘$value’ of input ‘$name’ does not specify a Hydra evaluation. " + . "It should be either the number of a specific evaluation, the name of " + . "a jobset (given as :), or the name of a job (::).") + if $type eq "eval" && $value !~ /^\d+$/ + && $value !~ /^$projectNameRE:$jobsetNameRE$/ + && $value !~ /^$projectNameRE:$jobsetNameRE:$jobNameRE$/; return $value; } @@ -237,7 +243,7 @@ sub updateJobset { my @values = ref($values) eq 'ARRAY' ? @{$values} : ($values); my $altnr = 0; foreach my $value (@values) { - $value = checkInputValue($c, $type, $value); + $value = checkInputValue($c, $name, $type, $value); $input->jobsetinputalts->create({altnr => $altnr++, value => $value}); } } @@ -294,7 +300,7 @@ sub evals_GET { # Redirect to the latest finished evaluation of this jobset. sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') { my ($self, $c, @args) = @_; - my $eval = getLatestFinishedEval($c, $c->stash->{jobset}) + my $eval = getLatestFinishedEval($c->stash->{jobset}) or notFound($c, "No evaluation found."); $c->res->redirect($c->uri_for($c->controller('JobsetEval')->action_for("view"), [$eval->id], @args, $c->req->params)); } diff --git a/src/lib/Hydra/Controller/JobsetEval.pm b/src/lib/Hydra/Controller/JobsetEval.pm index 74ab3bce..a2d8187c 100644 --- a/src/lib/Hydra/Controller/JobsetEval.pm +++ b/src/lib/Hydra/Controller/JobsetEval.pm @@ -47,7 +47,7 @@ sub view : Chained('eval') PathPart('') Args(0) { } elsif (defined $compare && $compare =~ /^($jobsetNameRE)$/) { my $j = $c->stash->{project}->jobsets->find({name => $compare}) or notFound($c, "Jobset $compare doesn't exist."); - $eval2 = getLatestFinishedEval($c, $j); + $eval2 = getLatestFinishedEval($j); } else { notFound($c, "Unknown comparison source ‘$compare’."); } diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index ad34ae40..0d7660a0 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -37,7 +37,8 @@ sub begin :Private { 'boolean' => 'Boolean', 'nix' => 'Nix expression', 'build' => 'Build output', - 'sysbuild' => 'Build output (same system)' + 'sysbuild' => 'Build output (same system)', + 'eval' => 'Previous Hydra evaluation' }; $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins}; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index a7e5c9c8..9e432473 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -17,6 +17,7 @@ use File::Temp; use File::Spec; use File::Slurp; use Hydra::Helper::PluginHooks; +use Hydra::Helper::CatalystUtils; our @ISA = qw(Exporter); our @EXPORT = qw( @@ -147,6 +148,51 @@ sub fetchInputSystemBuild { return @inputs; } + +sub fetchInputEval { + my ($db, $project, $jobset, $name, $value) = @_; + + my $eval; + + if ($value =~ /^\d+$/) { + $eval = $db->resultset('JobsetEvals')->find({ id => int($value) }); + die "evaluation $eval->{id} does not exist\n" unless defined $eval; + } elsif ($value =~ /^($projectNameRE):($jobsetNameRE)$/) { + my $jobset = $db->resultset('Jobsets')->find({ project => $1, name => $2 }); + die "jobset ‘$value’ does not exist\n" unless defined $jobset; + $eval = getLatestFinishedEval($jobset); + die "jobset ‘$value’ does not have a finished evaluation\n" unless defined $eval; + } elsif ($value =~ /^($projectNameRE):($jobsetNameRE):($jobNameRE)$/) { + $eval = $db->resultset('JobsetEvals')->find( + { project => $1, jobset => $2, hasnewbuilds => 1 }, + { order_by => "id DESC", rows => 1 + , where => + \ [ # All builds in this jobset should be finished... + "not exists (select 1 from JobsetEvalMembers m join Builds b on m.build = b.id where m.eval = me.id and b.finished = 0) " + # ...and the specified build must have succeeded. + . "and exists (select 1 from JobsetEvalMembers m join Builds b on m.build = b.id where m.eval = me.id and b.job = ? and b.buildstatus = 0)" + , [ 'name', $3 ] ] + }); + die "there is no successful build of ‘$value’ in a finished evaluation\n" unless defined $eval; + } else { + die; + } + + my $jobs = {}; + foreach my $build ($eval->builds) { + next unless $build->finished == 1 && $build->buildstatus == 0; + # FIXME: Handle multiple outputs. + my $out = $build->buildoutputs->find({ name => "out" }); + next unless defined $out; + # FIXME: Should we fail if the path is not valid? + next unless isValidPath($out->path); + $jobs->{$build->get_column('job')} = $out->path; + } + + return { jobs => $jobs }; +} + + sub fetchInput { my ($plugins, $db, $project, $jobset, $name, $type, $value, $emailresponsible) = @_; my @inputs; @@ -157,6 +203,9 @@ sub fetchInput { elsif ($type eq "sysbuild") { @inputs = fetchInputSystemBuild($db, $project, $jobset, $name, $value); } + elsif ($type eq "eval") { + @inputs = fetchInputEval($db, $project, $jobset, $name, $value); + } elsif ($type eq "string" || $type eq "nix") { die unless defined $value; @inputs = { value => $value }; @@ -245,8 +294,18 @@ sub inputsToArgs { push @res, "--arg", $input, booleanToString($exprType, $alt->{value}); } when ("nix") { + die "input type ‘nix’ only supported for Nix-based jobsets\n" unless $exprType eq "nix"; push @res, "--arg", $input, $alt->{value}; } + when ("eval") { + die "input type ‘eval’ only supported for Nix-based jobsets\n" unless $exprType eq "nix"; + my $s = "{ "; + # FIXME: escape $_. But dots should not be escaped. + $s .= "$_ = builtins.storePath ${\$alt->{jobs}->{$_}}; " + foreach keys %{$alt->{jobs}}; + $s .= "}"; + push @res, "--arg", $input, $s; + } default { push @res, "--arg", $input, buildInputToString($exprType, $alt); } diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index e83bd0ea..22fe3474 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -176,7 +176,7 @@ sub trim { sub getLatestFinishedEval { - my ($c, $jobset) = @_; + my ($jobset) = @_; my ($eval) = $jobset->jobsetevals->search( { hasnewbuilds => 1 }, { order_by => "id DESC", rows => 1