Support passing a jobset evaluation as an input

All successful, non-garbage-collected builds in the evaluation are
passed in a attribute set.  So if you declare a Hydra input named
‘foo’ of type ‘eval’, you get a set with members ‘foo.<jobname>’.  For
instance, if you passed a Nixpkgs eval as an input named ‘nixpkgs’,
then you could get the Firefox build for x86_64-linux as
‘nixpkgs.firefox.x86_64-linux’.

Inputs of type ‘eval’ can be specified in three ways:

* As the number of the evaluation.

* As a jobset identifier (‘<project>:<jobset>’), which will yield the
  latest finished evaluation of that jobset.  Note that there is no
  guarantee that any job in that evaluation has succeeded, so it might
  not be very useful.

* As a job identifier (‘<project>:<jobset>:<job>’), which will yield
  the latest finished evaluation of that jobset in which <job>
  succeeded.  In conjunction with aggregate jobs, this allows you to
  make sure that the evaluation contains the desired builds.
This commit is contained in:
Eelco Dolstra 2013-11-11 21:17:22 +00:00
parent 7b35e4d0de
commit 8f104396ec
5 changed files with 73 additions and 7 deletions

View file

@ -173,10 +173,16 @@ sub nixExprPathFromParams {
sub checkInputValue { sub checkInputValue {
my ($c, $type, $value) = @_; my ($c, $name, $type, $value) = @_;
$value = trim $value; $value = trim $value;
error($c, "Invalid Boolean value $value.") if error($c, "The value $value of input $name is not a Boolean (true or false).") if
$type eq "boolean" && !($value eq "true" || $value eq "false"); $type eq "boolean" && !($value eq "true" || $value eq "false");
error($c, "The value $value of input $name does not specify a Hydra evaluation. "
. "It should be either the number of a specific evaluation, the name of "
. "a jobset (given as <project>:<jobset>), or the name of a job (<project>:<jobset>:<job>).")
if $type eq "eval" && $value !~ /^\d+$/
&& $value !~ /^$projectNameRE:$jobsetNameRE$/
&& $value !~ /^$projectNameRE:$jobsetNameRE:$jobNameRE$/;
return $value; return $value;
} }
@ -237,7 +243,7 @@ sub updateJobset {
my @values = ref($values) eq 'ARRAY' ? @{$values} : ($values); my @values = ref($values) eq 'ARRAY' ? @{$values} : ($values);
my $altnr = 0; my $altnr = 0;
foreach my $value (@values) { foreach my $value (@values) {
$value = checkInputValue($c, $type, $value); $value = checkInputValue($c, $name, $type, $value);
$input->jobsetinputalts->create({altnr => $altnr++, value => $value}); $input->jobsetinputalts->create({altnr => $altnr++, value => $value});
} }
} }
@ -294,7 +300,7 @@ sub evals_GET {
# Redirect to the latest finished evaluation of this jobset. # Redirect to the latest finished evaluation of this jobset.
sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') { sub latest_eval : Chained('jobsetChain') PathPart('latest-eval') {
my ($self, $c, @args) = @_; my ($self, $c, @args) = @_;
my $eval = getLatestFinishedEval($c, $c->stash->{jobset}) my $eval = getLatestFinishedEval($c->stash->{jobset})
or notFound($c, "No evaluation found."); or notFound($c, "No evaluation found.");
$c->res->redirect($c->uri_for($c->controller('JobsetEval')->action_for("view"), [$eval->id], @args, $c->req->params)); $c->res->redirect($c->uri_for($c->controller('JobsetEval')->action_for("view"), [$eval->id], @args, $c->req->params));
} }

View file

@ -47,7 +47,7 @@ sub view : Chained('eval') PathPart('') Args(0) {
} elsif (defined $compare && $compare =~ /^($jobsetNameRE)$/) { } elsif (defined $compare && $compare =~ /^($jobsetNameRE)$/) {
my $j = $c->stash->{project}->jobsets->find({name => $compare}) my $j = $c->stash->{project}->jobsets->find({name => $compare})
or notFound($c, "Jobset $compare doesn't exist."); or notFound($c, "Jobset $compare doesn't exist.");
$eval2 = getLatestFinishedEval($c, $j); $eval2 = getLatestFinishedEval($j);
} else { } else {
notFound($c, "Unknown comparison source $compare."); notFound($c, "Unknown comparison source $compare.");
} }

View file

@ -37,7 +37,8 @@ sub begin :Private {
'boolean' => 'Boolean', 'boolean' => 'Boolean',
'nix' => 'Nix expression', 'nix' => 'Nix expression',
'build' => 'Build output', 'build' => 'Build output',
'sysbuild' => 'Build output (same system)' 'sysbuild' => 'Build output (same system)',
'eval' => 'Previous Hydra evaluation'
}; };
$_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins}; $_->supportedInputTypes($c->stash->{inputTypes}) foreach @{$c->hydra_plugins};

View file

@ -17,6 +17,7 @@ use File::Temp;
use File::Spec; use File::Spec;
use File::Slurp; use File::Slurp;
use Hydra::Helper::PluginHooks; use Hydra::Helper::PluginHooks;
use Hydra::Helper::CatalystUtils;
our @ISA = qw(Exporter); our @ISA = qw(Exporter);
our @EXPORT = qw( our @EXPORT = qw(
@ -147,6 +148,51 @@ sub fetchInputSystemBuild {
return @inputs; return @inputs;
} }
sub fetchInputEval {
my ($db, $project, $jobset, $name, $value) = @_;
my $eval;
if ($value =~ /^\d+$/) {
$eval = $db->resultset('JobsetEvals')->find({ id => int($value) });
die "evaluation $eval->{id} does not exist\n" unless defined $eval;
} elsif ($value =~ /^($projectNameRE):($jobsetNameRE)$/) {
my $jobset = $db->resultset('Jobsets')->find({ project => $1, name => $2 });
die "jobset $value does not exist\n" unless defined $jobset;
$eval = getLatestFinishedEval($jobset);
die "jobset $value does not have a finished evaluation\n" unless defined $eval;
} elsif ($value =~ /^($projectNameRE):($jobsetNameRE):($jobNameRE)$/) {
$eval = $db->resultset('JobsetEvals')->find(
{ project => $1, jobset => $2, hasnewbuilds => 1 },
{ order_by => "id DESC", rows => 1
, where =>
\ [ # All builds in this jobset should be finished...
"not exists (select 1 from JobsetEvalMembers m join Builds b on m.build = b.id where m.eval = me.id and b.finished = 0) "
# ...and the specified build must have succeeded.
. "and exists (select 1 from JobsetEvalMembers m join Builds b on m.build = b.id where m.eval = me.id and b.job = ? and b.buildstatus = 0)"
, [ 'name', $3 ] ]
});
die "there is no successful build of $value in a finished evaluation\n" unless defined $eval;
} else {
die;
}
my $jobs = {};
foreach my $build ($eval->builds) {
next unless $build->finished == 1 && $build->buildstatus == 0;
# FIXME: Handle multiple outputs.
my $out = $build->buildoutputs->find({ name => "out" });
next unless defined $out;
# FIXME: Should we fail if the path is not valid?
next unless isValidPath($out->path);
$jobs->{$build->get_column('job')} = $out->path;
}
return { jobs => $jobs };
}
sub fetchInput { sub fetchInput {
my ($plugins, $db, $project, $jobset, $name, $type, $value, $emailresponsible) = @_; my ($plugins, $db, $project, $jobset, $name, $type, $value, $emailresponsible) = @_;
my @inputs; my @inputs;
@ -157,6 +203,9 @@ sub fetchInput {
elsif ($type eq "sysbuild") { elsif ($type eq "sysbuild") {
@inputs = fetchInputSystemBuild($db, $project, $jobset, $name, $value); @inputs = fetchInputSystemBuild($db, $project, $jobset, $name, $value);
} }
elsif ($type eq "eval") {
@inputs = fetchInputEval($db, $project, $jobset, $name, $value);
}
elsif ($type eq "string" || $type eq "nix") { elsif ($type eq "string" || $type eq "nix") {
die unless defined $value; die unless defined $value;
@inputs = { value => $value }; @inputs = { value => $value };
@ -245,8 +294,18 @@ sub inputsToArgs {
push @res, "--arg", $input, booleanToString($exprType, $alt->{value}); push @res, "--arg", $input, booleanToString($exprType, $alt->{value});
} }
when ("nix") { when ("nix") {
die "input type nix only supported for Nix-based jobsets\n" unless $exprType eq "nix";
push @res, "--arg", $input, $alt->{value}; push @res, "--arg", $input, $alt->{value};
} }
when ("eval") {
die "input type eval only supported for Nix-based jobsets\n" unless $exprType eq "nix";
my $s = "{ ";
# FIXME: escape $_. But dots should not be escaped.
$s .= "$_ = builtins.storePath ${\$alt->{jobs}->{$_}}; "
foreach keys %{$alt->{jobs}};
$s .= "}";
push @res, "--arg", $input, $s;
}
default { default {
push @res, "--arg", $input, buildInputToString($exprType, $alt); push @res, "--arg", $input, buildInputToString($exprType, $alt);
} }

View file

@ -176,7 +176,7 @@ sub trim {
sub getLatestFinishedEval { sub getLatestFinishedEval {
my ($c, $jobset) = @_; my ($jobset) = @_;
my ($eval) = $jobset->jobsetevals->search( my ($eval) = $jobset->jobsetevals->search(
{ hasnewbuilds => 1 }, { hasnewbuilds => 1 },
{ order_by => "id DESC", rows => 1 { order_by => "id DESC", rows => 1