diff --git a/src/Hydra/lib/Hydra/Controller/Root.pm b/src/Hydra/lib/Hydra/Controller/Root.pm index cfcca540..2ba4e732 100644 --- a/src/Hydra/lib/Hydra/Controller/Root.pm +++ b/src/Hydra/lib/Hydra/Controller/Root.pm @@ -56,8 +56,9 @@ sub index :Path :Args(0) { # Get the latest finished build for each unique job. $c->stash->{latestBuilds} = [$c->model('DB::Builds')->search(undef, { join => 'resultInfo' - , where => "finished != 0 and timestamp = (select max(timestamp) from Builds where project == me.project and attrName == me.attrName and finished != 0)" - , order_by => "project, attrname" + , where => "finished != 0 and timestamp = (select max(timestamp) from Builds " . + "where project == me.project and attrName == me.attrName and finished != 0 and system == me.system)" + , order_by => "project, attrname, system" })]; } diff --git a/src/Hydra/lib/Hydra/Schema.pm b/src/Hydra/lib/Hydra/Schema.pm index 3be5f210..e192e083 100644 --- a/src/Hydra/lib/Hydra/Schema.pm +++ b/src/Hydra/lib/Hydra/Schema.pm @@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema'; __PACKAGE__->load_classes; -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8AQc7Q1I5DrtSjFp722iMA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:iYwU4nHj9C2HkHwpYlVygQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildinputs.pm b/src/Hydra/lib/Hydra/Schema/Buildinputs.pm index 15efc07f..0c5eb4e8 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildinputs.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildinputs.pm @@ -36,8 +36,8 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); __PACKAGE__->belongs_to("dependency", "Hydra::Schema::Builds", { id => "dependency" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:NSUTUwG2qbvbFkZxezeSEA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H8JAWhQt1ehZTEEPH2KGBw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildproducts.pm b/src/Hydra/lib/Hydra/Schema/Buildproducts.pm index 2b1a933e..9b0fa38a 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildproducts.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildproducts.pm @@ -33,8 +33,8 @@ __PACKAGE__->set_primary_key("build", "productnr"); __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9b3ZNy73Cjat0gr5nQHIpA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EAo0rZSn+5Q5OY7Y1xjuyA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm b/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm index 98460dca..6c994ef9 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildresultinfo.pm @@ -27,8 +27,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uTiPHxrs6S19Laa/qeb7CA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5R03Kdy5ujkr7s9OfCNbTA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Builds.pm b/src/Hydra/lib/Hydra/Schema/Builds.pm index 70091220..68eca555 100644 --- a/src/Hydra/lib/Hydra/Schema/Builds.pm +++ b/src/Hydra/lib/Hydra/Schema/Builds.pm @@ -70,8 +70,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Ky0EF25kaZIaY+ofkP0ohw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:G1KBKGvEgnTcrgDkUnG1UQ __PACKAGE__->has_many(dependents => 'Hydra::Schema::Buildinputs', 'dependency'); diff --git a/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm b/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm index 658cd3a5..3e3d2ba4 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildschedulinginfo.pm @@ -23,8 +23,8 @@ __PACKAGE__->set_primary_key("id"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kt5gBt14ay7/DiIaQpKopA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MkMR6pE6mrN2BQL18ROYfA # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Buildsteps.pm b/src/Hydra/lib/Hydra/Schema/Buildsteps.pm index 420d86f8..73eae700 100644 --- a/src/Hydra/lib/Hydra/Schema/Buildsteps.pm +++ b/src/Hydra/lib/Hydra/Schema/Buildsteps.pm @@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id", "stepnr"); __PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ljGW5oOaJQ/uUKQx80W5SA +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:JbawGso3rboFeQMX6XfgXQ # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm b/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm index 59e9e62b..095b39bb 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsetinputalts.pm @@ -31,8 +31,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:mG0ka8XxC3ZEs/KgGKe5Hg +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7+oJOQegjLhp5JeuCS0KPw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm b/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm index c43d0af4..6bcfc9be 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsetinputs.pm @@ -43,8 +43,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:trcxVA3lLfgTC0TCHL3mVw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:BCzeQkhJYieHoL4ppjdJvw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Jobsets.pm b/src/Hydra/lib/Hydra/Schema/Jobsets.pm index b1564fc8..f158e9f2 100644 --- a/src/Hydra/lib/Hydra/Schema/Jobsets.pm +++ b/src/Hydra/lib/Hydra/Schema/Jobsets.pm @@ -18,6 +18,10 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 0, size => undef }, "nixexprpath", { data_type => "text", is_nullable => 0, size => undef }, + "errormsg", + { data_type => "text", is_nullable => 0, size => undef }, + "errortime", + { data_type => "integer", is_nullable => 0, size => undef }, ); __PACKAGE__->set_primary_key("project", "name"); __PACKAGE__->has_many( @@ -44,8 +48,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MYl8lWfWLCIAGSulR3m5zw +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:kEcdBc9XKVhz6zO/XEkAxg # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/lib/Hydra/Schema/Projects.pm b/src/Hydra/lib/Hydra/Schema/Projects.pm index 0ea71191..2e00b952 100644 --- a/src/Hydra/lib/Hydra/Schema/Projects.pm +++ b/src/Hydra/lib/Hydra/Schema/Projects.pm @@ -30,8 +30,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 11:59:19 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tLlYQAkX5CeUR2GJir8ifg +# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-25 12:58:40 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:rRYzLbd9BlZAQdOjT/GQcw # You can replace this text with custom content, and it will be preserved on regeneration diff --git a/src/Hydra/programs/Scheduler.pl b/src/Hydra/programs/Scheduler.pl index 2d7f6381..8c8746ef 100644 --- a/src/Hydra/programs/Scheduler.pl +++ b/src/Hydra/programs/Scheduler.pl @@ -3,6 +3,7 @@ use strict; use XML::Simple; use Hydra::Schema; +use IPC::Run; my $db = Hydra::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {}); @@ -16,6 +17,13 @@ sub isValidPath { } +sub captureStdoutStderr { + my $stdin = ""; my $stdout; my $stderr; + my $res = IPC::Run::run(\@_, \$stdin, \$stdout, \$stderr); + return ($res, $stdout, $stderr); +} + + sub getStorePathHash { my ($storePath) = @_; my $hash = `nix-store --query --hash $storePath` @@ -36,10 +44,11 @@ sub fetchInput { if ($type eq "path") { my $uri = $alt->value; + print "copying input ", $input->name, " from $uri\n"; + my $storePath = `nix-store --add "$uri"` or die "cannot copy path $uri to the Nix store"; chomp $storePath; - print " copied to $storePath\n"; $$inputInfo{$input->name} = { type => $type @@ -64,13 +73,16 @@ sub checkJob { my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs) = @_; # Instantiate the store derivation. - my $drvPath = `nix-instantiate $nixExprPath --attr $jobName $extraArgs` - or die "cannot evaluate the Nix expression containing the job definitions: $?"; + (my $res, my $drvPath, my $stderr) = captureStdoutStderr( + "nix-instantiate", $nixExprPath, "--attr", $jobName, @{$extraArgs}); + die "cannot evaluate the Nix expression for job `$jobName':\n$stderr" unless $res; chomp $drvPath; - # Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...). - my $infoXml = `nix-env -f $nixExprPath --query --available "*" --attr-path --out-path --drv-path --meta --xml --system-filter "*" --attr $jobName $extraArgs` - or die "cannot get information about the job: $?"; + # Call nix-env --xml to get info about this job (drvPath, outPath, meta attributes, ...). + ($res, my $infoXml, $stderr) = captureStdoutStderr( + qw(nix-env --query --available * --attr-path --out-path --drv-path --meta --xml --system-filter *), + "-f", $nixExprPath, "--attr", $jobName, @{$extraArgs}); + die "cannot get information about the job `$jobName':\n$stderr" unless $res; my $info = XMLin($infoXml, ForceArray => 1, KeyAttr => ['attrPath', 'name']) or die "cannot parse XML output"; @@ -87,11 +99,11 @@ sub checkJob { { project => $project->name, jobset => $jobset->name , attrname => $jobName, outPath => $outPath })) > 0) { - print " already scheduled/done\n"; + print "already scheduled/done\n"; return; } - print " adding to queue\n"; + print "adding to queue\n"; my $build = $db->resultset('Builds')->create( { finished => 0 @@ -132,6 +144,18 @@ sub checkJob { }; +sub setJobsetError { + my ($jobset, $errorMsg) = @_; + eval { + $db->txn_do(sub { + $jobset->errormsg($errorMsg); + $jobset->errortime(time); + $jobset->update; + }); + }; +} + + sub checkJobAlternatives { my ($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs, $argsNeeded, $n) = @_; @@ -139,12 +163,15 @@ sub checkJobAlternatives { eval { checkJob($project, $jobset, $inputInfo, $nixExprPath, $jobName, $jobExpr, $extraArgs); }; - warn $@ if $@; + if ($@) { + print "error evaluating job `", $jobName, "': $@"; + setJobsetError($jobset, $@); + } return; } my $argName = @{$argsNeeded}[$n]; - print " finding alternatives for argument $argName\n"; + #print "finding alternatives for argument $argName\n"; my ($input) = $jobset->jobsetinputs->search({name => $argName}); @@ -153,18 +180,17 @@ sub checkJobAlternatives { if (defined $input) { foreach my $alt ($input->jobsetinputalts) { - print " INPUT ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n"; + #print "input ", $input->name, " (type ", $input->type, ") alt ", $alt->altnr, "\n"; fetchInput($input, $alt, $inputInfo); # !!! caching - my $newArgs = ""; + my @newArgs = @{$extraArgs}; if (defined $inputInfo->{$argName}->{storePath}) { - # !!! escaping - $newArgs = " --arg $argName '{path = " . $inputInfo->{$argName}->{storePath} . ";}'"; + push @newArgs, "--arg", $argName, "{path = " . $inputInfo->{$argName}->{storePath} . ";}"; } elsif (defined $inputInfo->{$argName}->{value}) { - $newArgs = " --argstr $argName '" . $inputInfo->{$argName}->{value} . "'"; + push @newArgs, "--argstr", $argName, $inputInfo->{$argName}->{value}; } checkJobAlternatives( $project, $jobset, $inputInfo, $nixExprPath, - $jobName, $jobExpr, $extraArgs . $newArgs, $argsNeeded, $n + 1); + $jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1); } } @@ -192,15 +218,16 @@ sub checkJobAlternatives { , id => $prevBuild->id }; + my @newArgs = @{$extraArgs}; + push @newArgs, "--arg", $argName, "{path = " . $prevBuild->outpath . ";}"; + checkJobAlternatives( $project, $jobset, $inputInfo, $nixExprPath, - $jobName, $jobExpr, - $extraArgs . " --arg $argName '{path = " . $prevBuild->outpath . ";}'", - $argsNeeded, $n + 1); + $jobName, $jobExpr, \@newArgs, $argsNeeded, $n + 1); } } - + sub checkJobSet { my ($project, $jobset) = @_; my $inputInfo = {}; @@ -216,10 +243,11 @@ sub checkJobSet { # Evaluate the Nix expression. my $nixExprPath = $inputInfo->{$jobset->nixexprinput}->{storePath} . "/" . $jobset->nixexprpath; - print " EVALUATING $nixExprPath\n"; - - my $jobsXml = `nix-instantiate $nixExprPath --eval-only --strict --xml` - or die "cannot evaluate the Nix expression containing the jobs: $?"; + print "evaluating $nixExprPath\n"; + + (my $res, my $jobsXml, my $stderr) = captureStdoutStderr( + "nix-instantiate", $nixExprPath, "--eval-only", "--strict", "--xml"); + die "cannot evaluate the Nix expression containing the jobs:\n$stderr" unless $res; my $jobs = XMLin($jobsXml, ForceArray => ['value', 'attr'], @@ -235,7 +263,7 @@ sub checkJobSet { # function, then fill in the function arguments with the # (alternative) values supplied in the jobsetinputs table. foreach my $jobName (keys(%{$jobs->{attrs}->{attr}})) { - print " JOB $jobName\n"; + print "considering job $jobName\n"; my @argsNeeded = (); @@ -244,7 +272,7 @@ sub checkJobSet { # !!! fix the case where there is only 1 attr, XML::Simple fucks up as usual if (defined $jobExpr->{function}->{attrspat}) { foreach my $argName (keys(%{$jobExpr->{function}->{attrspat}->{attr}})) { - print " needs input $argName\n"; + #print "needs input $argName\n"; push @argsNeeded, $argName; } } @@ -252,7 +280,7 @@ sub checkJobSet { eval { checkJobAlternatives( $project, $jobset, {}, $nixExprPath, - $jobName, $jobExpr, "", \@argsNeeded, 0); + $jobName, $jobExpr, [], \@argsNeeded, 0); }; warn $@ if $@; } @@ -262,14 +290,17 @@ sub checkJobSet { sub checkJobs { foreach my $project ($db->resultset('Projects')->search({enabled => 1})) { - print "PROJECT ", $project->name, "\n"; + print "considering project ", $project->name, "\n"; foreach my $jobset ($project->jobsets->all) { - print " JOBSET ", $jobset->name, "\n"; + print "considering jobset ", $jobset->name, " in ", $project->name, "\n"; eval { checkJobSet($project, $jobset); - }; - warn $@ if $@; - } + }; + if ($@) { + print "error evaluating jobset ", $jobset->name, ": $@"; + setProjectError($jobset, $@); + } + } } } diff --git a/src/Hydra/root/build.tt b/src/Hydra/root/build.tt index 4576f02b..9c054dba 100644 --- a/src/Hydra/root/build.tt +++ b/src/Hydra/root/build.tt @@ -1,7 +1,6 @@ [% WRAPPER layout.tt title="Hydra Overview" %] [% PROCESS common.tt %] [% USE HTML %] -[% USE date %] [% USE mibs=format("%.2f") %]