diff --git a/.gitignore b/.gitignore index a073f4f1..5a0ab4a2 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ hydra-config.h.in result tests/jobs/config.nix outputs +config +stamp-h1 +src/hydra-evaluator/hydra-evaluator +src/hydra-queue-runner/hydra-queue-runner diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 44beb7fd..b8f6c2a2 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -54,6 +54,16 @@ __PACKAGE__->table("jobsetevals"); is_foreign_key: 1 is_nullable: 0 +=head2 errormsg + + data_type: 'text' + is_nullable: 1 + +=head2 errortime + + data_type: 'integer' + is_nullable: 1 + =head2 timestamp data_type: 'integer' @@ -108,6 +118,10 @@ __PACKAGE__->add_columns( { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "jobset", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "errormsg", + { data_type => "text", is_nullable => 1 }, + "errortime", + { data_type => "integer", is_nullable => 1 }, "timestamp", { data_type => "integer", is_nullable => 0 }, "checkouttime", @@ -201,8 +215,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:M61ikfnjORU7jDAH8P/j7w +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg __PACKAGE__->has_many( "buildIds", diff --git a/src/root/common.tt b/src/root/common.tt index bc777ede..8d4f4f70 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -476,6 +476,9 @@ BLOCK renderEvals %] ELSE %] - [% END %] + [% IF eval.errormsg %] + Eval Errors + [% END %] [% e.nrSucceeded %] diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index 29b8b6f6..139ee2fe 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -88,6 +88,11 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  • Queued jobs ([% unfinished.size %])
  • [% END %]
  • Inputs
  • + + [% IF eval.errormsg %] +
  • Evaluation errors
  • + [% END %] + [% BLOCK renderSome %] @@ -103,6 +108,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
    + [% IF eval.errormsg %] +
    +

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].

    +
    [% HTML.escape(eval.errormsg) %]
    +
    + [% END %] +
    [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
    @@ -160,6 +172,12 @@ c.uri_for(c.controller('JobsetEval').action_for('view'), [% END %]
    + [% IF eval.errormsg %] +
    +

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].

    +
    [% HTML.escape(eval.errormsg) %]
    +
    + [% END %] [% END %] diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index c6c0fc64..78938758 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -493,12 +493,12 @@ sub fetchInputs { sub setJobsetError { - my ($jobset, $errorMsg) = @_; + my ($jobset, $errorMsg, $errorTime) = @_; my $prevError = $jobset->errormsg; eval { $db->txn_do(sub { - $jobset->update({ errormsg => $errorMsg, errortime => time, fetcherrormsg => undef }); + $jobset->update({ errormsg => $errorMsg, errortime => $errorTime, fetcherrormsg => undef }); }); }; if (defined $errorMsg && $errorMsg ne ($prevError // "") || $ENV{'HYDRA_MAIL_TEST'}) { @@ -680,6 +680,18 @@ sub checkJobsetWrapped { my $jobsetChanged = 0; my $dbStart = clock_gettime(CLOCK_MONOTONIC); + + # Store the error messages for jobs that failed to evaluate. + my $evaluationErrorTime = time; + my $evaluationErrorMsg = ""; + foreach my $job (values %{$jobs}) { + next unless defined $job->{error}; + $evaluationErrorMsg .= + ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . + ":\n" . $job->{error} . "\n\n"; + } + setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); + my %buildMap; $db->txn_do(sub { @@ -706,6 +718,8 @@ sub checkJobsetWrapped { { hash => $argsHash , timestamp => time , checkouttime => abs(int($checkoutStop - $checkoutStart)) + , errormsg => $evaluationErrorMsg + , errortime => $evaluationErrorTime , evaltime => abs(int($evalStop - $evalStart)) , hasnewbuilds => $jobsetChanged ? 1 : 0 , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef @@ -791,16 +805,6 @@ sub checkJobsetWrapped { Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000)); Net::Statsd::increment("hydra.evaluator.evals"); Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged; - - # Store the error messages for jobs that failed to evaluate. - my $msg = ""; - foreach my $job (values %{$jobs}) { - next unless defined $job->{error}; - $msg .= - ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . - ":\n" . $job->{error} . "\n\n"; - } - setJobsetError($jobset, $msg); } @@ -827,9 +831,10 @@ sub checkJobset { my $failed = 0; if ($checkError) { print STDERR $checkError; + my $eventTime = time; $db->txn_do(sub { - $jobset->update({lastcheckedtime => time}); - setJobsetError($jobset, $checkError); + $jobset->update({lastcheckedtime => $eventTime}); + setJobsetError($jobset, $checkError, $eventTime); $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId)); }) if !$dryRun; $failed = 1; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 7f12ccfe..d8e2805c 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -440,6 +440,9 @@ create table JobsetEvals ( project text not null, jobset text not null, + errorMsg text, -- error output from the evaluator + errorTime integer, -- timestamp associated with errorMsg + timestamp integer not null, -- when this entry was added checkoutTime integer not null, -- how long obtaining the inputs took (in seconds) evalTime integer not null, -- how long evaluation took (in seconds) diff --git a/src/sql/upgrade-70.sql b/src/sql/upgrade-70.sql new file mode 100644 index 00000000..f0d75e38 --- /dev/null +++ b/src/sql/upgrade-70.sql @@ -0,0 +1,34 @@ +ALTER TABLE JobsetEvals + ADD COLUMN errorMsg text, + ADD COLUMN errorTime integer NULL; + +-- Copy the current error in jobsets to the latest field in jobsetevals +UPDATE jobsetevals + SET errorMsg = j.errorMsg, + errorTime = j.errorTime + FROM ( + SELECT + jobsets.errorMsg, + jobsets.errorTime, + jobsets.id AS jobset_id, + latesteval.id AS eval_id + FROM jobsets + LEFT JOIN + ( + SELECT + MAX(id) AS id, + project, + jobset + FROM jobsetevals + GROUP BY project, jobset + ORDER BY project, jobset + ) + AS latesteval + ON + jobsets.name = latesteval.jobset + AND jobsets.project = latesteval.project + WHERE latesteval.id IS NOT NULL + ORDER BY jobsets.id +) +AS j +WHERE id = j.eval_id; diff --git a/tests/set-up.pl b/tests/set-up.pl index d7aa35cc..4fb99a49 100644 --- a/tests/set-up.pl +++ b/tests/set-up.pl @@ -1,5 +1,5 @@ use strict; -system("initdb -D postgres") == 0 or die; -system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die; -system("createdb -p 6433 hydra-test-suite") == 0 or die; +system("initdb -D postgres --locale C.UTF-8 ") == 0 or die; +system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die; +system("createdb -l C.UTF-8 -p 6433 hydra-test-suite") == 0 or die; system("hydra-init") == 0 or die;