From d9989b7fa1c91bdaa13630c3029d36945b0edc9f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 21 Jan 2021 09:36:57 -0500 Subject: [PATCH 1/6] Schema: add errorMsg, errorTime to JobsetEvals --- src/lib/Hydra/Schema/JobsetEvals.pm | 18 +++++++++++++-- src/sql/hydra.sql | 3 +++ src/sql/upgrade-70.sql | 34 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 src/sql/upgrade-70.sql diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 44beb7fd..b8f6c2a2 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -54,6 +54,16 @@ __PACKAGE__->table("jobsetevals"); is_foreign_key: 1 is_nullable: 0 +=head2 errormsg + + data_type: 'text' + is_nullable: 1 + +=head2 errortime + + data_type: 'integer' + is_nullable: 1 + =head2 timestamp data_type: 'integer' @@ -108,6 +118,10 @@ __PACKAGE__->add_columns( { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "jobset", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "errormsg", + { data_type => "text", is_nullable => 1 }, + "errortime", + { data_type => "integer", is_nullable => 1 }, "timestamp", { data_type => "integer", is_nullable => 0 }, "checkouttime", @@ -201,8 +215,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:M61ikfnjORU7jDAH8P/j7w +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg __PACKAGE__->has_many( "buildIds", diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 7f12ccfe..d8e2805c 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -440,6 +440,9 @@ create table JobsetEvals ( project text not null, jobset text not null, + errorMsg text, -- error output from the evaluator + errorTime integer, -- timestamp associated with errorMsg + timestamp integer not null, -- when this entry was added checkoutTime integer not null, -- how long obtaining the inputs took (in seconds) evalTime integer not null, -- how long evaluation took (in seconds) diff --git a/src/sql/upgrade-70.sql b/src/sql/upgrade-70.sql new file mode 100644 index 00000000..f0d75e38 --- /dev/null +++ b/src/sql/upgrade-70.sql @@ -0,0 +1,34 @@ +ALTER TABLE JobsetEvals + ADD COLUMN errorMsg text, + ADD COLUMN errorTime integer NULL; + +-- Copy the current error in jobsets to the latest field in jobsetevals +UPDATE jobsetevals + SET errorMsg = j.errorMsg, + errorTime = j.errorTime + FROM ( + SELECT + jobsets.errorMsg, + jobsets.errorTime, + jobsets.id AS jobset_id, + latesteval.id AS eval_id + FROM jobsets + LEFT JOIN + ( + SELECT + MAX(id) AS id, + project, + jobset + FROM jobsetevals + GROUP BY project, jobset + ORDER BY project, jobset + ) + AS latesteval + ON + jobsets.name = latesteval.jobset + AND jobsets.project = latesteval.project + WHERE latesteval.id IS NOT NULL + ORDER BY jobsets.id +) +AS j +WHERE id = j.eval_id; From fb6b10a86ca76d78aa9486e42ff8b3f8a342c359 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 21 Jan 2021 11:12:49 -0500 Subject: [PATCH 2/6] gitignore: artifacts --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a073f4f1..5a0ab4a2 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ hydra-config.h.in result tests/jobs/config.nix outputs +config +stamp-h1 +src/hydra-evaluator/hydra-evaluator +src/hydra-queue-runner/hydra-queue-runner From 086eed5147f68e4d4bd635dc7012792797604bbe Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 21 Jan 2021 11:14:37 -0500 Subject: [PATCH 3/6] hydra-eval-jobs: write evaluation errorMsg to the jobseteval table --- src/script/hydra-eval-jobset | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index c6c0fc64..78938758 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -493,12 +493,12 @@ sub fetchInputs { sub setJobsetError { - my ($jobset, $errorMsg) = @_; + my ($jobset, $errorMsg, $errorTime) = @_; my $prevError = $jobset->errormsg; eval { $db->txn_do(sub { - $jobset->update({ errormsg => $errorMsg, errortime => time, fetcherrormsg => undef }); + $jobset->update({ errormsg => $errorMsg, errortime => $errorTime, fetcherrormsg => undef }); }); }; if (defined $errorMsg && $errorMsg ne ($prevError // "") || $ENV{'HYDRA_MAIL_TEST'}) { @@ -680,6 +680,18 @@ sub checkJobsetWrapped { my $jobsetChanged = 0; my $dbStart = clock_gettime(CLOCK_MONOTONIC); + + # Store the error messages for jobs that failed to evaluate. + my $evaluationErrorTime = time; + my $evaluationErrorMsg = ""; + foreach my $job (values %{$jobs}) { + next unless defined $job->{error}; + $evaluationErrorMsg .= + ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . + ":\n" . $job->{error} . "\n\n"; + } + setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime); + my %buildMap; $db->txn_do(sub { @@ -706,6 +718,8 @@ sub checkJobsetWrapped { { hash => $argsHash , timestamp => time , checkouttime => abs(int($checkoutStop - $checkoutStart)) + , errormsg => $evaluationErrorMsg + , errortime => $evaluationErrorTime , evaltime => abs(int($evalStop - $evalStart)) , hasnewbuilds => $jobsetChanged ? 1 : 0 , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef @@ -791,16 +805,6 @@ sub checkJobsetWrapped { Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000)); Net::Statsd::increment("hydra.evaluator.evals"); Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged; - - # Store the error messages for jobs that failed to evaluate. - my $msg = ""; - foreach my $job (values %{$jobs}) { - next unless defined $job->{error}; - $msg .= - ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") . - ":\n" . $job->{error} . "\n\n"; - } - setJobsetError($jobset, $msg); } @@ -827,9 +831,10 @@ sub checkJobset { my $failed = 0; if ($checkError) { print STDERR $checkError; + my $eventTime = time; $db->txn_do(sub { - $jobset->update({lastcheckedtime => time}); - setJobsetError($jobset, $checkError); + $jobset->update({lastcheckedtime => $eventTime}); + setJobsetError($jobset, $checkError, $eventTime); $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId)); }) if !$dryRun; $failed = 1; From 805dd6e7ee15ab3647e283ebeb64d279dc49c7ff Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 21 Jan 2021 12:13:09 -0500 Subject: [PATCH 4/6] Evaluation page: render evaluation errors --- src/root/jobset-eval.tt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index 29b8b6f6..99cc3720 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -103,6 +103,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
+ [% IF eval.errormsg %] +
+

Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].

+ 
[% HTML.escape(eval.errormsg) %]
+
+ [% END %] +
[% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
From c64c4aac4f8bb60b267128bafe4348f8b97cb799 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 21 Jan 2021 12:59:03 -0500 Subject: [PATCH 5/6] jobset page: render error labels per eval --- src/root/common.tt | 3 +++ src/root/jobset-eval.tt | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/src/root/common.tt b/src/root/common.tt index bc777ede..8d4f4f70 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -476,6 +476,9 @@ BLOCK renderEvals %] ELSE %] - [% END %] + [% IF eval.errormsg %] + Eval Errors + [% END %] [% e.nrSucceeded %] diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index 99cc3720..139ee2fe 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -88,6 +88,11 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
  • Queued jobs ([% unfinished.size %])
    • [% END %]
  • Inputs
    • + + [% IF eval.errormsg %] +
  • Evaluation errors
    • + [% END %] + [% BLOCK renderSome %] @@ -167,6 +172,12 @@ c.uri_for(c.controller('JobsetEval').action_for('view'), [% END %]
    + [% IF eval.errormsg %] +
    +

    Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].

    + 
    [% HTML.escape(eval.errormsg) %]
    +
    + [% END %] [% END %] From bd99052a6fe832db9770b5d5073a16a1b0ed70f6 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 21 Jan 2021 14:42:12 -0500 Subject: [PATCH 6/6] tests: create database with the utf-8 locale Otherwise tests may fail with wide character errors. --- tests/set-up.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/set-up.pl b/tests/set-up.pl index d7aa35cc..4fb99a49 100644 --- a/tests/set-up.pl +++ b/tests/set-up.pl @@ -1,5 +1,5 @@ use strict; -system("initdb -D postgres") == 0 or die; -system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die; -system("createdb -p 6433 hydra-test-suite") == 0 or die; +system("initdb -D postgres --locale C.UTF-8 ") == 0 or die; +system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die; +system("createdb -l C.UTF-8 -p 6433 hydra-test-suite") == 0 or die; system("hydra-init") == 0 or die;