Merge pull request #847 from grahamc/jobsetevals-evaluation-errors

JobsetEvals: record evaluation errors

.gitignore

@@ -35,3 +35,7 @@ hydra-config.h.in
 result
 tests/jobs/config.nix
 outputs
+config
+stamp-h1
+src/hydra-evaluator/hydra-evaluator
+src/hydra-queue-runner/hydra-queue-runner

src/lib/Hydra/Schema/JobsetEvals.pm

@@ -54,6 +54,16 @@ __PACKAGE__->table("jobsetevals");
   is_foreign_key: 1
   is_nullable: 0
 
+=head2 errormsg
+
+  data_type: 'text'
+  is_nullable: 1
+
+=head2 errortime
+
+  data_type: 'integer'
+  is_nullable: 1
+
 =head2 timestamp
 
   data_type: 'integer'
@@ -108,6 +118,10 @@ __PACKAGE__->add_columns(
   { data_type => "text", is_foreign_key => 1, is_nullable => 0 },
   "jobset",
   { data_type => "text", is_foreign_key => 1, is_nullable => 0 },
+  "errormsg",
+  { data_type => "text", is_nullable => 1 },
+  "errortime",
+  { data_type => "integer", is_nullable => 1 },
   "timestamp",
   { data_type => "integer", is_nullable => 0 },
   "checkouttime",
@@ -201,8 +215,8 @@ __PACKAGE__->belongs_to(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:M61ikfnjORU7jDAH8P/j7w
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg
 
 __PACKAGE__->has_many(
   "buildIds",

src/root/common.tt

@@ -476,6 +476,9 @@ BLOCK renderEvals %]
             ELSE %]
              -
             [% END %]
+            [% IF eval.errormsg %]
+              <span class="label label-warning">Eval Errors</span>
+            [% END %]
           </td>
           <td align='right' class="nowrap">
             <span class="label label-success">[% e.nrSucceeded %]</span>

src/root/jobset-eval.tt

@@ -88,6 +88,11 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
     <li><a href="#tabs-unfinished" data-toggle="tab">Queued jobs ([% unfinished.size %])</a></li>
   [% END %]
   <li><a href="#tabs-inputs" data-toggle="tab">Inputs</a></li>
+
+  [% IF eval.errormsg %]
+    <li><a href="#tabs-errors" data-toggle="tab"><span class="text-warning">Evaluation errors</span></a></li>
+  [% END %]
+
 </ul>
 
 [% BLOCK renderSome %]
@@ -103,6 +108,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
 
 <div class="tab-content">
 
+  [% IF eval.errormsg %]
+    <div id="tabs-errors" class="tab-pane">
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].</p>
+      <pre class="alert alert-error">[% HTML.escape(eval.errormsg) %]</pre>
+    </div>
+  [% END %]
+
   <div id="tabs-aborted" class="tab-pane">
     [% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
   </div>
@@ -160,6 +172,12 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
     [% END %]
   </div>
 
+  [% IF eval.errormsg %]
+    <div id="tabs-errors" class="tab-pane">
+      <p>Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].</p>
+      <pre class="alert alert-error">[% HTML.escape(eval.errormsg) %]</pre>
+    </div>
+  [% END %]
 </div>
 
 [% END %]

src/script/hydra-eval-jobset

@@ -493,12 +493,12 @@ sub fetchInputs {
 
 
 sub setJobsetError {
-    my ($jobset, $errorMsg) = @_;
+    my ($jobset, $errorMsg, $errorTime) = @_;
     my $prevError = $jobset->errormsg;
 
     eval {
         $db->txn_do(sub {
-            $jobset->update({ errormsg => $errorMsg, errortime => time, fetcherrormsg => undef });
+            $jobset->update({ errormsg => $errorMsg, errortime => $errorTime, fetcherrormsg => undef });
         });
     };
     if (defined $errorMsg && $errorMsg ne ($prevError // "") || $ENV{'HYDRA_MAIL_TEST'}) {
@@ -680,6 +680,18 @@ sub checkJobsetWrapped {
     my $jobsetChanged = 0;
     my $dbStart = clock_gettime(CLOCK_MONOTONIC);
 
+
+    # Store the error messages for jobs that failed to evaluate.
+    my $evaluationErrorTime = time;
+    my $evaluationErrorMsg = "";
+    foreach my $job (values %{$jobs}) {
+        next unless defined $job->{error};
+        $evaluationErrorMsg .=
+            ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
+            ":\n" . $job->{error} . "\n\n";
+    }
+    setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
+
     my %buildMap;
     $db->txn_do(sub {
 
@@ -706,6 +718,8 @@ sub checkJobsetWrapped {
             { hash => $argsHash
             , timestamp => time
             , checkouttime => abs(int($checkoutStop - $checkoutStart))
+            , errormsg => $evaluationErrorMsg
+            , errortime => $evaluationErrorTime
             , evaltime => abs(int($evalStop - $evalStart))
             , hasnewbuilds => $jobsetChanged ? 1 : 0
             , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
@@ -791,16 +805,6 @@ sub checkJobsetWrapped {
     Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
     Net::Statsd::increment("hydra.evaluator.evals");
     Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
-
-    # Store the error messages for jobs that failed to evaluate.
-    my $msg = "";
-    foreach my $job (values %{$jobs}) {
-        next unless defined $job->{error};
-        $msg .=
-            ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
-            ":\n" . $job->{error} . "\n\n";
-    }
-    setJobsetError($jobset, $msg);
 }
 
 
@@ -827,9 +831,10 @@ sub checkJobset {
     my $failed = 0;
     if ($checkError) {
         print STDERR $checkError;
+        my $eventTime = time;
         $db->txn_do(sub {
-            $jobset->update({lastcheckedtime => time});
+            $jobset->update({lastcheckedtime => $eventTime});
-            setJobsetError($jobset, $checkError);
+            setJobsetError($jobset, $checkError, $eventTime);
             $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
         }) if !$dryRun;
         $failed = 1;

src/sql/hydra.sql

@@ -440,6 +440,9 @@ create table JobsetEvals (
     project       text not null,
     jobset        text not null,
 
+    errorMsg      text, -- error output from the evaluator
+    errorTime     integer, -- timestamp associated with errorMsg
+
     timestamp     integer not null, -- when this entry was added
     checkoutTime  integer not null, -- how long obtaining the inputs took (in seconds)
     evalTime      integer not null, -- how long evaluation took (in seconds)

src/sql/upgrade-70.sql

@@ -0,0 +1,34 @@
+ALTER TABLE JobsetEvals
+    ADD COLUMN errorMsg text,
+    ADD COLUMN errorTime integer NULL;
+
+-- Copy the current error in jobsets to the latest field in jobsetevals
+UPDATE jobsetevals
+    SET errorMsg = j.errorMsg,
+        errorTime = j.errorTime
+  FROM (
+    SELECT
+        jobsets.errorMsg,
+        jobsets.errorTime,
+        jobsets.id AS jobset_id,
+        latesteval.id AS eval_id
+    FROM jobsets
+    LEFT JOIN
+        (
+            SELECT
+                MAX(id) AS id,
+                project,
+                jobset
+            FROM jobsetevals
+            GROUP BY project, jobset
+            ORDER BY project, jobset
+        )
+        AS latesteval
+        ON
+            jobsets.name = latesteval.jobset
+            AND jobsets.project = latesteval.project
+    WHERE latesteval.id IS NOT NULL
+    ORDER BY jobsets.id
+)
+AS j
+WHERE id = j.eval_id;

tests/set-up.pl

@@ -1,5 +1,5 @@
 use strict;
-system("initdb -D postgres") == 0 or die;
+system("initdb -D postgres --locale C.UTF-8 ") == 0 or die;
-system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die;
+system("pg_ctl -D postgres -o \"-F -p 6433  -h '' -k /tmp \" -w start") == 0 or die;
-system("createdb -p 6433 hydra-test-suite") == 0 or die;
+system("createdb -l C.UTF-8 -p 6433 hydra-test-suite") == 0 or die;
 system("hydra-init") == 0 or die;