diff --git a/.gitignore b/.gitignore
index a073f4f1..5a0ab4a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,7 @@ hydra-config.h.in
result
tests/jobs/config.nix
outputs
+config
+stamp-h1
+src/hydra-evaluator/hydra-evaluator
+src/hydra-queue-runner/hydra-queue-runner
diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm
index 44beb7fd..b8f6c2a2 100644
--- a/src/lib/Hydra/Schema/JobsetEvals.pm
+++ b/src/lib/Hydra/Schema/JobsetEvals.pm
@@ -54,6 +54,16 @@ __PACKAGE__->table("jobsetevals");
is_foreign_key: 1
is_nullable: 0
+=head2 errormsg
+
+ data_type: 'text'
+ is_nullable: 1
+
+=head2 errortime
+
+ data_type: 'integer'
+ is_nullable: 1
+
=head2 timestamp
data_type: 'integer'
@@ -108,6 +118,10 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_foreign_key => 1, is_nullable => 0 },
"jobset",
{ data_type => "text", is_foreign_key => 1, is_nullable => 0 },
+ "errormsg",
+ { data_type => "text", is_nullable => 1 },
+ "errortime",
+ { data_type => "integer", is_nullable => 1 },
"timestamp",
{ data_type => "integer", is_nullable => 0 },
"checkouttime",
@@ -201,8 +215,8 @@ __PACKAGE__->belongs_to(
);
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-05-27 17:40:41
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:M61ikfnjORU7jDAH8P/j7w
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2021-01-21 11:13:38
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zDBtAFc4HiFUcL/TpkuCcg
__PACKAGE__->has_many(
"buildIds",
diff --git a/src/root/common.tt b/src/root/common.tt
index bc777ede..8d4f4f70 100644
--- a/src/root/common.tt
+++ b/src/root/common.tt
@@ -476,6 +476,9 @@ BLOCK renderEvals %]
ELSE %]
-
[% END %]
+ [% IF eval.errormsg %]
+ Eval Errors
+ [% END %]
[% e.nrSucceeded %]
diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt
index 29b8b6f6..139ee2fe 100644
--- a/src/root/jobset-eval.tt
+++ b/src/root/jobset-eval.tt
@@ -88,6 +88,11 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
Queued jobs ([% unfinished.size %])
[% END %]
Inputs
+
+ [% IF eval.errormsg %]
+ Evaluation errors
+ [% END %]
+
[% BLOCK renderSome %]
@@ -103,6 +108,13 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
+ [% IF eval.errormsg %]
+
+ Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].
+ [% HTML.escape(eval.errormsg) %]
+
+ [% END %]
+
[% INCLUDE renderSome builds=aborted tabname="#tabs-aborted" %]
@@ -160,6 +172,12 @@ c.uri_for(c.controller('JobsetEval').action_for('view'),
[% END %]
+ [% IF eval.errormsg %]
+
+ Errors occurred at [% INCLUDE renderDateTime timestamp=(eval.errortime || eval.timestamp) %].
+ [% HTML.escape(eval.errormsg) %]
+
+ [% END %]
[% END %]
diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset
index c6c0fc64..78938758 100755
--- a/src/script/hydra-eval-jobset
+++ b/src/script/hydra-eval-jobset
@@ -493,12 +493,12 @@ sub fetchInputs {
sub setJobsetError {
- my ($jobset, $errorMsg) = @_;
+ my ($jobset, $errorMsg, $errorTime) = @_;
my $prevError = $jobset->errormsg;
eval {
$db->txn_do(sub {
- $jobset->update({ errormsg => $errorMsg, errortime => time, fetcherrormsg => undef });
+ $jobset->update({ errormsg => $errorMsg, errortime => $errorTime, fetcherrormsg => undef });
});
};
if (defined $errorMsg && $errorMsg ne ($prevError // "") || $ENV{'HYDRA_MAIL_TEST'}) {
@@ -680,6 +680,18 @@ sub checkJobsetWrapped {
my $jobsetChanged = 0;
my $dbStart = clock_gettime(CLOCK_MONOTONIC);
+
+ # Store the error messages for jobs that failed to evaluate.
+ my $evaluationErrorTime = time;
+ my $evaluationErrorMsg = "";
+ foreach my $job (values %{$jobs}) {
+ next unless defined $job->{error};
+ $evaluationErrorMsg .=
+ ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
+ ":\n" . $job->{error} . "\n\n";
+ }
+ setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
+
my %buildMap;
$db->txn_do(sub {
@@ -706,6 +718,8 @@ sub checkJobsetWrapped {
{ hash => $argsHash
, timestamp => time
, checkouttime => abs(int($checkoutStop - $checkoutStart))
+ , errormsg => $evaluationErrorMsg
+ , errortime => $evaluationErrorTime
, evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
@@ -791,16 +805,6 @@ sub checkJobsetWrapped {
Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
Net::Statsd::increment("hydra.evaluator.evals");
Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
-
- # Store the error messages for jobs that failed to evaluate.
- my $msg = "";
- foreach my $job (values %{$jobs}) {
- next unless defined $job->{error};
- $msg .=
- ($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
- ":\n" . $job->{error} . "\n\n";
- }
- setJobsetError($jobset, $msg);
}
@@ -827,9 +831,10 @@ sub checkJobset {
my $failed = 0;
if ($checkError) {
print STDERR $checkError;
+ my $eventTime = time;
$db->txn_do(sub {
- $jobset->update({lastcheckedtime => time});
- setJobsetError($jobset, $checkError);
+ $jobset->update({lastcheckedtime => $eventTime});
+ setJobsetError($jobset, $checkError, $eventTime);
$db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
}) if !$dryRun;
$failed = 1;
diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql
index 7f12ccfe..d8e2805c 100644
--- a/src/sql/hydra.sql
+++ b/src/sql/hydra.sql
@@ -440,6 +440,9 @@ create table JobsetEvals (
project text not null,
jobset text not null,
+ errorMsg text, -- error output from the evaluator
+ errorTime integer, -- timestamp associated with errorMsg
+
timestamp integer not null, -- when this entry was added
checkoutTime integer not null, -- how long obtaining the inputs took (in seconds)
evalTime integer not null, -- how long evaluation took (in seconds)
diff --git a/src/sql/upgrade-70.sql b/src/sql/upgrade-70.sql
new file mode 100644
index 00000000..f0d75e38
--- /dev/null
+++ b/src/sql/upgrade-70.sql
@@ -0,0 +1,34 @@
+ALTER TABLE JobsetEvals
+ ADD COLUMN errorMsg text,
+ ADD COLUMN errorTime integer NULL;
+
+-- Copy the current error in jobsets to the latest field in jobsetevals
+UPDATE jobsetevals
+ SET errorMsg = j.errorMsg,
+ errorTime = j.errorTime
+ FROM (
+ SELECT
+ jobsets.errorMsg,
+ jobsets.errorTime,
+ jobsets.id AS jobset_id,
+ latesteval.id AS eval_id
+ FROM jobsets
+ LEFT JOIN
+ (
+ SELECT
+ MAX(id) AS id,
+ project,
+ jobset
+ FROM jobsetevals
+ GROUP BY project, jobset
+ ORDER BY project, jobset
+ )
+ AS latesteval
+ ON
+ jobsets.name = latesteval.jobset
+ AND jobsets.project = latesteval.project
+ WHERE latesteval.id IS NOT NULL
+ ORDER BY jobsets.id
+)
+AS j
+WHERE id = j.eval_id;
diff --git a/tests/set-up.pl b/tests/set-up.pl
index d7aa35cc..4fb99a49 100644
--- a/tests/set-up.pl
+++ b/tests/set-up.pl
@@ -1,5 +1,5 @@
use strict;
-system("initdb -D postgres") == 0 or die;
-system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die;
-system("createdb -p 6433 hydra-test-suite") == 0 or die;
+system("initdb -D postgres --locale C.UTF-8 ") == 0 or die;
+system("pg_ctl -D postgres -o \"-F -p 6433 -h '' -k /tmp \" -w start") == 0 or die;
+system("createdb -l C.UTF-8 -p 6433 hydra-test-suite") == 0 or die;
system("hydra-init") == 0 or die;
|