ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

hydra-eval-jobs: write evaluation errorMsg to the jobseteval table

Browse source
This commit is contained in:
Graham Christensen 2021-01-21 11:14:37 -05:00
parent fb6b10a86c
commit 086eed5147
No known key found for this signature in database
GPG key ID: FE918C3A98C1030F
1 changed files with 19 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/script/hydra-eval-jobset
View file

@ -493,12 +493,12 @@ sub fetchInputs {
sub setJobsetError { sub setJobsetError {
my ($jobset, $errorMsg) = @_; my ($jobset, $errorMsg, $errorTime) = @_;
my $prevError = $jobset->errormsg; my $prevError = $jobset->errormsg;
eval { eval {
$db->txn_do(sub { $db->txn_do(sub {
$jobset->update({ errormsg => $errorMsg, errortime => time, fetcherrormsg => undef }); $jobset->update({ errormsg => $errorMsg, errortime => $errorTime, fetcherrormsg => undef });
}); });
}; };
if (defined $errorMsg && $errorMsg ne ($prevError // "") || $ENV{'HYDRA_MAIL_TEST'}) { if (defined $errorMsg && $errorMsg ne ($prevError // "") || $ENV{'HYDRA_MAIL_TEST'}) {
@ -680,6 +680,18 @@ sub checkJobsetWrapped {
my $jobsetChanged = 0; my $jobsetChanged = 0;
my $dbStart = clock_gettime(CLOCK_MONOTONIC); my $dbStart = clock_gettime(CLOCK_MONOTONIC);
# Store the error messages for jobs that failed to evaluate.
my $evaluationErrorTime = time;
my $evaluationErrorMsg = "";
foreach my $job (values %{$jobs}) {
next unless defined $job->{error};
$evaluationErrorMsg .=
($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
":\n" . $job->{error} . "\n\n";
}
setJobsetError($jobset, $evaluationErrorMsg, $evaluationErrorTime);
my %buildMap; my %buildMap;
$db->txn_do(sub { $db->txn_do(sub {
@ -706,6 +718,8 @@ sub checkJobsetWrapped {
{ hash => $argsHash { hash => $argsHash
, timestamp => time , timestamp => time
, checkouttime => abs(int($checkoutStop - $checkoutStart)) , checkouttime => abs(int($checkoutStop - $checkoutStart))
, errormsg => $evaluationErrorMsg
, errortime => $evaluationErrorTime
, evaltime => abs(int($evalStop - $evalStart)) , evaltime => abs(int($evalStop - $evalStart))
, hasnewbuilds => $jobsetChanged ? 1 : 0 , hasnewbuilds => $jobsetChanged ? 1 : 0
, nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef
@ -791,16 +805,6 @@ sub checkJobsetWrapped {
Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000)); Net::Statsd::timing("hydra.evaluator.db_time", int(($dbStop - $dbStart) * 1000));
Net::Statsd::increment("hydra.evaluator.evals"); Net::Statsd::increment("hydra.evaluator.evals");
Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged; Net::Statsd::increment("hydra.evaluator.cached_evals") unless $jobsetChanged;
# Store the error messages for jobs that failed to evaluate.
my $msg = "";
foreach my $job (values %{$jobs}) {
next unless defined $job->{error};
$msg .=
($job->{jobName} ne "" ? "in job ‘$job->{jobName}’" : "at top-level") .
":\n" . $job->{error} . "\n\n";
}
setJobsetError($jobset, $msg);
} }
@ -827,9 +831,10 @@ sub checkJobset {
my $failed = 0; my $failed = 0;
if ($checkError) { if ($checkError) {
print STDERR $checkError; print STDERR $checkError;
my $eventTime = time;
$db->txn_do(sub { $db->txn_do(sub {
$jobset->update({lastcheckedtime => time}); $jobset->update({lastcheckedtime => $eventTime});
setJobsetError($jobset, $checkError); setJobsetError($jobset, $checkError, $eventTime);
$db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId)); $db->storage->dbh->do("notify eval_failed, ?", undef, join('\t', $tmpId));
}) if !$dryRun; }) if !$dryRun;
$failed = 1; $failed = 1;