hydra/src/script/hydra-evaluator

268 lines
8.3 KiB
Perl
Executable file

#! /var/run/current-system/sw/bin/perl -w
use strict;
use feature 'switch';
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::AddBuilds;
use Hydra::Model::DB;
use Digest::SHA qw(sha256_hex);
use Email::Sender::Simple qw(sendmail);
use Email::Sender::Transport::SMTP;
use Email::Simple;
use Email::Simple::Creator;
use Sys::Hostname::Long;
use Config::General;
use Data::Dump qw(dump);
STDOUT->autoflush();
my $db = Hydra::Model::DB->new();
my %config = new Config::General(getHydraConf)->getall;
sub fetchInputs {
my ($project, $jobset, $inputInfo) = @_;
foreach my $input ($jobset->jobsetinputs->all) {
foreach my $alt ($input->jobsetinputalts->all) {
my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
foreach my $info_el (@info) {
push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
}
}
}
}
sub setJobsetError {
my ($jobset, $errorMsg) = @_;
eval {
txn_do($db, sub {
$jobset->update({errormsg => $errorMsg, errortime => time});
});
};
sendJobsetErrorNotification($jobset, $errorMsg);
}
sub sendJobsetErrorNotification() {
my ($jobset, $errorMsg) = @_;
return if $jobset->project->owner->emailonerror == 0;
return if $errorMsg eq "";
my $url = hostname_long;
my $projectName = $jobset->project->name;
my $jobsetName = $jobset->name;
my $sender = $config{'notification_sender'} ||
(($ENV{'USER'} || "hydra") . "@" . $url);
my $body = "Hi,\n"
. "\n"
. "This is to let you know that Hydra jobset evalation of $projectName:$jobsetName "
. "resulted in the following error:\n"
. "\n"
. "$errorMsg"
. "\n"
. "Regards,\n\nThe Hydra build daemon.\n";
my $email = Email::Simple->create(
header => [
To => $jobset->project->owner->emailaddress,
From => "Hydra Build Daemon <$sender>",
Subject => "Hydra $projectName:$jobsetName evaluation error",
'X-Hydra-Instance' => $url,
'X-Hydra-Project' => $projectName,
'X-Hydra-Jobset' => $jobsetName
],
body => ""
);
$email->body_set($body);
print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'};
sendmail($email);
}
sub permute {
my @list = @_;
for (my $n = scalar @list - 1; $n > 0; $n--) {
my $k = int(rand($n + 1)); # 0 <= $k <= $n
@list[$n, $k] = @list[$k, $n];
}
return @list;
}
sub checkJobset {
my ($project, $jobset) = @_;
my $inputInfo = {};
# Fetch all values for all inputs.
my $checkoutStart = time;
fetchInputs($project, $jobset, $inputInfo);
my $checkoutStop = time;
# Hash the arguments to hydra-eval-jobs and check the
# JobsetInputHashes to see if the previous evaluation had the same
# inputs. If so, bail out.
my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo));
my $argsHash = sha256_hex("@args");
my $prevEval = getPrevJobsetEval($db, $jobset, 0);
if (defined $prevEval && $prevEval->hash eq $argsHash) {
print STDERR " jobset is unchanged, skipping\n";
txn_do($db, sub {
$jobset->update({lastcheckedtime => time});
});
return;
}
# Evaluate the job expression.
my $evalStart = time;
my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath);
my $evalStop = time;
my $jobOutPathMap = {};
txn_do($db, sub {
my $prevEval = getPrevJobsetEval($db, $jobset, 1);
# Clear the "current" flag on all builds. Since we're in a
# transaction this will only become visible after the new
# current builds have been added.
$jobset->builds->search({iscurrent => 1})->update({iscurrent => 0});
# Schedule each successfully evaluated job.
my %buildIds;
foreach my $job (permute @{$jobs->{job}}) {
next if $job->{jobName} eq "";
print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap);
}
# Update the last checked times and error messages for each
# job.
my %failedJobNames;
push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}};
$jobset->update({lastcheckedtime => time});
$_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef })
foreach $jobset->jobs->all;
my $hasNewBuilds = 0;
while (my ($id, $new) = each %buildIds) {
$hasNewBuilds = 1 if $new;
}
my $ev = $jobset->jobsetevals->create(
{ hash => $argsHash
, timestamp => time
, checkouttime => abs($checkoutStop - $checkoutStart)
, evaltime => abs($evalStop - $evalStart)
, hasnewbuilds => $hasNewBuilds
, nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef
});
if ($hasNewBuilds) {
while (my ($id, $new) = each %buildIds) {
$ev->jobsetevalmembers->create({ build => $id, isnew => $new });
}
foreach my $name (keys %{$inputInfo}) {
for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) {
my $input = $inputInfo->{$name}->[$n];
$ev->jobsetevalinputs->create(
{ name => $name
, altnr => $n
, type => $input->{type}
, uri => $input->{uri}
, revision => $input->{revision}
, value => $input->{value}
, dependency => $input->{id}
, path => $input->{storePath} || "" # !!! temporary hack
, sha256hash => $input->{sha256hash}
});
}
}
print STDERR " created new eval ", $ev->id, "\n";
$ev->builds->update({iscurrent => 1});
} else {
print STDERR " created cached eval ", $ev->id, "\n";
$prevEval->builds->update({iscurrent => 1}) if defined $prevEval;
}
});
# Store the error messages for jobs that failed to evaluate.
my $msg = "";
foreach my $error (@{$jobs->{error}}) {
my $bindings = "";
foreach my $arg (@{$error->{arg}}) {
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
$bindings .= ", " if $bindings ne "";
$bindings .= $arg->{name} . " = ";
given ($input->{type}) {
when ("string") { $bindings .= "\"" . $input->{value} . "\""; }
when ("boolean") { $bindings .= $input->{value}; }
default { $bindings .= "..."; }
}
}
$msg .= "at `" . $error->{location} . "' [$bindings]:\n" . $error->{msg} . "\n\n";
}
setJobsetError($jobset, $msg);
}
sub checkJobsetWrapped {
my ($project, $jobset) = @_;
print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n";
eval {
checkJobset($project, $jobset);
};
if ($@) {
my $msg = $@;
print STDERR "error evaluating jobset ", $jobset->name, ": $msg";
txn_do($db, sub {
$jobset->update({lastcheckedtime => time});
setJobsetError($jobset, $msg);
});
}
}
sub checkProjects {
foreach my $project ($db->resultset('Projects')->search({enabled => 1})) {
print STDERR "considering project ", $project->name, "\n";
checkJobsetWrapped($project, $_)
foreach $project->jobsets->search({enabled => 1});
}
}
# For testing: evaluate a single jobset, then exit.
if (scalar @ARGV == 2) {
my $projectName = $ARGV[0];
my $jobsetName = $ARGV[1];
my $jobset = $db->resultset('Jobsets')->find($projectName, $jobsetName) or die;
checkJobsetWrapped($jobset->project, $jobset);
exit 0;
}
while (1) {
eval {
checkProjects;
};
if ($@) { print STDERR "$@"; }
print STDERR "sleeping...\n";
sleep 30;
}