2011-11-30 16:32:50 +00:00
|
|
|
#! /var/run/current-system/sw/bin/perl -w
|
2008-10-10 16:05:05 +00:00
|
|
|
|
|
|
|
use strict;
|
2009-03-09 13:04:46 +00:00
|
|
|
use feature 'switch';
|
2008-11-25 11:09:15 +00:00
|
|
|
use Hydra::Schema;
|
2008-11-28 14:36:04 +00:00
|
|
|
use Hydra::Helper::Nix;
|
2009-10-26 15:39:14 +00:00
|
|
|
use Hydra::Helper::AddBuilds;
|
2012-03-13 11:10:19 +00:00
|
|
|
use Hydra::Model::DB;
|
2009-11-17 13:55:22 +00:00
|
|
|
use Digest::SHA qw(sha256_hex);
|
2009-12-18 12:07:45 +00:00
|
|
|
use Email::Sender::Simple qw(sendmail);
|
|
|
|
use Email::Sender::Transport::SMTP;
|
|
|
|
use Email::Simple;
|
|
|
|
use Email::Simple::Creator;
|
|
|
|
use Sys::Hostname::Long;
|
|
|
|
use Config::General;
|
2010-01-19 14:15:31 +00:00
|
|
|
use Data::Dump qw(dump);
|
2008-10-28 10:18:03 +00:00
|
|
|
|
2009-04-22 22:59:54 +00:00
|
|
|
STDOUT->autoflush();
|
|
|
|
|
2012-03-13 11:10:19 +00:00
|
|
|
my $db = Hydra::Model::DB->new();
|
2011-03-07 15:06:32 +00:00
|
|
|
my %config = new Config::General(getHydraConf)->getall;
|
2008-11-05 04:52:52 +00:00
|
|
|
|
2010-03-05 15:41:10 +00:00
|
|
|
|
2009-03-09 13:04:46 +00:00
|
|
|
sub fetchInputs {
|
2009-03-09 13:58:43 +00:00
|
|
|
my ($project, $jobset, $inputInfo) = @_;
|
2009-03-09 13:04:46 +00:00
|
|
|
foreach my $input ($jobset->jobsetinputs->all) {
|
|
|
|
foreach my $alt ($input->jobsetinputalts->all) {
|
2011-12-05 16:13:20 +00:00
|
|
|
my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
|
|
|
|
foreach my $info_el (@info) {
|
|
|
|
push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
|
|
|
|
}
|
2009-03-09 13:04:46 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-11-25 13:27:57 +00:00
|
|
|
sub setJobsetError {
|
|
|
|
my ($jobset, $errorMsg) = @_;
|
|
|
|
eval {
|
2009-04-22 22:43:04 +00:00
|
|
|
txn_do($db, sub {
|
2009-03-09 16:22:41 +00:00
|
|
|
$jobset->update({errormsg => $errorMsg, errortime => time});
|
2008-11-25 13:27:57 +00:00
|
|
|
});
|
|
|
|
};
|
2009-12-18 12:07:45 +00:00
|
|
|
sendJobsetErrorNotification($jobset, $errorMsg);
|
2008-11-25 13:27:57 +00:00
|
|
|
}
|
|
|
|
|
2012-04-03 10:10:45 +00:00
|
|
|
|
2009-12-18 12:07:45 +00:00
|
|
|
sub sendJobsetErrorNotification() {
|
|
|
|
my ($jobset, $errorMsg) = @_;
|
|
|
|
|
|
|
|
return if $jobset->project->owner->emailonerror == 0;
|
2010-01-12 08:39:30 +00:00
|
|
|
return if $errorMsg eq "";
|
2009-12-18 12:07:45 +00:00
|
|
|
|
2010-04-26 14:36:56 +00:00
|
|
|
my $url = hostname_long;
|
2009-12-18 12:07:45 +00:00
|
|
|
my $projectName = $jobset->project->name;
|
|
|
|
my $jobsetName = $jobset->name;
|
|
|
|
|
|
|
|
my $sender = $config{'notification_sender'} ||
|
2010-04-26 14:36:56 +00:00
|
|
|
(($ENV{'USER'} || "hydra") . "@" . $url);
|
2009-12-18 12:07:45 +00:00
|
|
|
|
|
|
|
my $body = "Hi,\n"
|
|
|
|
. "\n"
|
|
|
|
. "This is to let you know that Hydra jobset evalation of $projectName:$jobsetName "
|
|
|
|
. "resulted in the following error:\n"
|
|
|
|
. "\n"
|
|
|
|
. "$errorMsg"
|
|
|
|
. "\n"
|
|
|
|
. "Regards,\n\nThe Hydra build daemon.\n";
|
|
|
|
|
|
|
|
my $email = Email::Simple->create(
|
|
|
|
header => [
|
|
|
|
To => $jobset->project->owner->emailaddress,
|
|
|
|
From => "Hydra Build Daemon <$sender>",
|
|
|
|
Subject => "Hydra $projectName:$jobsetName evaluation error",
|
2010-04-26 14:36:56 +00:00
|
|
|
|
|
|
|
'X-Hydra-Instance' => $url,
|
|
|
|
'X-Hydra-Project' => $projectName,
|
|
|
|
'X-Hydra-Jobset' => $jobsetName
|
|
|
|
],
|
2010-01-07 13:53:05 +00:00
|
|
|
body => ""
|
2009-12-18 12:07:45 +00:00
|
|
|
);
|
2010-01-07 13:53:05 +00:00
|
|
|
$email->body_set($body);
|
2009-12-18 12:07:45 +00:00
|
|
|
|
2012-03-07 17:48:10 +00:00
|
|
|
print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'};
|
2009-12-18 12:07:45 +00:00
|
|
|
|
|
|
|
sendmail($email);
|
|
|
|
}
|
2008-11-25 13:27:57 +00:00
|
|
|
|
2012-03-07 17:48:10 +00:00
|
|
|
|
2009-04-23 15:40:36 +00:00
|
|
|
sub permute {
|
|
|
|
my @list = @_;
|
|
|
|
for (my $n = scalar @list - 1; $n > 0; $n--) {
|
|
|
|
my $k = int(rand($n + 1)); # 0 <= $k <= $n
|
|
|
|
@list[$n, $k] = @list[$k, $n];
|
|
|
|
}
|
|
|
|
return @list;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2009-03-20 14:50:09 +00:00
|
|
|
sub checkJobset {
|
2008-11-07 14:51:44 +00:00
|
|
|
my ($project, $jobset) = @_;
|
|
|
|
my $inputInfo = {};
|
2008-11-26 13:39:15 +00:00
|
|
|
|
2009-03-09 13:04:46 +00:00
|
|
|
# Fetch all values for all inputs.
|
2010-03-05 15:41:10 +00:00
|
|
|
my $checkoutStart = time;
|
2009-03-09 13:58:43 +00:00
|
|
|
fetchInputs($project, $jobset, $inputInfo);
|
2010-03-05 15:41:10 +00:00
|
|
|
my $checkoutStop = time;
|
2008-11-07 14:51:44 +00:00
|
|
|
|
2011-11-30 17:13:35 +00:00
|
|
|
# Hash the arguments to hydra-eval-jobs and check the
|
2012-03-07 17:48:10 +00:00
|
|
|
# JobsetInputHashes to see if the previous evaluation had the same
|
2009-11-17 13:55:22 +00:00
|
|
|
# inputs. If so, bail out.
|
|
|
|
my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo));
|
|
|
|
my $argsHash = sha256_hex("@args");
|
2012-03-12 20:13:28 +00:00
|
|
|
my $prevEval = getPrevJobsetEval($db, $jobset, 0);
|
|
|
|
if (defined $prevEval && $prevEval->hash eq $argsHash) {
|
2012-03-07 17:48:10 +00:00
|
|
|
print STDERR " jobset is unchanged, skipping\n";
|
2009-11-17 13:55:22 +00:00
|
|
|
txn_do($db, sub {
|
|
|
|
$jobset->update({lastcheckedtime => time});
|
|
|
|
});
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2009-03-09 13:04:46 +00:00
|
|
|
# Evaluate the job expression.
|
2010-03-05 15:41:10 +00:00
|
|
|
my $evalStart = time;
|
2009-10-26 17:01:23 +00:00
|
|
|
my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath);
|
2010-03-05 15:41:10 +00:00
|
|
|
my $evalStop = time;
|
2009-03-09 15:16:11 +00:00
|
|
|
|
2012-04-02 15:56:29 +00:00
|
|
|
my $jobOutPathMap = {};
|
|
|
|
|
2009-04-22 22:43:04 +00:00
|
|
|
txn_do($db, sub {
|
2009-10-02 16:06:28 +00:00
|
|
|
|
2012-03-12 19:28:44 +00:00
|
|
|
my $prevEval = getPrevJobsetEval($db, $jobset, 1);
|
|
|
|
|
2012-03-07 17:48:10 +00:00
|
|
|
# Clear the "current" flag on all builds. Since we're in a
|
|
|
|
# transaction this will only become visible after the new
|
|
|
|
# current builds have been added.
|
|
|
|
$jobset->builds->search({iscurrent => 1})->update({iscurrent => 0});
|
|
|
|
|
2010-03-05 15:41:10 +00:00
|
|
|
# Schedule each successfully evaluated job.
|
2012-03-07 17:48:10 +00:00
|
|
|
my %buildIds;
|
2010-03-05 15:41:10 +00:00
|
|
|
foreach my $job (permute @{$jobs->{job}}) {
|
|
|
|
next if $job->{jobName} eq "";
|
2012-03-07 17:48:10 +00:00
|
|
|
print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
|
2012-04-02 15:56:29 +00:00
|
|
|
checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap);
|
2010-03-05 15:41:10 +00:00
|
|
|
}
|
|
|
|
|
2009-10-08 11:19:39 +00:00
|
|
|
# Update the last checked times and error messages for each
|
|
|
|
# job.
|
2009-10-02 16:06:28 +00:00
|
|
|
my %failedJobNames;
|
|
|
|
push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}};
|
|
|
|
|
2009-03-20 14:50:09 +00:00
|
|
|
$jobset->update({lastcheckedtime => time});
|
|
|
|
|
2012-03-07 17:48:10 +00:00
|
|
|
$_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef })
|
|
|
|
foreach $jobset->jobs->all;
|
2009-11-17 13:55:22 +00:00
|
|
|
|
2010-03-05 15:41:10 +00:00
|
|
|
my $hasNewBuilds = 0;
|
2012-03-07 17:48:10 +00:00
|
|
|
while (my ($id, $new) = each %buildIds) {
|
2010-03-05 15:41:10 +00:00
|
|
|
$hasNewBuilds = 1 if $new;
|
|
|
|
}
|
|
|
|
|
|
|
|
my $ev = $jobset->jobsetevals->create(
|
|
|
|
{ hash => $argsHash
|
|
|
|
, timestamp => time
|
|
|
|
, checkouttime => abs($checkoutStop - $checkoutStart)
|
|
|
|
, evaltime => abs($evalStop - $evalStart)
|
|
|
|
, hasnewbuilds => $hasNewBuilds
|
2012-04-15 22:57:10 +00:00
|
|
|
, nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef
|
2010-03-05 15:41:10 +00:00
|
|
|
});
|
|
|
|
|
|
|
|
if ($hasNewBuilds) {
|
2012-03-07 17:48:10 +00:00
|
|
|
while (my ($id, $new) = each %buildIds) {
|
2010-03-05 15:41:10 +00:00
|
|
|
$ev->jobsetevalmembers->create({ build => $id, isnew => $new });
|
|
|
|
}
|
2012-04-15 18:36:36 +00:00
|
|
|
|
|
|
|
foreach my $name (keys %{$inputInfo}) {
|
|
|
|
for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) {
|
|
|
|
my $input = $inputInfo->{$name}->[$n];
|
|
|
|
$ev->jobsetevalinputs->create(
|
|
|
|
{ name => $name
|
|
|
|
, altnr => $n
|
|
|
|
, type => $input->{type}
|
|
|
|
, uri => $input->{uri}
|
|
|
|
, revision => $input->{revision}
|
|
|
|
, value => $input->{value}
|
|
|
|
, dependency => $input->{id}
|
|
|
|
, path => $input->{storePath} || "" # !!! temporary hack
|
|
|
|
, sha256hash => $input->{sha256hash}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-03-12 19:28:44 +00:00
|
|
|
print STDERR " created new eval ", $ev->id, "\n";
|
2012-03-13 12:09:10 +00:00
|
|
|
$ev->builds->update({iscurrent => 1});
|
2012-03-12 19:28:44 +00:00
|
|
|
} else {
|
|
|
|
print STDERR " created cached eval ", $ev->id, "\n";
|
2012-04-17 10:32:44 +00:00
|
|
|
$prevEval->builds->update({iscurrent => 1}) if defined $prevEval;
|
2010-03-05 15:41:10 +00:00
|
|
|
}
|
2009-10-02 16:06:28 +00:00
|
|
|
});
|
|
|
|
|
2012-03-07 17:48:10 +00:00
|
|
|
# Store the error messages for jobs that failed to evaluate.
|
2009-03-09 15:16:11 +00:00
|
|
|
my $msg = "";
|
|
|
|
foreach my $error (@{$jobs->{error}}) {
|
2009-03-20 17:06:50 +00:00
|
|
|
my $bindings = "";
|
|
|
|
foreach my $arg (@{$error->{arg}}) {
|
|
|
|
my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
|
|
|
|
$bindings .= ", " if $bindings ne "";
|
|
|
|
$bindings .= $arg->{name} . " = ";
|
|
|
|
given ($input->{type}) {
|
|
|
|
when ("string") { $bindings .= "\"" . $input->{value} . "\""; }
|
|
|
|
when ("boolean") { $bindings .= $input->{value}; }
|
|
|
|
default { $bindings .= "..."; }
|
|
|
|
}
|
|
|
|
}
|
|
|
|
$msg .= "at `" . $error->{location} . "' [$bindings]:\n" . $error->{msg} . "\n\n";
|
2009-03-09 15:16:11 +00:00
|
|
|
}
|
2010-01-12 08:39:30 +00:00
|
|
|
setJobsetError($jobset, $msg);
|
2008-11-04 18:23:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2009-03-20 14:50:09 +00:00
|
|
|
sub checkJobsetWrapped {
|
|
|
|
my ($project, $jobset) = @_;
|
|
|
|
|
2012-03-07 17:48:10 +00:00
|
|
|
print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n";
|
2009-03-20 14:50:09 +00:00
|
|
|
|
|
|
|
eval {
|
|
|
|
checkJobset($project, $jobset);
|
|
|
|
};
|
|
|
|
|
|
|
|
if ($@) {
|
|
|
|
my $msg = $@;
|
2012-03-07 17:48:10 +00:00
|
|
|
print STDERR "error evaluating jobset ", $jobset->name, ": $msg";
|
2009-04-22 22:43:04 +00:00
|
|
|
txn_do($db, sub {
|
2009-03-20 14:50:09 +00:00
|
|
|
$jobset->update({lastcheckedtime => time});
|
2010-01-12 08:39:30 +00:00
|
|
|
setJobsetError($jobset, $msg);
|
2009-03-20 14:50:09 +00:00
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
2008-11-04 18:23:28 +00:00
|
|
|
|
2009-03-20 14:50:09 +00:00
|
|
|
|
2009-10-26 15:55:19 +00:00
|
|
|
sub checkProjects {
|
2008-11-18 12:48:58 +00:00
|
|
|
foreach my $project ($db->resultset('Projects')->search({enabled => 1})) {
|
2012-03-07 17:48:10 +00:00
|
|
|
print STDERR "considering project ", $project->name, "\n";
|
2009-10-08 11:39:16 +00:00
|
|
|
checkJobsetWrapped($project, $_)
|
|
|
|
foreach $project->jobsets->search({enabled => 1});
|
2008-11-04 18:23:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2009-03-05 12:32:14 +00:00
|
|
|
# For testing: evaluate a single jobset, then exit.
|
|
|
|
if (scalar @ARGV == 2) {
|
|
|
|
my $projectName = $ARGV[0];
|
|
|
|
my $jobsetName = $ARGV[1];
|
|
|
|
my $jobset = $db->resultset('Jobsets')->find($projectName, $jobsetName) or die;
|
2009-03-20 14:50:09 +00:00
|
|
|
checkJobsetWrapped($jobset->project, $jobset);
|
2009-03-05 12:32:14 +00:00
|
|
|
exit 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-11-11 10:27:36 +00:00
|
|
|
while (1) {
|
2009-03-23 01:13:37 +00:00
|
|
|
eval {
|
2009-10-26 15:55:19 +00:00
|
|
|
checkProjects;
|
2009-03-23 01:13:37 +00:00
|
|
|
};
|
2012-03-07 17:48:10 +00:00
|
|
|
if ($@) { print STDERR "$@"; }
|
|
|
|
print STDERR "sleeping...\n";
|
2008-11-29 01:20:13 +00:00
|
|
|
sleep 30;
|
2008-11-11 10:27:36 +00:00
|
|
|
}
|