#! /var/run/current-system/sw/bin/perl -w use strict; use feature 'switch'; use Hydra::Schema; use Hydra::Helper::Nix; use Hydra::Helper::AddBuilds; use Digest::SHA qw(sha256_hex); use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::SMTP; use Email::Simple; use Email::Simple::Creator; use Sys::Hostname::Long; use Config::General; use Data::Dump qw(dump); STDOUT->autoflush(); my $db = openHydraDB; my %config = new Config::General(getHydraConf)->getall; sub fetchInputs { my ($project, $jobset, $inputInfo) = @_; foreach my $input ($jobset->jobsetinputs->all) { foreach my $alt ($input->jobsetinputalts->all) { my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value); foreach my $info_el (@info) { push @{$$inputInfo{$input->name}}, $info_el if defined $info_el; } } } } sub setJobsetError { my ($jobset, $errorMsg) = @_; eval { txn_do($db, sub { $jobset->update({errormsg => $errorMsg, errortime => time}); }); }; sendJobsetErrorNotification($jobset, $errorMsg); } sub sendJobsetErrorNotification() { my ($jobset, $errorMsg) = @_; return if $jobset->project->owner->emailonerror == 0; return if $errorMsg eq ""; my $url = hostname_long; my $projectName = $jobset->project->name; my $jobsetName = $jobset->name; my $sender = $config{'notification_sender'} || (($ENV{'USER'} || "hydra") . "@" . $url); my $body = "Hi,\n" . "\n" . "This is to let you know that Hydra jobset evalation of $projectName:$jobsetName " . "resulted in the following error:\n" . "\n" . "$errorMsg" . "\n" . "Regards,\n\nThe Hydra build daemon.\n"; my $email = Email::Simple->create( header => [ To => $jobset->project->owner->emailaddress, From => "Hydra Build Daemon <$sender>", Subject => "Hydra $projectName:$jobsetName evaluation error", 'X-Hydra-Instance' => $url, 'X-Hydra-Project' => $projectName, 'X-Hydra-Jobset' => $jobsetName ], body => "" ); $email->body_set($body); print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'}; sendmail($email); } sub permute { my @list = @_; for (my $n = scalar @list - 1; $n > 0; $n--) { my $k = int(rand($n + 1)); # 0 <= $k <= $n @list[$n, $k] = @list[$k, $n]; } return @list; } sub checkJobset { my ($project, $jobset) = @_; my $inputInfo = {}; # Fetch all values for all inputs. my $checkoutStart = time; fetchInputs($project, $jobset, $inputInfo); my $checkoutStop = time; # Hash the arguments to hydra-eval-jobs and check the # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo)); my $argsHash = sha256_hex("@args"); my $prevEval = getPrevJobsetEval($db, $jobset); if ($prevEval->hash eq $argsHash) { print STDERR " jobset is unchanged, skipping\n"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); }); return; } # Evaluate the job expression. my $evalStart = time; my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath); my $evalStop = time; txn_do($db, sub { # Clear the "current" flag on all builds. Since we're in a # transaction this will only become visible after the new # current builds have been added. $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); # Schedule each successfully evaluated job. my %buildIds; foreach my $job (permute @{$jobs->{job}}) { next if $job->{jobName} eq ""; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval); } # Update the last checked times and error messages for each # job. my %failedJobNames; push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}}; $jobset->update({lastcheckedtime => time}); $_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef }) foreach $jobset->jobs->all; my $hasNewBuilds = 0; while (my ($id, $new) = each %buildIds) { $hasNewBuilds = 1 if $new; } my $ev = $jobset->jobsetevals->create( { hash => $argsHash , timestamp => time , checkouttime => abs($checkoutStop - $checkoutStart) , evaltime => abs($evalStop - $evalStart) , hasnewbuilds => $hasNewBuilds }); if ($hasNewBuilds) { while (my ($id, $new) = each %buildIds) { $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); } } }); # Store the error messages for jobs that failed to evaluate. my $msg = ""; foreach my $error (@{$jobs->{error}}) { my $bindings = ""; foreach my $arg (@{$error->{arg}}) { my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input"; $bindings .= ", " if $bindings ne ""; $bindings .= $arg->{name} . " = "; given ($input->{type}) { when ("string") { $bindings .= "\"" . $input->{value} . "\""; } when ("boolean") { $bindings .= $input->{value}; } default { $bindings .= "..."; } } } $msg .= "at `" . $error->{location} . "' [$bindings]:\n" . $error->{msg} . "\n\n"; } setJobsetError($jobset, $msg); } sub checkJobsetWrapped { my ($project, $jobset) = @_; print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n"; eval { checkJobset($project, $jobset); }; if ($@) { my $msg = $@; print STDERR "error evaluating jobset ", $jobset->name, ": $msg"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); setJobsetError($jobset, $msg); }); } } sub checkProjects { foreach my $project ($db->resultset('Projects')->search({enabled => 1})) { print STDERR "considering project ", $project->name, "\n"; checkJobsetWrapped($project, $_) foreach $project->jobsets->search({enabled => 1}); } } # For testing: evaluate a single jobset, then exit. if (scalar @ARGV == 2) { my $projectName = $ARGV[0]; my $jobsetName = $ARGV[1]; my $jobset = $db->resultset('Jobsets')->find($projectName, $jobsetName) or die; checkJobsetWrapped($jobset->project, $jobset); exit 0; } while (1) { eval { checkProjects; }; if ($@) { print STDERR "$@"; } print STDERR "sleeping...\n"; sleep 30; }