#! /var/run/current-system/sw/bin/perl -w use strict; use feature 'switch'; use Hydra::Schema; use Hydra::Helper::Nix; use Hydra::Helper::AddBuilds; use Hydra::Model::DB; use Digest::SHA qw(sha256_hex); use Email::Sender::Simple qw(sendmail); use Email::Sender::Transport::SMTP; use Email::Simple; use Email::Simple::Creator; use Sys::Hostname::Long; use Config::General; use Data::Dump qw(dump); STDOUT->autoflush(); my $db = Hydra::Model::DB->new(); my $config = getHydraConfig(); # Don't check a jobset more than once every five minutes. my $minCheckInterval = 5 * 60; sub fetchInputs { my ($project, $jobset, $inputInfo) = @_; foreach my $input ($jobset->jobsetinputs->all) { foreach my $alt ($input->jobsetinputalts->all) { my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value); foreach my $info_el (@info) { push @{$$inputInfo{$input->name}}, $info_el if defined $info_el; } } } } sub setJobsetError { my ($jobset, $errorMsg) = @_; eval { txn_do($db, sub { $jobset->update({errormsg => $errorMsg, errortime => time}); }); }; sendJobsetErrorNotification($jobset, $errorMsg); } sub sendJobsetErrorNotification() { my ($jobset, $errorMsg) = @_; return if $jobset->project->owner->emailonerror == 0; return if $errorMsg eq ""; my $url = hostname_long; my $projectName = $jobset->project->name; my $jobsetName = $jobset->name; my $sender = $config->{'notification_sender'} || (($ENV{'USER'} || "hydra") . "@" . $url); my $body = "Hi,\n" . "\n" . "This is to let you know that Hydra jobset evalation of $projectName:$jobsetName " . "resulted in the following error:\n" . "\n" . "$errorMsg" . "\n" . "Regards,\n\nThe Hydra build daemon.\n"; my $email = Email::Simple->create( header => [ To => $jobset->project->owner->emailaddress, From => "Hydra Build Daemon <$sender>", Subject => "Hydra $projectName:$jobsetName evaluation error", 'X-Hydra-Instance' => $url, 'X-Hydra-Project' => $projectName, 'X-Hydra-Jobset' => $jobsetName ], body => "" ); $email->body_set($body); print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'}; sendmail($email); } sub permute { my @list = @_; for (my $n = scalar @list - 1; $n > 0; $n--) { my $k = int(rand($n + 1)); # 0 <= $k <= $n @list[$n, $k] = @list[$k, $n]; } return @list; } sub checkJobsetWrapped { my ($jobset) = @_; my $project = $jobset->project; my $inputInfo = {}; my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix"; # Fetch all values for all inputs. my $checkoutStart = time; fetchInputs($project, $jobset, $inputInfo); my $checkoutStop = time; # Hash the arguments to hydra-eval-jobs and check the # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType)); my $argsHash = sha256_hex("@args"); my $prevEval = getPrevJobsetEval($db, $jobset, 0); if (defined $prevEval && $prevEval->hash eq $argsHash) { print STDERR " jobset is unchanged, skipping\n"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); }); return; } # Evaluate the job expression. my $evalStart = time; my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); my $evalStop = time; my $jobOutPathMap = {}; txn_do($db, sub { my $prevEval = getPrevJobsetEval($db, $jobset, 1); # Clear the "current" flag on all builds. Since we're in a # transaction this will only become visible after the new # current builds have been added. $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); # Schedule each successfully evaluated job. my %buildIds; foreach my $job (permute @{$jobs->{job}}) { next if $job->{jobName} eq ""; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap); } # Update the last checked times and error messages for each # job. my %failedJobNames; push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}}; $jobset->update({lastcheckedtime => time}); $_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef }) foreach $jobset->jobs->all; my $hasNewBuilds = 0; while (my ($id, $new) = each %buildIds) { $hasNewBuilds = 1 if $new; } my $ev = $jobset->jobsetevals->create( { hash => $argsHash , timestamp => time , checkouttime => abs($checkoutStop - $checkoutStart) , evaltime => abs($evalStop - $evalStart) , hasnewbuilds => $hasNewBuilds , nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef }); if ($hasNewBuilds) { while (my ($id, $new) = each %buildIds) { $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); } foreach my $name (keys %{$inputInfo}) { for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) { my $input = $inputInfo->{$name}->[$n]; $ev->jobsetevalinputs->create( { name => $name , altnr => $n , type => $input->{type} , uri => $input->{uri} , revision => $input->{revision} , value => $input->{value} , dependency => $input->{id} , path => $input->{storePath} || "" # !!! temporary hack , sha256hash => $input->{sha256hash} }); } } print STDERR " created new eval ", $ev->id, "\n"; $ev->builds->update({iscurrent => 1}); } else { print STDERR " created cached eval ", $ev->id, "\n"; $prevEval->builds->update({iscurrent => 1}) if defined $prevEval; } }); # Store the error messages for jobs that failed to evaluate. my $msg = ""; foreach my $error (@{$jobs->{error}}) { my $bindings = ""; foreach my $arg (@{$error->{arg}}) { my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input"; $bindings .= ", " if $bindings ne ""; $bindings .= $arg->{name} . " = "; given ($input->{type}) { when ("string") { $bindings .= "\"" . $input->{value} . "\""; } when ("boolean") { $bindings .= $input->{value}; } default { $bindings .= "..."; } } } $msg .= "at `" . $error->{location} . "' [$bindings]:\n" . $error->{msg} . "\n\n"; } setJobsetError($jobset, $msg); } sub checkJobset { my ($jobset) = @_; print STDERR "considering jobset ", $jobset->project->name, ":", $jobset->name, $jobset->lastcheckedtime ? " (last checked " . (time() - $jobset->lastcheckedtime) . "s ago)\n" : " (never checked)\n"; my $triggerTime = $jobset->triggertime; eval { checkJobsetWrapped($jobset); }; if ($@) { my $msg = $@; print STDERR "error evaluating jobset ", $jobset->name, ": $msg"; txn_do($db, sub { $jobset->update({lastcheckedtime => time}); setJobsetError($jobset, $msg); }); } if (defined $triggerTime) { txn_do($db, sub { # Only clear the trigger time if the jobset hasn't been # triggered in the meantime. In that case, we need to # evaluate again. my $new = $jobset->get_from_storage(); $jobset->update({ triggertime => undef }) if $new->triggertime == $triggerTime; }); } } sub checkSomeJobset { # If any jobset has been triggered by a push, check it. my ($jobset) = $db->resultset('Jobsets')->search( { 'project.enabled' => 1, 'me.enabled' => 1, 'triggertime' => { '!=', undef }, , -or => [ 'lastcheckedtime' => undef, 'lastcheckedtime' => { '<', time() - $minCheckInterval } ] }, { join => 'project', order_by => [ 'triggertime' ], rows => 1 }); # Otherwise, check the jobset that hasn't been checked for the # longest time (but don't check more often than the minimal check # interval). ($jobset) = $db->resultset('Jobsets')->search( { 'project.enabled' => 1, 'me.enabled' => 1, , -or => [ 'lastcheckedtime' => undef, 'lastcheckedtime' => { '<', time() - $minCheckInterval } ] }, { join => 'project', order_by => [ 'lastcheckedtime nulls first' ], rows => 1 }) unless defined $jobset; return 0 unless defined $jobset; checkJobset($jobset); return 1; } # For testing: evaluate a single jobset, then exit. if (scalar @ARGV == 2) { my $projectName = $ARGV[0]; my $jobsetName = $ARGV[1]; my $jobset = $db->resultset('Jobsets')->find($projectName, $jobsetName) or die; checkJobset($jobset); exit 0; } while (1) { eval { if (checkSomeJobset) { # Just so we don't go completely crazy if lastcheckedtime # isn't updated properly. sleep 1; } else { # print STDERR "sleeping...\n"; sleep 30; } }; if ($@) { print STDERR "$@"; } }