hydra/src/script/hydra-evaluator

#! /var/run/current-system/sw/bin/perl -w

use strict;
use feature 'switch';
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::AddBuilds;
use Hydra::Model::DB;
use Digest::SHA qw(sha256_hex);
use Email::Sender::Simple qw(sendmail);
use Email::Sender::Transport::SMTP;
use Email::Simple;
use Email::Simple::Creator;
use Sys::Hostname::Long;
use Config::General;
use Data::Dump qw(dump);

STDOUT->autoflush();

my $db = Hydra::Model::DB->new();
my %config = new Config::General(getHydraConf)->getall;


sub fetchInputs {
    my ($project, $jobset, $inputInfo) = @_;
    foreach my $input ($jobset->jobsetinputs->all) {
        foreach my $alt ($input->jobsetinputalts->all) {
	    my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
	    foreach my $info_el (@info) {
		push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
	    }
        }
    }
}


sub setJobsetError {
    my ($jobset, $errorMsg) = @_;
    eval {
        txn_do($db, sub {
            $jobset->update({errormsg => $errorMsg, errortime => time});
        });
    };
    sendJobsetErrorNotification($jobset, $errorMsg); 
}


sub sendJobsetErrorNotification() {
    my ($jobset, $errorMsg) = @_;

    return if $jobset->project->owner->emailonerror == 0;
    return if $errorMsg eq ""; 

    my $url = hostname_long;
    my $projectName = $jobset->project->name;
    my $jobsetName = $jobset->name;

    my $sender = $config{'notification_sender'} ||
        (($ENV{'USER'} || "hydra") .  "@" . $url);

    my $body = "Hi,\n"
        . "\n"
        . "This is to let you know that Hydra jobset evalation of $projectName:$jobsetName "
        . "resulted in the following error:\n"
        . "\n"
        . "$errorMsg"
        . "\n"
        . "Regards,\n\nThe Hydra build daemon.\n";

    my $email = Email::Simple->create(
        header => [
            To      => $jobset->project->owner->emailaddress,
            From    => "Hydra Build Daemon <$sender>",
            Subject => "Hydra $projectName:$jobsetName evaluation error",

            'X-Hydra-Instance' => $url,
            'X-Hydra-Project' => $projectName,
            'X-Hydra-Jobset'  => $jobsetName
        ], 
        body => ""
    );
    $email->body_set($body);

    print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'};

    sendmail($email);
}


sub permute {
    my @list = @_;
    for (my $n = scalar @list - 1; $n > 0; $n--) {
        my $k = int(rand($n + 1)); # 0 <= $k <= $n 
        @list[$n, $k] = @list[$k, $n];
    }
    return @list;
}


sub checkJobset {
    my ($project, $jobset) = @_;
    my $inputInfo = {};
    
    # Fetch all values for all inputs.
    my $checkoutStart = time;
    fetchInputs($project, $jobset, $inputInfo);
    my $checkoutStop = time;

    # Hash the arguments to hydra-eval-jobs and check the
    # JobsetInputHashes to see if the previous evaluation had the same
    # inputs.  If so, bail out.
    my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo));
    my $argsHash = sha256_hex("@args");
    my $prevEval = getPrevJobsetEval($db, $jobset, 0);
    if (defined $prevEval && $prevEval->hash eq $argsHash) {
        print STDERR "  jobset is unchanged, skipping\n";
        txn_do($db, sub {
            $jobset->update({lastcheckedtime => time});
        });
        return;
    }
    
    # Evaluate the job expression.
    my $evalStart = time;
    my ($jobs, $nixExprInput) = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath);
    my $evalStop = time;

    my $jobOutPathMap = {};

    txn_do($db, sub {
        
        my $prevEval = getPrevJobsetEval($db, $jobset, 1);

        # Clear the "current" flag on all builds.  Since we're in a
        # transaction this will only become visible after the new
        # current builds have been added.
        $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0});

        # Schedule each successfully evaluated job.
        my %buildIds;
        foreach my $job (permute @{$jobs->{job}}) {
            next if $job->{jobName} eq "";
            print STDERR "  considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
            checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap);
        }

        # Update the last checked times and error messages for each
        # job.
        my %failedJobNames;
        push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}};

        $jobset->update({lastcheckedtime => time});
        
        $_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef })
            foreach $jobset->jobs->all;

        my $hasNewBuilds = 0;
        while (my ($id, $new) = each %buildIds) {
            $hasNewBuilds = 1 if $new;
        }

        my $ev = $jobset->jobsetevals->create(
            { hash => $argsHash
            , timestamp => time
            , checkouttime => abs($checkoutStop - $checkoutStart)
            , evaltime => abs($evalStop - $evalStart)
            , hasnewbuilds => $hasNewBuilds
            });

        if ($hasNewBuilds) {
            while (my ($id, $new) = each %buildIds) {
                $ev->jobsetevalmembers->create({ build => $id, isnew => $new });
            }
            print STDERR "  created new eval ", $ev->id, "\n";
            $ev->builds->update({iscurrent => 1});
        } else {
            print STDERR "  created cached eval ", $ev->id, "\n";
            $prevEval->builds->update({iscurrent => 1});
        }
    });
       
    # Store the error messages for jobs that failed to evaluate.
    my $msg = "";
    foreach my $error (@{$jobs->{error}}) {
        my $bindings = "";
        foreach my $arg (@{$error->{arg}}) {
            my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
            $bindings .= ", " if $bindings ne "";
            $bindings .= $arg->{name} . " = ";
            given ($input->{type}) {
                when ("string") { $bindings .= "\"" . $input->{value} . "\""; }
                when ("boolean") { $bindings .= $input->{value}; }
                default { $bindings .= "..."; }
            }
        }
        $msg .= "at `" . $error->{location} . "' [$bindings]:\n" . $error->{msg} . "\n\n";
    }
    setJobsetError($jobset, $msg);
}


sub checkJobsetWrapped {
    my ($project, $jobset) = @_;
    
    print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n";
    
    eval {
        checkJobset($project, $jobset);
    };
    
    if ($@) {
        my $msg = $@;
        print STDERR "error evaluating jobset ", $jobset->name, ": $msg";
        txn_do($db, sub {
            $jobset->update({lastcheckedtime => time});
            setJobsetError($jobset, $msg);
        });
    }
}


sub checkProjects {
    foreach my $project ($db->resultset('Projects')->search({enabled => 1})) {
        print STDERR "considering project ", $project->name, "\n";
        checkJobsetWrapped($project, $_)
            foreach $project->jobsets->search({enabled => 1});
    }
}


# For testing: evaluate a single jobset, then exit.
if (scalar @ARGV == 2) {
    my $projectName = $ARGV[0];
    my $jobsetName = $ARGV[1];
    my $jobset = $db->resultset('Jobsets')->find($projectName, $jobsetName) or die;
    checkJobsetWrapped($jobset->project, $jobset);
    exit 0;
}


while (1) {
    eval {
        checkProjects;
    };
    if ($@) { print STDERR "$@"; }
    print STDERR "sleeping...\n";
    sleep 30;
}