hydra/src/script/hydra-evaluator

#! /var/run/current-system/sw/bin/perl -w

use strict;
use feature 'switch';
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::AddBuilds;
use Hydra::Model::DB;
use Digest::SHA qw(sha256_hex);
use Email::Sender::Simple qw(sendmail);
use Email::Sender::Transport::SMTP;
use Email::Simple;
use Email::Simple::Creator;
use Sys::Hostname::Long;
use Config::General;
use Data::Dump qw(dump);

STDOUT->autoflush();

my $db = Hydra::Model::DB->new();
my $config = getHydraConfig();

# Don't check a jobset more than once every five minutes.
my $minCheckInterval = 5 * 60;



sub fetchInputs {
    my ($project, $jobset, $inputInfo) = @_;
    foreach my $input ($jobset->jobsetinputs->all) {
        foreach my $alt ($input->jobsetinputalts->all) {
            my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
            foreach my $info_el (@info) {
                push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
            }
        }
    }
}


sub setJobsetError {
    my ($jobset, $errorMsg) = @_;
    eval {
        txn_do($db, sub {
            $jobset->update({errormsg => $errorMsg, errortime => time});
        });
    };
    sendJobsetErrorNotification($jobset, $errorMsg);
}


sub sendJobsetErrorNotification() {
    my ($jobset, $errorMsg) = @_;

    return if $jobset->project->owner->emailonerror == 0;
    return if $errorMsg eq "";

    my $url = hostname_long;
    my $projectName = $jobset->project->name;
    my $jobsetName = $jobset->name;

    my $sender = $config->{'notification_sender'} ||
        (($ENV{'USER'} || "hydra") .  "@" . $url);

    my $body = "Hi,\n"
        . "\n"
        . "This is to let you know that Hydra jobset evalation of $projectName:$jobsetName "
        . "resulted in the following error:\n"
        . "\n"
        . "$errorMsg"
        . "\n"
        . "Regards,\n\nThe Hydra build daemon.\n";

    my $email = Email::Simple->create(
        header => [
            To      => $jobset->project->owner->emailaddress,
            From    => "Hydra Build Daemon <$sender>",
            Subject => "Hydra $projectName:$jobsetName evaluation error",

            'X-Hydra-Instance' => $url,
            'X-Hydra-Project' => $projectName,
            'X-Hydra-Jobset'  => $jobsetName
        ],
        body => ""
    );
    $email->body_set($body);

    print STDERR $email->as_string if $ENV{'HYDRA_MAIL_TEST'};

    sendmail($email);
}


sub permute {
    my @list = @_;
    for (my $n = scalar @list - 1; $n > 0; $n--) {
        my $k = int(rand($n + 1)); # 0 <= $k <= $n
        @list[$n, $k] = @list[$k, $n];
    }
    return @list;
}


sub checkJobsetWrapped {
    my ($jobset) = @_;
    my $project = $jobset->project;
    my $inputInfo = {};
    my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix";

    # Fetch all values for all inputs.
    my $checkoutStart = time;
    fetchInputs($project, $jobset, $inputInfo);
    my $checkoutStop = time;

    # Hash the arguments to hydra-eval-jobs and check the
    # JobsetInputHashes to see if the previous evaluation had the same
    # inputs.  If so, bail out.
    my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType));
    my $argsHash = sha256_hex("@args");
    my $prevEval = getPrevJobsetEval($db, $jobset, 0);
    if (defined $prevEval && $prevEval->hash eq $argsHash) {
        print STDERR "  jobset is unchanged, skipping\n";
        txn_do($db, sub {
            $jobset->update({lastcheckedtime => time});
        });
        return;
    }

    # Evaluate the job expression.
    my $evalStart = time;
    my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
    my $evalStop = time;

    my $jobOutPathMap = {};

    txn_do($db, sub {

        my $prevEval = getPrevJobsetEval($db, $jobset, 1);

        # Clear the "current" flag on all builds.  Since we're in a
        # transaction this will only become visible after the new
        # current builds have been added.
        $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0});

        # Schedule each successfully evaluated job.
        my %buildIds;
        foreach my $job (permute @{$jobs->{job}}) {
            next if $job->{jobName} eq "";
            print STDERR "  considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n";
            checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap);
        }

        # Update the last checked times and error messages for each
        # job.
        my %failedJobNames;
        push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}};

        $jobset->update({lastcheckedtime => time});

        $_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef })
            foreach $jobset->jobs->all;

        my $hasNewBuilds = 0;
        while (my ($id, $new) = each %buildIds) {
            $hasNewBuilds = 1 if $new;
        }

        my $ev = $jobset->jobsetevals->create(
            { hash => $argsHash
            , timestamp => time
            , checkouttime => abs($checkoutStop - $checkoutStart)
            , evaltime => abs($evalStop - $evalStart)
            , hasnewbuilds => $hasNewBuilds
            , nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef
            });

        if ($hasNewBuilds) {
            while (my ($id, $new) = each %buildIds) {
                $ev->jobsetevalmembers->create({ build => $id, isnew => $new });
            }

            foreach my $name (keys %{$inputInfo}) {
                for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) {
                    my $input = $inputInfo->{$name}->[$n];
                    $ev->jobsetevalinputs->create(
                        { name => $name
                        , altnr => $n
                        , type => $input->{type}
                        , uri => $input->{uri}
                        , revision => $input->{revision}
                        , value => $input->{value}
                        , dependency => $input->{id}
                        , path => $input->{storePath} || "" # !!! temporary hack
                        , sha256hash => $input->{sha256hash}
                        });
                }
            }

            print STDERR "  created new eval ", $ev->id, "\n";
            $ev->builds->update({iscurrent => 1});
        } else {
            print STDERR "  created cached eval ", $ev->id, "\n";
            $prevEval->builds->update({iscurrent => 1}) if defined $prevEval;
        }
    });

    # Store the error messages for jobs that failed to evaluate.
    my $msg = "";
    foreach my $error (@{$jobs->{error}}) {
        my $bindings = "";
        foreach my $arg (@{$error->{arg}}) {
            my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}] or die "invalid input";
            $bindings .= ", " if $bindings ne "";
            $bindings .= $arg->{name} . " = ";
            given ($input->{type}) {
                when ("string") { $bindings .= "\"" . $input->{value} . "\""; }
                when ("boolean") { $bindings .= $input->{value}; }
                default { $bindings .= "..."; }
            }
        }
        $msg .= "at `" . $error->{location} . "' [$bindings]:\n" . $error->{msg} . "\n\n";
    }
    setJobsetError($jobset, $msg);
}


sub checkJobset {
    my ($jobset) = @_;

    print STDERR "considering jobset ", $jobset->project->name, ":", $jobset->name,
      $jobset->lastcheckedtime
          ? " (last checked " . (time() - $jobset->lastcheckedtime) . "s ago)\n"
          : " (never checked)\n";

    my $triggerTime = $jobset->triggertime;

    eval {
        checkJobsetWrapped($jobset);
    };

    if ($@) {
        my $msg = $@;
        print STDERR "error evaluating jobset ", $jobset->name, ": $msg";
        txn_do($db, sub {
            $jobset->update({lastcheckedtime => time});
            setJobsetError($jobset, $msg);
        });
    }

    if (defined $triggerTime) {
        txn_do($db, sub {
            # Only clear the trigger time if the jobset hasn't been
            # triggered in the meantime.  In that case, we need to
            # evaluate again.
            my $new = $jobset->get_from_storage();
            $jobset->update({ triggertime => undef })
                if $new->triggertime == $triggerTime;
        });
    }
}


sub checkSomeJobset {
    # If any jobset has been triggered by a push, check it.
    my ($jobset) = $db->resultset('Jobsets')->search(
        { 'project.enabled' => 1, 'me.enabled' => 1, 'triggertime' => { '!=', undef },
        , -or => [ 'lastcheckedtime' => undef, 'lastcheckedtime' => { '<', time() - $minCheckInterval } ] },
        { join => 'project', order_by => [ 'triggertime' ], rows => 1 });

    # Otherwise, check the jobset that hasn't been checked for the
    # longest time (but don't check more often than the minimal check
    # interval).
    ($jobset) = $db->resultset('Jobsets')->search(
        { 'project.enabled' => 1, 'me.enabled' => 1,
        , -or => [ 'lastcheckedtime' => undef, 'lastcheckedtime' => { '<', time() - $minCheckInterval } ] },
        { join => 'project', order_by => [ 'lastcheckedtime nulls first' ], rows => 1 })
        unless defined $jobset;

    return 0 unless defined $jobset;

    checkJobset($jobset);

    return 1;
}


# For testing: evaluate a single jobset, then exit.
if (scalar @ARGV == 2) {
    my $projectName = $ARGV[0];
    my $jobsetName = $ARGV[1];
    my $jobset = $db->resultset('Jobsets')->find($projectName, $jobsetName) or die;
    checkJobset($jobset);
    exit 0;
}


while (1) {
    eval {
        if (checkSomeJobset) {
            # Just so we don't go completely crazy if lastcheckedtime
            # isn't updated properly.
            sleep 1;
        } else {
            # print STDERR "sleeping...\n";
            sleep 30;
        }
    };
    if ($@) { print STDERR "$@"; }
}