hydra/src/script/hydra-build

#! /var/run/current-system/sw/bin/perl -w

use strict;
use List::MoreUtils qw(all);
use File::Basename;
use File::stat;
use Nix::Store;
use Hydra::Plugin;
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::PluginHooks;
use Hydra::Model::DB;
use Hydra::Helper::AddBuilds;
use Set::Scalar;

STDOUT->autoflush();

my $db = Hydra::Model::DB->new();

my $config = getHydraConfig();

my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);


sub addBuildStepOutputs {
    my ($step) = @_;
    my $drv = derivationFromPath($step->drvpath);
    $step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} })
        foreach keys %{$drv->{outputs}};
}


sub nextFreeStepNr {
    my ($build) = @_;
    my $max = $build->buildsteps->find(
        {}, {select => {max => 'stepnr + 1'}, as => ['max']});
    return (defined $max && defined $max->get_column('max')) ? $max->get_column('max') : 1;
}


sub failDependents {
    my ($drvPath, $status, $errorMsg, $dependents) = @_;

    # Get the referrer closure of $drvPath.
    my $dependentDrvs = Set::Scalar->new(computeFSClosure(1, 0, $drvPath));

    my $time = time();

    txn_do($db, sub {

        my @dependentBuilds = $db->resultset('Builds')->search(
            { finished => 0, busy => 0 },
            { columns => ["id", "project", "jobset", "job", "drvpath", "finished", "busy"] });

        for my $d (@dependentBuilds) {
            next unless $dependentDrvs->has($d->drvpath);
            print STDERR "failing dependent build ", $d->id, " of ", $d->project->name, ":", $d->jobset->name, ":", $d->job->name, "\n";
            $d->update(
                { finished => 1
                , logfile => ''
                , iscachedbuild => 0
                , buildstatus => $drvPath eq $d->drvpath ? 1 : 2
                , starttime => $time
                , stoptime => $time
                , errormsg => undef
                });

            my $step = $d->buildsteps->create(
                { stepnr => nextFreeStepNr($d)
                , type => 0 # = build
                , drvpath => $drvPath
                , busy => 0
                , status => $status
                , starttime => $time
                , stoptime => $time
                , errormsg => $errorMsg
                });
            addBuildStepOutputs($step);

            push @$dependents, $d;
        }

    });
}


sub doBuild {
    my ($build) = @_;

    my %outputs;
    $outputs{$_->name} = $_->path foreach $build->buildoutputs->all;

    my $drvPath   = $build->drvpath;
    my $maxsilent = $build->maxsilent;
    my $timeout   = $build->timeout;

    my $isCachedBuild = 1;
    my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
    my $startTime = time();
    my $stopTime = undef;

    my $buildStatus = 0; # = succeeded

    my $errormsg = undef;

    my $dependents = [];

    if (!isValidPath($drvPath)) {
        $buildStatus = 3;
        $errormsg = "derivation was garbage-collected prior to build";
        goto done;
    }

    unless (all { isValidPath($_) } values(%outputs)) {
        $isCachedBuild = 0;

        # Do the build.
        my $thisBuildFailed = 0;
        my $someBuildFailed = 0;

        # Run Nix to perform the build, and monitor the stderr output
        # to get notifications about specific build steps, the
        # associated log files, etc.
        my $cmd = "nix-store --realise $drvPath " .
            "--timeout $timeout " .
            "--max-silent-time $maxsilent " .
            "--option build-max-log-size 67108864 " .
            "--keep-going --fallback " .
            "--no-build-output --log-type flat --print-build-trace " .
            "--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1";

        my $buildStepNr = nextFreeStepNr($build);
        my %buildSteps;

        open OUT, "$cmd |" or die;

        while (<OUT>) {
            $errormsg .= $_;

            unless (/^@\s+/) {
                print STDERR "$_";
                next;
            }

            # Hack to handle timeouts, which Nix doesn't report
            # properly when they occur remotely.  If we get a "hook
            # failed" error and $maxsilent seconds have passed since
            # the start of the build step, then assume that a timeout
            # occured.
            if (/^@\s+hook-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/ && $3 eq "256") {
                my $drvPathStep = $1;
                if ($buildSteps{$drvPathStep}) {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    print STDERR $step->starttime, " ", time(), "\n";
                    if ($step->starttime + $maxsilent <= time) {
                        $_ = "@ build-failed $1 $2 timeout $4";
                    }
                }
            }

            if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
                my $drvPathStep = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->create(
                        { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
                        , type => 0 # = build
                        , drvpath => $drvPathStep
                        , system => $3
                        , busy => 1
                        , starttime => time
                        });
                    addBuildStepOutputs($step);
                });
            }

            elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) {
                my $drvPathStep = $1;
                my $machine = $2;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    $step->update({machine => $machine});
                });
            }

            elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
                my $drvPathStep = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                    $step->update({busy => 0, status => 0, stoptime => time});
                });
            }

            elsif (/^@\s+build-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
                my $drvPathStep = $1;
                $someBuildFailed = 1;
                $thisBuildFailed = 1 if $drvPath eq $drvPathStep;
                my $errorMsg;
                my $status = 1;
                if ($3 eq "cached") {
                    $status = 8;
                } elsif ($3 eq "timeout") {
                    $status = 7;
                } else {
                    $errorMsg = $4;
                }
                txn_do($db, sub {
                    if ($buildSteps{$drvPathStep}) {
                        my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
                        $step->update({busy => 0, status => $status, errormsg => $errorMsg, stoptime => time});
                    }
                    # Don't write a record if this derivation already
                    # failed previously.  This can happen if this is a
                    # restarted build.
                    elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
                        my $step = $build->buildsteps->create(
                            { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
                            , type => 0 # = build
                            , drvpath => $drvPathStep
                            , busy => 0
                            , status => $status
                            , starttime => time
                            , stoptime => time
                            , errormsg => $errorMsg
                            });
                        addBuildStepOutputs($step);
                    }
                });

                # Immediately fail all builds that depend on this derivation.
                failDependents($drvPathStep, $status, $errorMsg, $dependents);
            }

            elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
                my $path = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->create(
                        { stepnr => ($buildSteps{$path} = $buildStepNr++)
                        , type => 1 # = substitution
                        , busy => 1
                        , starttime => time
                        });
                    # "out" is kinda fake (substitutions don't have named outputs).
                    $step->buildstepoutputs->create({ name => "out", path => $path });
                });
            }

            elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
                my $path = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
                    $step->update({busy => 0, status => 0, stoptime => time});
                });
            }

            elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
                my $path = $1;
                txn_do($db, sub {
                    my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
                    $step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
                });
            }

            else {
                print STDERR "unknown Nix trace message: $_";
            }
        }

        close OUT;

        my $res = $?;

        $stopTime = time();

        if ($res != 0) {
            if ($thisBuildFailed) { $buildStatus = 1; }
            elsif ($someBuildFailed) { $buildStatus = 2; }
            else { $buildStatus = 3; }
        }

        # Only store the output of running Nix if we have a miscellaneous error.
        $errormsg = undef unless $buildStatus == 3;
    }

  done:

    txn_do($db, sub {
        if ($buildStatus == 0) {

            my $size = 0;
            my $closureSize = 0;
            my $releaseName;

            my @closure = computeFSClosure(0, 0, values %outputs);
            foreach my $path (@closure) {
                my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
                $closureSize += $narSize;
                $size += $narSize if grep { $path eq $_ } values(%outputs);
            }

            foreach my $path (values %outputs) {
                $buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed";
                $releaseName //= getReleaseName($path);
            }

            $build->update(
                { releasename => $releaseName
                , size => $size
                , closuresize => $closureSize
                });

            addBuildProducts($db, $build);
        }

        # Mark any remaining active build steps as aborted.
        $build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });

        $build->update(
            { finished => 1
            , busy => 0
            , locker => ''
            , logfile => ''
            , iscachedbuild => $isCachedBuild
            , buildstatus => $buildStatus
            , starttime => $startTime
            , stoptime => $stopTime // time()
            , errormsg => $errormsg
            });

    });

    notifyBuildFinished(\@plugins, $build, $dependents);
}


my $buildId = $ARGV[0] or die "syntax: $0 BUILD-ID\n";
print STDERR "performing build $buildId\n";

if ($ENV{'HYDRA_MAIL_TEST'}) {
    my $build = $db->resultset('Builds')->find($buildId);
    notifyBuildFinished(\@plugins, $build, []);
    exit 0;
}

# Lock the build.  If necessary, steal the lock from the parent
# process (runner.pl).  This is so that if the runner dies, the
# children (i.e. the build.pl instances) can continue to run and won't
# have the lock taken away.
my $build;
txn_do($db, sub {
    $build = $db->resultset('Builds')->find($buildId);
    die "build $buildId doesn't exist\n" unless defined $build;
    die "build $buildId already done\n" if $build->finished;
    if ($build->busy != 0 && $build->locker != getppid) {
        die "build $buildId is already being built";
    }
    $build->update({busy => 1, locker => $$});
    $build->buildsteps->search({busy => 1})->delete;
    $build->buildproducts->delete;
});

die unless $build;

# Do the build.  If it throws an error, unlock the build so that it
# can be retried.
eval {
    doBuild $build;
    print "done\n";
};
if ($@) {
    warn $@;
    txn_do($db, sub {
        $build->update({busy => 0, locker => $$});
    });
}