forked from lix-project/hydra
366 lines
12 KiB
Perl
Executable file
366 lines
12 KiB
Perl
Executable file
#! /var/run/current-system/sw/bin/perl -w
|
|
|
|
use strict;
|
|
use List::MoreUtils qw(all);
|
|
use File::Basename;
|
|
use File::stat;
|
|
use Nix::Store;
|
|
use Hydra::Plugin;
|
|
use Hydra::Schema;
|
|
use Hydra::Helper::Nix;
|
|
use Hydra::Model::DB;
|
|
use Hydra::Helper::AddBuilds;
|
|
|
|
STDOUT->autoflush();
|
|
|
|
my $db = Hydra::Model::DB->new();
|
|
|
|
my $config = getHydraConfig();
|
|
|
|
my @plugins = Hydra::Plugin->plugins(db => $db, config => $config);
|
|
|
|
|
|
sub addBuildStepOutputs {
|
|
my ($step) = @_;
|
|
my $drv = derivationFromPath($step->drvpath);
|
|
$step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} })
|
|
foreach keys %{$drv->{outputs}};
|
|
}
|
|
|
|
|
|
sub nextFreeStepNr {
|
|
my ($build) = @_;
|
|
my $max = $build->buildsteps->find(
|
|
{}, {select => {max => 'stepnr + 1'}, as => ['max']});
|
|
return (defined $max && defined $max->get_column('max')) ? $max->get_column('max') : 1;
|
|
}
|
|
|
|
|
|
sub failDependents {
|
|
my ($drvPath, $status, $errorMsg, $dependents) = @_;
|
|
|
|
# Get the referrer closure of $drvPath.
|
|
my @dependentDrvs = computeFSClosure(1, 0, $drvPath);
|
|
|
|
my $time = time();
|
|
|
|
txn_do($db, sub {
|
|
|
|
my @dependentBuilds = $db->resultset('Builds')->search(
|
|
{ drvpath => [ @dependentDrvs ], finished => 0, busy => 0 });
|
|
|
|
for my $d (@dependentBuilds) {
|
|
print STDERR "failing dependent build ", $d->id, " of ", $d->project->name, ":", $d->jobset->name, ":", $d->job->name, "\n";
|
|
$d->update(
|
|
{ finished => 1
|
|
, logfile => ''
|
|
, iscachedbuild => 0
|
|
, buildstatus => $drvPath eq $d->drvpath ? 1 : 2
|
|
, starttime => $time
|
|
, stoptime => $time
|
|
, errormsg => undef
|
|
});
|
|
|
|
my $step = $d->buildsteps->create(
|
|
{ stepnr => nextFreeStepNr($d)
|
|
, type => 0 # = build
|
|
, drvpath => $drvPath
|
|
, busy => 0
|
|
, status => $status
|
|
, starttime => time
|
|
, stoptime => time
|
|
, errormsg => $errorMsg
|
|
});
|
|
addBuildStepOutputs($step);
|
|
|
|
push @$dependents, $d;
|
|
}
|
|
|
|
});
|
|
}
|
|
|
|
|
|
sub notify {
|
|
my ($build, $dependents) = @_;
|
|
foreach my $plugin (@plugins) {
|
|
eval {
|
|
$plugin->buildFinished($build, $dependents);
|
|
};
|
|
if ($@) {
|
|
print STDERR "$plugin->buildFinished: $@\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
sub doBuild {
|
|
my ($build) = @_;
|
|
|
|
my %outputs;
|
|
$outputs{$_->name} = $_->path foreach $build->buildoutputs->all;
|
|
|
|
my $drvPath = $build->drvpath;
|
|
my $maxsilent = $build->maxsilent;
|
|
my $timeout = $build->timeout;
|
|
|
|
my $isCachedBuild = 1;
|
|
my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
|
|
my $startTime = time();
|
|
my $stopTime = undef;
|
|
|
|
my $buildStatus = 0; # = succeeded
|
|
|
|
my $errormsg = undef;
|
|
|
|
my $dependents = [];
|
|
|
|
if (!isValidPath($drvPath)) {
|
|
$buildStatus = 3;
|
|
$errormsg = "derivation was garbage-collected prior to build";
|
|
goto done;
|
|
}
|
|
|
|
unless (all { isValidPath($_) } values(%outputs)) {
|
|
$isCachedBuild = 0;
|
|
|
|
# Do the build.
|
|
my $thisBuildFailed = 0;
|
|
my $someBuildFailed = 0;
|
|
|
|
# Run Nix to perform the build, and monitor the stderr output
|
|
# to get notifications about specific build steps, the
|
|
# associated log files, etc.
|
|
my $cmd = "nix-store --realise $drvPath " .
|
|
"--timeout $timeout " .
|
|
"--max-silent-time $maxsilent --keep-going --fallback " .
|
|
"--no-build-output --log-type flat --print-build-trace " .
|
|
"--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1";
|
|
|
|
my $buildStepNr = nextFreeStepNr($build);
|
|
my %buildSteps;
|
|
|
|
open OUT, "$cmd |" or die;
|
|
|
|
while (<OUT>) {
|
|
$errormsg .= $_;
|
|
|
|
unless (/^@\s+/) {
|
|
print STDERR "$_";
|
|
next;
|
|
}
|
|
|
|
if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
|
|
my $drvPathStep = $1;
|
|
txn_do($db, sub {
|
|
my $step = $build->buildsteps->create(
|
|
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
|
|
, type => 0 # = build
|
|
, drvpath => $drvPathStep
|
|
, system => $3
|
|
, busy => 1
|
|
, starttime => time
|
|
});
|
|
addBuildStepOutputs($step);
|
|
});
|
|
}
|
|
|
|
elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) {
|
|
my $drvPathStep = $1;
|
|
my $machine = $2;
|
|
txn_do($db, sub {
|
|
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
|
|
$step->update({machine => $machine});
|
|
});
|
|
}
|
|
|
|
elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
|
|
my $drvPathStep = $1;
|
|
txn_do($db, sub {
|
|
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
|
|
$step->update({busy => 0, status => 0, stoptime => time});
|
|
});
|
|
}
|
|
|
|
elsif (/^@\s+build-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
|
|
my $drvPathStep = $1;
|
|
$someBuildFailed = 1;
|
|
$thisBuildFailed = 1 if $drvPath eq $drvPathStep;
|
|
my $errorMsg;
|
|
my $status = 1;
|
|
if ($3 eq "cached") {
|
|
$status = 8;
|
|
} elsif ($3 eq "timeout") {
|
|
$status = 7;
|
|
} else {
|
|
$errorMsg = $4;
|
|
}
|
|
txn_do($db, sub {
|
|
if ($buildSteps{$drvPathStep}) {
|
|
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
|
|
$step->update({busy => 0, status => $status, errormsg => $errorMsg, stoptime => time});
|
|
}
|
|
# Don't write a record if this derivation already
|
|
# failed previously. This can happen if this is a
|
|
# restarted build.
|
|
elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
|
|
my $step = $build->buildsteps->create(
|
|
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
|
|
, type => 0 # = build
|
|
, drvpath => $drvPathStep
|
|
, busy => 0
|
|
, status => $status
|
|
, starttime => time
|
|
, stoptime => time
|
|
, errormsg => $errorMsg
|
|
});
|
|
addBuildStepOutputs($step);
|
|
}
|
|
});
|
|
|
|
# Immediately fail all builds that depend on this derivation.
|
|
failDependents($drvPathStep, $status, $errorMsg, $dependents);
|
|
}
|
|
|
|
elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
|
|
my $path = $1;
|
|
txn_do($db, sub {
|
|
my $step = $build->buildsteps->create(
|
|
{ stepnr => ($buildSteps{$path} = $buildStepNr++)
|
|
, type => 1 # = substitution
|
|
, busy => 1
|
|
, starttime => time
|
|
});
|
|
# "out" is kinda fake (substitutions don't have named outputs).
|
|
$step->buildstepoutputs->create({ name => "out", path => $path });
|
|
});
|
|
}
|
|
|
|
elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
|
|
my $path = $1;
|
|
txn_do($db, sub {
|
|
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
|
|
$step->update({busy => 0, status => 0, stoptime => time});
|
|
});
|
|
}
|
|
|
|
elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
|
|
my $path = $1;
|
|
txn_do($db, sub {
|
|
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
|
|
$step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
|
|
});
|
|
}
|
|
|
|
else {
|
|
print STDERR "unknown Nix trace message: $_";
|
|
}
|
|
}
|
|
|
|
close OUT;
|
|
|
|
my $res = $?;
|
|
|
|
$stopTime = time();
|
|
|
|
if ($res != 0) {
|
|
if ($thisBuildFailed) { $buildStatus = 1; }
|
|
elsif ($someBuildFailed) { $buildStatus = 2; }
|
|
else { $buildStatus = 3; }
|
|
}
|
|
|
|
# Only store the output of running Nix if we have a miscellaneous error.
|
|
$errormsg = undef unless $buildStatus == 3;
|
|
}
|
|
|
|
done:
|
|
|
|
txn_do($db, sub {
|
|
if ($buildStatus == 0) {
|
|
|
|
my $size = 0;
|
|
my $closureSize = 0;
|
|
my $releaseName;
|
|
|
|
my @closure = computeFSClosure(0, 0, values %outputs);
|
|
foreach my $path (@closure) {
|
|
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
|
|
$closureSize += $narSize;
|
|
$size += $narSize if grep { $path eq $_ } values(%outputs);
|
|
}
|
|
|
|
foreach my $path (values %outputs) {
|
|
$buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed";
|
|
$releaseName //= getReleaseName($path);
|
|
}
|
|
|
|
$build->update(
|
|
{ releasename => $releaseName
|
|
, size => $size
|
|
, closuresize => $closureSize
|
|
});
|
|
|
|
addBuildProducts($db, $build);
|
|
}
|
|
|
|
# Mark any remaining active build steps as aborted.
|
|
$build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
|
|
|
|
$build->update(
|
|
{ finished => 1
|
|
, busy => 0
|
|
, locker => ''
|
|
, logfile => ''
|
|
, iscachedbuild => $isCachedBuild
|
|
, buildstatus => $buildStatus
|
|
, starttime => $startTime
|
|
, stoptime => $stopTime // time()
|
|
, errormsg => $errormsg
|
|
});
|
|
|
|
});
|
|
|
|
notify($build, $dependents);
|
|
}
|
|
|
|
|
|
my $buildId = $ARGV[0] or die "syntax: $0 BUILD-ID\n";
|
|
print STDERR "performing build $buildId\n";
|
|
|
|
if ($ENV{'HYDRA_MAIL_TEST'}) {
|
|
my $build = $db->resultset('Builds')->find($buildId);
|
|
notify($build, []);
|
|
exit 0;
|
|
}
|
|
|
|
# Lock the build. If necessary, steal the lock from the parent
|
|
# process (runner.pl). This is so that if the runner dies, the
|
|
# children (i.e. the build.pl instances) can continue to run and won't
|
|
# have the lock taken away.
|
|
my $build;
|
|
txn_do($db, sub {
|
|
$build = $db->resultset('Builds')->find($buildId);
|
|
die "build $buildId doesn't exist\n" unless defined $build;
|
|
die "build $buildId already done\n" if $build->finished;
|
|
if ($build->busy != 0 && $build->locker != getppid) {
|
|
die "build $buildId is already being built";
|
|
}
|
|
$build->update({busy => 1, locker => $$});
|
|
$build->buildsteps->search({busy => 1})->delete_all;
|
|
$build->buildproducts->delete_all;
|
|
});
|
|
|
|
die unless $build;
|
|
|
|
# Do the build. If it throws an error, unlock the build so that it
|
|
# can be retried.
|
|
eval {
|
|
doBuild $build;
|
|
print "done\n";
|
|
};
|
|
if ($@) {
|
|
warn $@;
|
|
txn_do($db, sub {
|
|
$build->update({busy => 0, locker => $$});
|
|
});
|
|
}
|