hydra/src/script/hydra_build.pl

497 lines
16 KiB
Perl
Raw Normal View History

2008-11-28 16:13:06 +00:00
#! /var/run/current-system/sw/bin/perl -w
2008-11-10 13:33:12 +00:00
use strict;
use File::Basename;
2008-11-12 14:29:32 +00:00
use File::stat;
2008-11-25 11:09:15 +00:00
use Hydra::Schema;
2008-11-28 14:36:04 +00:00
use Hydra::Helper::Nix;
2009-07-07 16:15:38 +00:00
use Email::Sender::Simple qw(sendmail);
use Email::Sender::Transport::SMTP;
use Email::Simple;
use Email::Simple::Creator;
2009-07-08 15:52:55 +00:00
use Sys::Hostname::Long;
use Config::General;
use Text::Table;
use POSIX qw(strftime);
use Net::Twitter::Lite;
use Data::Dump qw(dump);
use Switch;
2008-11-10 13:33:12 +00:00
2009-04-22 22:59:54 +00:00
STDOUT->autoflush();
2008-11-28 14:36:04 +00:00
my $db = openHydraDB;
2008-11-10 13:33:12 +00:00
2009-07-10 14:43:21 +00:00
my %config = new Config::General($ENV{"HYDRA_CONFIG"})->getall;
2009-03-26 12:53:39 +00:00
sub getBuildLog {
my ($drvPath) = @_;
my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
return -e $logPath ? $logPath : undef;
}
sub sendTwitterNotification {
my ($build) = @_;
return unless (defined $ENV{'TWITTER_USER'} && defined $ENV{'TWITTER_PASS'});
my $addURL = defined $ENV{'HYDRA_BUILD_BASEURL'};
my $jobName = $build->project->name . ":" . $build->jobset->name . ":" . $build->job->name;
my $status = $build->resultInfo->buildstatus == 0 ? "SUCCEEDED" : "FAILED";
my $system = $build->system;
my $duration = ($build->resultInfo->stoptime - $build->resultInfo->starttime) . " seconds";
my $url = $ENV{'HYDRA_BUILD_BASEURL'}."/".$build->id ;
my $nt = Net::Twitter::Lite->new(
username => $ENV{'TWITTER_USER'},
2009-10-15 13:35:52 +00:00
password => $ENV{'TWITTER_PASS'},
clientname => "Hydra Build Daemon"
);
my $tag = $build->project->name;
my $msg = "$jobName ($system): $status in $duration #$tag";
if (length($msg) + 1 + length($url) <= 140) {
$msg = "$msg $url" ;
}
eval {
my $result = eval { $nt->update($msg) };
};
warn "$@\n" if $@;
}
2009-03-26 12:53:39 +00:00
sub statusDescription {
my ($buildstatus) = @_;
my $status = "Unknown failure";
switch ($buildstatus) {
case 0 { $status = "Success"; }
case 1 { $status = "Failed with non-zero exit code"; }
case 2 { $status = "Dependency failed"; }
case 4 { $status = "Cancelled"; }
}
return $status;
}
2009-07-07 16:15:38 +00:00
sub sendEmailNotification {
my ($build) = @_;
2009-07-08 15:52:55 +00:00
die unless defined $build->resultInfo;
return if !($build->maintainers || $build->jobset->enableemail);
2009-07-07 16:15:38 +00:00
2009-07-08 15:52:55 +00:00
# Do we want to send mail?
my $prevBuild;
($prevBuild) = $db->resultset('Builds')->search(
{ project => $build->project->name
, jobset => $build->jobset->name
, job => $build->job->name
, system => $build->system
, finished => 1
, id => { '!=', $build->id }
}, { order_by => ["timestamp DESC"] }
);
if (defined $prevBuild && ($build->resultInfo->buildstatus == $prevBuild->resultInfo->buildstatus)) {
return;
2009-07-08 15:52:55 +00:00
}
# Send mail.
# !!! should use the Template Toolkit here.
print STDERR "sending mail notification to ", $build->maintainers, "\n";
my $jobName = $build->project->name . ":" . $build->jobset->name . ":" . $build->job->name;
my $status = statusDescription($build->resultInfo->buildstatus);
2009-07-08 15:52:55 +00:00
my $url = hostname_long ;
my $sender = $config{'notification_sender'} ||
(($ENV{'USER'} || "hydra") . "@" . $url);
2009-07-08 15:52:55 +00:00
my $selfURI = $config{'base_uri'} || "http://localhost:3000";
2009-07-08 15:52:55 +00:00
sub showTime { my ($x) = @_; return strftime('%Y-%m-%d %H:%M:%S', localtime($x)); }
2009-08-12 13:50:59 +00:00
my $infoTable = Text::Table->new({ align => "left" }, \ " | ", { align => "left" });
my @lines = (
[ "Build ID:", $build->id ],
[ "Nix name:", $build->nixname ],
[ "Short description:", $build->description || '(not given)' ],
[ "Maintainer(s):", $build->maintainers ],
[ "System:", $build->system ],
[ "Derivation store path:", $build->drvpath ],
[ "Output store path:", $build->outpath ],
[ "Time added:", showTime $build->timestamp ],
);
push @lines, (
[ "Build started:", showTime $build->resultInfo->starttime ],
[ "Build finished:", showTime $build->resultInfo->stoptime ],
[ "Duration:", $build->resultInfo->stoptime - $build->resultInfo->starttime . "s" ],
) if $build->resultInfo->starttime;
$infoTable->load(@lines);
my $inputsTable = Text::Table->new(
2009-08-12 13:50:59 +00:00
{ title => "Name", align => "left" }, \ " | ",
{ title => "Type", align => "left" }, \ " | ",
{ title => "Value", align => "left" });
@lines = ();
foreach my $input ($build->inputs) {
my $type = $input->type;
push @lines,
[ $input->name
, $input->type
, ( $input->type eq "build" || $input->type eq "sysbuild")
? $input->dependency->id
: ($input->type eq "string" || $input->type eq "boolean")
? $input->value : ($input->uri . ':' . $input->revision)
];
}
$inputsTable->load(@lines);
my $loglines = 50;
my $logfile = $build->resultInfo->logfile;
my $logtext = defined $logfile && -e $logfile ? `tail -$loglines $logfile` : "No logfile available.\n";
2009-07-08 15:52:55 +00:00
my $body = "Hi,\n"
. "\n"
2010-02-10 12:27:16 +00:00
. "This is to let you know that Hydra build " . $build->id
. " of job " . $jobName . " " . (defined $prevBuild ? "has changed from '".statusDescription($prevBuild->resultInfo->buildstatus)."' to '$status'" : "has succeeded" ) .".\n"
2009-07-08 15:52:55 +00:00
. "\n"
2009-08-12 13:50:59 +00:00
. "Complete build information can be found on this page: "
2009-07-08 15:52:55 +00:00
. "$selfURI/build/" . $build->id . "\n"
. ($build->resultInfo->buildstatus != 0 ? "\nThe last $loglines lines of the build log are shown at the bottom of this email.\n" : "")
2009-07-08 15:52:55 +00:00
. "\n"
. "A summary of the build information follows:\n"
. "\n"
. $infoTable->body
. "\n"
. "The build inputs were:\n"
. "\n"
. $inputsTable->title
. $inputsTable->rule('-', '+')
. $inputsTable->body
. "\n"
. "Regards,\n\nThe Hydra build daemon.\n"
. ($build->resultInfo->buildstatus != 0 ? "\n---\n$logtext" : "");
# stripping trailing spaces from lines
$body =~ s/[\ ]+$//gm;
2009-07-08 15:52:55 +00:00
my $to = (!$build->jobset->emailoverride eq "") ? $build->jobset->emailoverride : $build->maintainers;
2009-07-07 16:15:38 +00:00
my $email = Email::Simple->create(
header => [
To => $to,
2009-07-08 15:52:55 +00:00
From => "Hydra Build Daemon <$sender>",
Subject => "Hydra job $jobName build " . $build->id . ": $status",
'X-Hydra-Instance' => $url,
'X-Hydra-Project' => $build->project->name,
'X-Hydra-Jobset' => $build->jobset->name,
'X-Hydra-Job' => $build->job->name
2009-07-07 16:15:38 +00:00
],
body => "",
2009-07-07 16:15:38 +00:00
);
$email->body_set($body);
2009-07-07 16:15:38 +00:00
2009-07-08 15:52:55 +00:00
print $email->as_string if $ENV{'HYDRA_MAIL_TEST'};
2009-07-07 16:15:38 +00:00
sendmail($email);
2009-07-07 16:15:38 +00:00
}
2008-11-11 12:54:37 +00:00
sub doBuild {
my ($build) = @_;
2008-11-10 13:33:12 +00:00
2008-11-11 12:54:37 +00:00
my $drvPath = $build->drvpath;
my $outPath = $build->outpath;
2008-11-10 13:33:12 +00:00
my $isCachedBuild = 1;
my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
my $startTime = 0;
my $stopTime = 0;
my $buildStatus = 0; # = succeeded
my $errormsg = undef;
2008-11-10 13:33:12 +00:00
if (!isValidPath($outPath)) {
$isCachedBuild = 0;
2009-03-09 16:22:41 +00:00
# Do the build.
2008-11-10 13:33:12 +00:00
$startTime = time();
my $thisBuildFailed = 0;
my $someBuildFailed = 0;
# Run Nix to perform the build, and monitor the stderr output
# to get notifications about specific build steps, the
# associated log files, etc.
my $cmd = "nix-store --realise $drvPath " .
"--max-silent-time 3600 --keep-going --fallback " .
"--no-build-output --log-type flat --print-build-trace " .
2009-03-15 11:56:11 +00:00
"--add-root " . gcRootFor $outPath . " 2>&1";
my $max = $build->buildsteps->find(
{}, {select => {max => 'stepnr + 1'}, as => ['max']});
my $buildStepNr = defined $max ? $max->get_column('max') : 1;
my %buildSteps;
open OUT, "$cmd |" or die;
while (<OUT>) {
$errormsg .= $_;
unless (/^@\s+/) {
print STDERR "$_";
next;
}
if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
2009-03-16 17:46:46 +00:00
my $drvPathStep = $1;
txn_do($db, sub {
$build->buildsteps->create(
2009-03-16 17:46:46 +00:00
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
2009-03-16 17:46:46 +00:00
, drvpath => $drvPathStep
, outpath => $2
, logfile => $4
, busy => 1
2008-11-12 11:09:21 +00:00
, starttime => time
});
});
}
2008-11-12 13:00:56 +00:00
elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
2009-03-16 17:46:46 +00:00
my $drvPathStep = $1;
txn_do($db, sub {
2009-03-16 17:46:46 +00:00
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
2009-03-10 08:52:42 +00:00
$step->update({busy => 0, status => 0, stoptime => time});
});
}
2008-11-12 13:00:56 +00:00
elsif (/^@\s+build-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
my $drvPathStep = $1;
$someBuildFailed = 1;
$thisBuildFailed = 1 if $drvPath eq $drvPathStep;
2009-03-26 12:53:39 +00:00
my $errorMsg = $4;
$errorMsg = "build failed previously (cached)" if $3 eq "cached";
txn_do($db, sub {
if ($buildSteps{$drvPathStep}) {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
2009-03-26 12:53:39 +00:00
$step->update({busy => 0, status => 1, errormsg => $errorMsg, stoptime => time});
}
# Don't write a record if this derivation already
# failed previously. This can happen if this is a
# restarted build.
elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
$build->buildsteps->create(
2009-03-26 12:53:39 +00:00
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
2008-11-12 11:09:21 +00:00
, type => 0 # = build
, drvpath => $drvPathStep
2008-11-12 11:09:21 +00:00
, outpath => $2
2009-03-26 12:53:39 +00:00
, logfile => getBuildLog($drvPathStep)
2008-11-12 11:09:21 +00:00
, busy => 0
, status => 1
, starttime => time
, stoptime => time
2009-03-26 12:53:39 +00:00
, errormsg => $errorMsg
2008-11-12 11:09:21 +00:00
});
}
});
}
2008-11-12 13:00:56 +00:00
elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
my $outPath = $1;
txn_do($db, sub {
$build->buildsteps->create(
{ stepnr => ($buildSteps{$outPath} = $buildStepNr++)
2008-11-12 13:00:56 +00:00
, type => 1 # = substitution
, outpath => $1
, busy => 1
, starttime => time
});
});
}
elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
my $outPath = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die;
2009-03-09 16:22:41 +00:00
$step->update({busy => 0, status => 0, stoptime => time});
2008-11-12 13:00:56 +00:00
});
}
elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $outPath = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$outPath}}) or die;
2009-03-09 16:22:41 +00:00
$step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
2008-11-12 13:00:56 +00:00
});
}
else {
print STDERR "unknown Nix trace message: $_";
}
}
close OUT;
my $res = $?;
2008-11-10 13:33:12 +00:00
$stopTime = time();
if ($res != 0) {
if ($thisBuildFailed) { $buildStatus = 1; }
elsif ($someBuildFailed) { $buildStatus = 2; }
else { $buildStatus = 3; }
}
2008-11-10 13:33:12 +00:00
# Only store the output of running Nix if we have a miscellaneous error.
$errormsg = undef unless $buildStatus == 3;
2008-11-10 13:33:12 +00:00
}
done:
txn_do($db, sub {
$build->update({finished => 1, timestamp => time});
2008-11-11 12:54:37 +00:00
2008-11-25 16:13:22 +00:00
my $releaseName;
if (-e "$outPath/nix-support/hydra-release-name") {
open FILE, "$outPath/nix-support/hydra-release-name" or die;
$releaseName = <FILE>;
chomp $releaseName;
close FILE;
}
$db->resultset('BuildResultInfo')->create(
2008-11-11 12:54:37 +00:00
{ id => $build->id
2008-11-10 13:33:12 +00:00
, iscachedbuild => $isCachedBuild
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime
, logfile => getBuildLog($drvPath)
, errormsg => $errormsg
2008-11-25 16:13:22 +00:00
, releasename => $releaseName
2008-11-10 13:33:12 +00:00
});
if ($buildStatus == 0) {
2008-11-10 13:33:12 +00:00
2008-11-12 14:29:32 +00:00
my $productnr = 1;
2008-11-10 13:33:12 +00:00
if (-e "$outPath/nix-support/hydra-build-products") {
open LIST, "$outPath/nix-support/hydra-build-products" or die;
while (<LIST>) {
/^([\w\-]+)\s+([\w\-]+)\s+(\S+)(\s+(\S+))?$/ or next;
2008-11-10 13:33:12 +00:00
my $type = $1;
2008-11-12 16:42:07 +00:00
my $subtype = $2 eq "none" ? "" : $2;
2008-11-10 13:33:12 +00:00
my $path = $3;
my $defaultPath = $5;
2008-11-29 01:27:17 +00:00
next unless -e $path;
2008-11-12 14:29:32 +00:00
my $fileSize, my $sha1, my $sha256;
# !!! validate $path, $defaultPath
if (-f $path) {
my $st = stat($path) or die "cannot stat $path: $!";
2008-11-12 16:42:07 +00:00
$fileSize = $st->size;
$sha1 = `nix-hash --flat --type sha1 $path`
or die "cannot hash $path: $?";;
chomp $sha1;
2008-11-12 14:29:32 +00:00
$sha256 = `nix-hash --flat --type sha256 $path`
or die "cannot hash $path: $?";;
chomp $sha256;
}
my $name = $path eq $outPath ? "" : basename $path;
2008-11-12 14:29:32 +00:00
$db->resultset('BuildProducts')->create(
2008-11-10 13:33:12 +00:00
{ build => $build->id
2008-11-12 14:29:32 +00:00
, productnr => $productnr++
2008-11-10 13:33:12 +00:00
, type => $type
, subtype => $subtype
, path => $path
, filesize => $fileSize
2008-11-12 14:29:32 +00:00
, sha1hash => $sha1
, sha256hash => $sha256
, name => $name
, defaultpath => $defaultPath
2008-11-10 13:33:12 +00:00
});
}
close LIST;
2008-11-12 14:29:32 +00:00
}
else {
$db->resultset('BuildProducts')->create(
2008-11-10 13:33:12 +00:00
{ build => $build->id
2008-11-12 14:29:32 +00:00
, productnr => $productnr++
2008-11-10 13:33:12 +00:00
, type => "nix-build"
, subtype => ""
, path => $outPath
2008-11-12 16:42:07 +00:00
, name => $build->nixname
2008-11-10 13:33:12 +00:00
});
}
}
2008-11-11 12:54:37 +00:00
$build->schedulingInfo->delete;
2008-11-10 13:33:12 +00:00
});
2009-07-07 16:15:38 +00:00
2009-07-08 15:52:55 +00:00
sendEmailNotification $build;
sendTwitterNotification $build;
2008-11-10 13:33:12 +00:00
}
2008-11-11 12:54:37 +00:00
my $buildId = $ARGV[0] or die;
print STDERR "performing build $buildId\n";
2008-11-10 13:33:12 +00:00
2009-07-08 15:52:55 +00:00
if ($ENV{'HYDRA_MAIL_TEST'}) {
sendEmailNotification $db->resultset('Builds')->find($buildId);
exit 0;
}
if ($ENV{'HYDRA_TWITTER_TEST'}) {
sendTwitterNotification $db->resultset('Builds')->find($buildId);
exit 0;
}
2009-07-07 16:15:38 +00:00
2008-11-11 12:54:37 +00:00
# Lock the build. If necessary, steal the lock from the parent
# process (runner.pl). This is so that if the runner dies, the
# children (i.e. the build.pl instances) can continue to run and won't
# have the lock taken away.
my $build;
txn_do($db, sub {
$build = $db->resultset('Builds')->find($buildId);
2008-11-11 12:54:37 +00:00
die "build $buildId doesn't exist" unless defined $build;
die "build $buildId already done" if defined $build->resultInfo;
2008-11-11 12:54:37 +00:00
if ($build->schedulingInfo->busy != 0 && $build->schedulingInfo->locker != getppid) {
die "build $buildId is already being built";
2008-11-10 13:33:12 +00:00
}
2009-03-09 16:22:41 +00:00
$build->schedulingInfo->update({busy => 1, locker => $$});
$build->buildsteps->search({busy => 1})->delete_all;
$build->buildproducts->delete_all;
2008-11-10 13:33:12 +00:00
});
2008-11-11 12:54:37 +00:00
die unless $build;
2008-11-10 13:33:12 +00:00
2008-11-11 12:54:37 +00:00
# Do the build. If it throws an error, unlock the build so that it
# can be retried.
2008-11-10 13:33:12 +00:00
eval {
2008-11-11 12:54:37 +00:00
doBuild $build;
2008-11-12 14:29:32 +00:00
print "done\n";
2008-11-10 13:33:12 +00:00
};
if ($@) {
warn $@;
txn_do($db, sub {
2009-03-09 16:22:41 +00:00
$build->schedulingInfo->update({busy => 0, locker => $$});
2008-11-10 13:33:12 +00:00
});
}