ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

hydra-evaluator: Don't require $HYDRA_CONFIG

Browse source
This commit is contained in:
Eelco Dolstra 2013-01-22 13:19:28 +01:00
parent fc39034772
commit f188fe5683
3 changed files with 41 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/lib/Hydra/Helper/Nix.pm
View file

@ -9,7 +9,7 @@ use Hydra::Model::DB;
our @ISA = qw(Exporter); our @ISA = qw(Exporter);
our @EXPORT = qw( our @EXPORT = qw(
getHydraHome getHydraConf txn_do getHydraHome getHydraConfig txn_do
registerRoot getGCRootsDir gcRootFor registerRoot getGCRootsDir gcRootFor
getPrimaryBuildsForView getPrimaryBuildsForView
getPrimaryBuildTotal getPrimaryBuildTotal
@ -23,10 +23,11 @@ sub getHydraHome {
} }
sub getHydraConf { sub getHydraConfig {
my $conf = $ENV{"HYDRA_CONFIG"} || (Hydra::Model::DB::getHydraPath . "/hydra.conf"); my $conf = $ENV{"HYDRA_CONFIG"} || (Hydra::Model::DB::getHydraPath . "/hydra.conf");
die "The HYDRA_CONFIG file ($conf) does not exist!\n" unless -f $conf; return {} unless -f $conf;
return $conf; my %config = new Config::General($conf)->getall;
return \%config;
} }

32
src/script/hydra-build
View file

@ -24,21 +24,21 @@ STDOUT->autoflush();
my $db = Hydra::Model::DB->new(); my $db = Hydra::Model::DB->new();
my $config = getHydraConfig();
my %config = new Config::General(getHydraConf)->getall;
sub sendTwitterNotification { sub sendTwitterNotification {
my ($build) = @_; my ($build) = @_;
return unless (defined $ENV{'TWITTER_USER'} && defined $ENV{'TWITTER_PASS'}); return unless (defined $ENV{'TWITTER_USER'} && defined $ENV{'TWITTER_PASS'});
my $addURL = defined $config{'base_uri'}; my $addURL = defined $config->{'base_uri'};
my $jobName = $build->project->name . ":" . $build->jobset->name . ":" . $build->job->name; my $jobName = $build->project->name . ":" . $build->jobset->name . ":" . $build->job->name;
my $status = $build->buildstatus == 0 ? "SUCCEEDED" : "FAILED"; my $status = $build->buildstatus == 0 ? "SUCCEEDED" : "FAILED";
my $system = $build->system; my $system = $build->system;
my $duration = ($build->stoptime - $build->starttime) . " seconds"; my $duration = ($build->stoptime - $build->starttime) . " seconds";
my $url = $config{'base_uri'}."/build/".$build->id ; my $url = $config->{'base_uri'}."/build/".$build->id ;
my $nt = Net::Twitter::Lite->new( my $nt = Net::Twitter::Lite->new(
username => $ENV{'TWITTER_USER'}, username => $ENV{'TWITTER_USER'},
@ -83,14 +83,14 @@ sub sendEmailNotification {
my $prevBuild; my $prevBuild;
($prevBuild) = $db->resultset('Builds')->search( ($prevBuild) = $db->resultset('Builds')->search(
{ project => $build->project->name { project => $build->project->name
, jobset => $build->jobset->name , jobset => $build->jobset->name
, job => $build->job->name , job => $build->job->name
, system => $build->system , system => $build->system
, finished => 1 , finished => 1
, id => { '<', $build->id } , id => { '<', $build->id }
, -not => { buildstatus => { -in => [4, 3]} } , -not => { buildstatus => { -in => [4, 3]} }
}, { order_by => ["id DESC"] } }, { order_by => ["id DESC"] }
); );
# if build is cancelled or aborted, do not send email # if build is cancelled or aborted, do not send email
@ -112,10 +112,10 @@ sub sendEmailNotification {
my $status = statusDescription($build->buildstatus); my $status = statusDescription($build->buildstatus);
my $baseurl = hostname_long ; my $baseurl = hostname_long ;
my $sender = $config{'notification_sender'} || my $sender = $config->{'notification_sender'} ||
(($ENV{'USER'} || "hydra") . "@" . $baseurl); (($ENV{'USER'} || "hydra") . "@" . $baseurl);
my $selfURI = $config{'base_uri'} || "http://localhost:3000"; my $selfURI = $config->{'base_uri'} || "http://localhost:3000";
sub showTime { my ($x) = @_; return strftime('%Y-%m-%d %H:%M:%S', localtime($x)); } sub showTime { my ($x) = @_; return strftime('%Y-%m-%d %H:%M:%S', localtime($x)); }
@ -235,9 +235,9 @@ sub doBuild {
# Run Nix to perform the build, and monitor the stderr output # Run Nix to perform the build, and monitor the stderr output
# to get notifications about specific build steps, the # to get notifications about specific build steps, the
# associated log files, etc. # associated log files, etc.
# Note: `--timeout' was added in Nix 1.0pre27564, June 2011. # Note: `--timeout' was added in Nix 1.0pre27564, June 2011.
my $cmd = "nix-store --realise $drvPath " . my $cmd = "nix-store --realise $drvPath " .
"--timeout $timeout " . "--timeout $timeout " .
"--max-silent-time $maxsilent --keep-going --fallback " . "--max-silent-time $maxsilent --keep-going --fallback " .
"--no-build-output --log-type flat --print-build-trace " . "--no-build-output --log-type flat --print-build-trace " .
"--add-root " . gcRootFor $outPath . " 2>&1"; "--add-root " . gcRootFor $outPath . " 2>&1";
@ -259,7 +259,7 @@ sub doBuild {
} }
if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1; my $drvPathStep = $1;
txn_do($db, sub { txn_do($db, sub {
$build->buildsteps->create( $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++) { stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
@ -276,12 +276,12 @@ sub doBuild {
elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) { elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1; my $drvPathStep = $1;
my $machine = $2; my $machine = $2;
txn_do($db, sub { txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die; my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({machine => $machine}); $step->update({machine => $machine});
}); });
} }
elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) { elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1; my $drvPathStep = $1;
@ -389,8 +389,8 @@ sub doBuild {
my @closure = computeFSClosure(0, 0, $outPath); my @closure = computeFSClosure(0, 0, $outPath);
foreach my $path (@closure) { foreach my $path (@closure) {
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0); my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
$closuresize += $narSize; $closuresize += $narSize;
} }
} }
@ -446,8 +446,8 @@ if ($ENV{'HYDRA_TWITTER_TEST'}) {
my $build; my $build;
txn_do($db, sub { txn_do($db, sub {
$build = $db->resultset('Builds')->find($buildId); $build = $db->resultset('Builds')->find($buildId);
die "build $buildId doesn't exist" unless defined $build; die "build $buildId doesn't exist\n" unless defined $build;
die "build $buildId already done" if $build->finished; die "build $buildId already done\n" if $build->finished;
if ($build->busy != 0 && $build->locker != getppid) { if ($build->busy != 0 && $build->locker != getppid) {
die "build $buildId is already being built"; die "build $buildId is already being built";
} }

40
src/script/hydra-evaluator
View file

@ -18,17 +18,17 @@ use Data::Dump qw(dump);
STDOUT->autoflush(); STDOUT->autoflush();
my $db = Hydra::Model::DB->new(); my $db = Hydra::Model::DB->new();
my %config = new Config::General(getHydraConf)->getall; my $config = getHydraConfig();
sub fetchInputs { sub fetchInputs {
my ($project, $jobset, $inputInfo) = @_; my ($project, $jobset, $inputInfo) = @_;
foreach my $input ($jobset->jobsetinputs->all) { foreach my $input ($jobset->jobsetinputs->all) {
foreach my $alt ($input->jobsetinputalts->all) { foreach my $alt ($input->jobsetinputalts->all) {
my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value); my @info = fetchInput($db, $project, $jobset, $input->name, $input->type, $alt->value);
foreach my $info_el (@info) { foreach my $info_el (@info) {
push @{$$inputInfo{$input->name}}, $info_el if defined $info_el; push @{$$inputInfo{$input->name}}, $info_el if defined $info_el;
} }
} }
} }
} }
@ -41,7 +41,7 @@ sub setJobsetError {
$jobset->update({errormsg => $errorMsg, errortime => time}); $jobset->update({errormsg => $errorMsg, errortime => time});
}); });
}; };
sendJobsetErrorNotification($jobset, $errorMsg); sendJobsetErrorNotification($jobset, $errorMsg);
} }
@ -49,13 +49,13 @@ sub sendJobsetErrorNotification() {
my ($jobset, $errorMsg) = @_; my ($jobset, $errorMsg) = @_;
return if $jobset->project->owner->emailonerror == 0; return if $jobset->project->owner->emailonerror == 0;
return if $errorMsg eq ""; return if $errorMsg eq "";
my $url = hostname_long; my $url = hostname_long;
my $projectName = $jobset->project->name; my $projectName = $jobset->project->name;
my $jobsetName = $jobset->name; my $jobsetName = $jobset->name;
my $sender = $config{'notification_sender'} || my $sender = $config->{'notification_sender'} ||
(($ENV{'USER'} || "hydra") . "@" . $url); (($ENV{'USER'} || "hydra") . "@" . $url);
my $body = "Hi,\n" my $body = "Hi,\n"
@ -76,7 +76,7 @@ sub sendJobsetErrorNotification() {
'X-Hydra-Instance' => $url, 'X-Hydra-Instance' => $url,
'X-Hydra-Project' => $projectName, 'X-Hydra-Project' => $projectName,
'X-Hydra-Jobset' => $jobsetName 'X-Hydra-Jobset' => $jobsetName
], ],
body => "" body => ""
); );
$email->body_set($body); $email->body_set($body);
@ -90,7 +90,7 @@ sub sendJobsetErrorNotification() {
sub permute { sub permute {
my @list = @_; my @list = @_;
for (my $n = scalar @list - 1; $n > 0; $n--) { for (my $n = scalar @list - 1; $n > 0; $n--) {
my $k = int(rand($n + 1)); # 0 <= $k <= $n my $k = int(rand($n + 1)); # 0 <= $k <= $n
@list[$n, $k] = @list[$k, $n]; @list[$n, $k] = @list[$k, $n];
} }
return @list; return @list;
@ -101,7 +101,7 @@ sub checkJobset {
my ($project, $jobset) = @_; my ($project, $jobset) = @_;
my $inputInfo = {}; my $inputInfo = {};
my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix"; my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix";
# Fetch all values for all inputs. # Fetch all values for all inputs.
my $checkoutStart = time; my $checkoutStart = time;
fetchInputs($project, $jobset, $inputInfo); fetchInputs($project, $jobset, $inputInfo);
@ -120,7 +120,7 @@ sub checkJobset {
}); });
return; return;
} }
# Evaluate the job expression. # Evaluate the job expression.
my $evalStart = time; my $evalStart = time;
my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath);
@ -129,7 +129,7 @@ sub checkJobset {
my $jobOutPathMap = {}; my $jobOutPathMap = {};
txn_do($db, sub { txn_do($db, sub {
my $prevEval = getPrevJobsetEval($db, $jobset, 1); my $prevEval = getPrevJobsetEval($db, $jobset, 1);
# Clear the "current" flag on all builds. Since we're in a # Clear the "current" flag on all builds. Since we're in a
@ -151,7 +151,7 @@ sub checkJobset {
push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}}; push @{$failedJobNames{$_->{location}}}, $_->{msg} foreach @{$jobs->{error}};
$jobset->update({lastcheckedtime => time}); $jobset->update({lastcheckedtime => time});
$_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef }) $_->update({ errormsg => $failedJobNames{$_->name} ? join '\n', @{$failedJobNames{$_->name}} : undef })
foreach $jobset->jobs->all; foreach $jobset->jobs->all;
@ -173,7 +173,7 @@ sub checkJobset {
while (my ($id, $new) = each %buildIds) { while (my ($id, $new) = each %buildIds) {
$ev->jobsetevalmembers->create({ build => $id, isnew => $new }); $ev->jobsetevalmembers->create({ build => $id, isnew => $new });
} }
foreach my $name (keys %{$inputInfo}) { foreach my $name (keys %{$inputInfo}) {
for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) { for (my $n = 0; $n < scalar(@{$inputInfo->{$name}}); $n++) {
my $input = $inputInfo->{$name}->[$n]; my $input = $inputInfo->{$name}->[$n];
@ -190,7 +190,7 @@ sub checkJobset {
}); });
} }
} }
print STDERR " created new eval ", $ev->id, "\n"; print STDERR " created new eval ", $ev->id, "\n";
$ev->builds->update({iscurrent => 1}); $ev->builds->update({iscurrent => 1});
} else { } else {
@ -198,7 +198,7 @@ sub checkJobset {
$prevEval->builds->update({iscurrent => 1}) if defined $prevEval; $prevEval->builds->update({iscurrent => 1}) if defined $prevEval;
} }
}); });
# Store the error messages for jobs that failed to evaluate. # Store the error messages for jobs that failed to evaluate.
my $msg = ""; my $msg = "";
foreach my $error (@{$jobs->{error}}) { foreach my $error (@{$jobs->{error}}) {
@ -221,13 +221,13 @@ sub checkJobset {
sub checkJobsetWrapped { sub checkJobsetWrapped {
my ($project, $jobset) = @_; my ($project, $jobset) = @_;
print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n"; print STDERR "considering jobset ", $project->name, ":", $jobset->name, "\n";
eval { eval {
checkJobset($project, $jobset); checkJobset($project, $jobset);
}; };
if ($@) { if ($@) {
my $msg = $@; my $msg = $@;
print STDERR "error evaluating jobset ", $jobset->name, ": $msg"; print STDERR "error evaluating jobset ", $jobset->name, ": $msg";