hydra/src/script/hydra_queue_runner.pl

151 lines
5.3 KiB
Perl
Raw Normal View History

2008-11-28 16:13:06 +00:00
#! /var/run/current-system/sw/bin/perl -w
2008-11-10 13:33:12 +00:00
use strict;
2008-11-11 14:45:33 +00:00
use Cwd;
2008-11-28 14:36:04 +00:00
use File::Basename;
use POSIX qw(dup2 :sys_wait_h);
2008-11-25 11:09:15 +00:00
use Hydra::Schema;
2008-11-28 14:36:04 +00:00
use Hydra::Helper::Nix;
2008-11-10 13:33:12 +00:00
2008-11-28 14:36:04 +00:00
chdir getHydraPath or die;
my $db = openHydraDB;
2008-11-10 13:33:12 +00:00
2008-11-28 14:36:04 +00:00
my $hydraHome = $ENV{"HYDRA_HOME"};
die "The HYDRA_HOME environment variable is not set!\n" unless defined $hydraHome;
#$SIG{CHLD} = 'IGNORE';
2008-11-29 01:01:22 +00:00
2008-11-10 13:33:12 +00:00
2008-11-28 11:16:53 +00:00
sub unlockDeadBuilds {
# Unlock builds whose building process has died.
$db->txn_do(sub {
2008-11-28 11:16:53 +00:00
my @builds = $db->resultset('Builds')->search(
{finished => 0, busy => 1}, {join => 'schedulingInfo'});
2008-11-28 11:16:53 +00:00
foreach my $build (@builds) {
my $pid = $build->schedulingInfo->locker;
my $unlock = 0;
if ($pid == $$) {
# Work around sqlite locking timeouts: if the child
# barfed because of a locked DB before updating the
# `locker' field, then `locker' is still set to $$.
2008-11-29 01:26:51 +00:00
# So if after a minute it hasn't been updated,
# unlock the build. !!! need a better fix for those
# locking timeouts.
2008-11-29 01:26:51 +00:00
if ($build->schedulingInfo->starttime + 60 < time) {
$unlock = 1;
}
} elsif (kill(0, $pid) != 1) { # see if we can signal the process
$unlock = 1;
}
if ($unlock) {
2008-11-28 11:16:53 +00:00
print "build ", $build->id, " pid $pid died, unlocking\n";
$build->schedulingInfo->busy(0);
$build->schedulingInfo->locker("");
$build->schedulingInfo->update;
}
2008-11-10 13:33:12 +00:00
}
});
}
2008-11-10 13:33:12 +00:00
2008-11-28 11:16:53 +00:00
sub checkBuilds {
print "looking for runnable builds...\n";
2008-11-10 13:33:12 +00:00
2008-11-28 11:16:53 +00:00
my @buildsStarted;
2008-11-10 13:33:12 +00:00
$db->txn_do(sub {
# Get the system types for the runnable builds.
my @systemTypes = $db->resultset('Builds')->search(
{ finished => 0, busy => 0, enabled => 1, disabled => 0 },
{ join => ['schedulingInfo', 'project'], select => [{distinct => 'system'}], as => ['system'] });
# For each system type, select up to the maximum number of
# concurrent build for that system type. Choose the highest
# priority builds first, then the oldest builds.
foreach my $system (@systemTypes) {
# How many builds are already currently executing for this
# system type?
my $nrActive = $db->resultset('Builds')->search(
{finished => 0, busy => 1, system => $system->system},
{join => 'schedulingInfo'})->count;
# How many extra builds can we start?
(my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system});
my $maxConcurrent = defined $systemTypeInfo ? $systemTypeInfo->maxconcurrent : 2;
my $extraAllowed = $maxConcurrent - $nrActive;
$extraAllowed = 0 if $extraAllowed < 0;
# Select the highest-priority builds to start.
2008-11-28 11:16:53 +00:00
my @builds = $extraAllowed == 0 ? () : $db->resultset('Builds')->search(
{ finished => 0, busy => 0, system => $system->system, enabled => 1, disabled => 0 },
{ join => ['schedulingInfo', 'project'], order_by => ["priority DESC", "timestamp"],
rows => $extraAllowed });
print "system type `", $system->system,
"': $nrActive active, $maxConcurrent allowed, ",
2008-11-28 11:16:53 +00:00
"starting ", scalar(@builds), " builds\n";
2008-11-28 11:16:53 +00:00
foreach my $build (@builds) {
my $logfile = getcwd . "/logs/" . $build->id;
2008-11-28 14:36:04 +00:00
mkdir(dirname $logfile);
2008-11-27 19:06:11 +00:00
unlink($logfile);
2008-11-28 11:16:53 +00:00
$build->schedulingInfo->busy(1);
$build->schedulingInfo->locker($$);
$build->schedulingInfo->logfile($logfile);
$build->schedulingInfo->starttime(time);
$build->schedulingInfo->update;
$build->buildsteps->delete_all;
push @buildsStarted, $build;
}
2008-11-10 13:33:12 +00:00
}
});
# Actually start the builds we just selected. We need to do this
# outside the transaction in case it aborts or something.
2008-11-28 11:16:53 +00:00
foreach my $build (@buildsStarted) {
my $id = $build->id;
print "starting build $id (", $build->project->name, ":", $build->attrname, ") on ", $build->system, "\n";
2008-11-10 13:33:12 +00:00
eval {
2008-11-28 11:16:53 +00:00
my $logfile = $build->schedulingInfo->logfile;
my $child = fork();
die unless defined $child;
if ($child == 0) {
2008-11-28 14:36:04 +00:00
eval {
open LOG, ">$logfile" or die "cannot create logfile $logfile";
POSIX::dup2(fileno(LOG), 1) or die;
POSIX::dup2(fileno(LOG), 2) or die;
2008-11-28 16:13:06 +00:00
exec("hydra_build.pl", $id);
2008-11-28 14:36:04 +00:00
};
warn "cannot start build $id: $@";
POSIX::_exit(1);
}
2008-11-10 13:33:12 +00:00
};
if ($@) {
warn $@;
$db->txn_do(sub {
2008-11-28 11:16:53 +00:00
$build->schedulingInfo->busy(0);
$build->schedulingInfo->locker($$);
$build->schedulingInfo->update;
2008-11-10 13:33:12 +00:00
});
}
}
}
while (1) {
eval {
# Clean up zombies.
while ((waitpid(-1, &WNOHANG)) > 0) { };
2008-11-28 11:16:53 +00:00
unlockDeadBuilds;
2008-11-28 11:16:53 +00:00
checkBuilds;
};
warn $@ if $@;
2008-11-10 13:33:12 +00:00
print "sleeping...\n";
2008-11-12 11:09:21 +00:00
sleep(5);
2008-11-10 13:33:12 +00:00
}