ma27/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Remove hydra-build and the old hydra-queue-runner

Browse source
This commit is contained in:
Eelco Dolstra 2015-06-22 15:43:15 +02:00
parent 41ba7418e2
commit e32ee3d5b9
5 changed files with 3 additions and 674 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
doc/manual/installation.xml
View file

@ -100,13 +100,8 @@ nix-env -i hydra</screen>
</para>
<para>
Command completion should reveal a number of command-line tools from Hydra:
<screen>
hydra-build hydra-init hydra-update-gc-roots
hydra-eval-jobs hydra-queue-runner
hydra-evaluator hydra-server
</screen>
Command completion should reveal a number of command-line tools
from Hydra, such as <command>hydra-queue-runner</command>.
</para>
</section>

2
src/script/Makefile.am
View file

@ -4,9 +4,7 @@ EXTRA_DIST = \
distributable_scripts = \
hydra-init \
hydra-build \
hydra-evaluator \
hydra-queue-runner \
hydra-server \
hydra-update-gc-roots \
hydra-s3-backup-collect-garbage \

385
src/script/hydra-build
View file

@ -1,385 +0,0 @@
#! /var/run/current-system/sw/bin/perl
use strict;
use List::MoreUtils qw(all);
use File::Basename;
use File::stat;
use Nix::Store;
use Hydra::Plugin;
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Helper::PluginHooks;
use Hydra::Model::DB;
use Hydra::Helper::AddBuilds;
use Set::Scalar;
STDOUT->autoflush();
my $db = Hydra::Model::DB->new();
my $config = getHydraConfig();
my @plugins = Hydra::Plugin->instantiate(db => $db, config => $config);
sub addBuildStepOutputs {
my ($step) = @_;
my $drv = derivationFromPath($step->drvpath);
$step->buildstepoutputs->create({ name => $_, path => $drv->{outputs}->{$_} })
foreach keys %{$drv->{outputs}};
}
sub nextFreeStepNr {
my ($build) = @_;
my $max = $build->buildsteps->find(
{}, {select => {max => 'stepnr + 1'}, as => ['max']});
return (defined $max && defined $max->get_column('max')) ? $max->get_column('max') : 1;
}
sub failDependents {
my ($drvPath, $status, $errorMsg, $dependents, $startTime, $stopTime, $machine, $propagatedFrom) = @_;
# Get the referrer closure of $drvPath.
my $dependentDrvs = Set::Scalar->new(computeFSClosure(1, 0, $drvPath));
my $time = time();
txn_do($db, sub {
my @dependentBuilds = $db->resultset('Builds')->search(
{ finished => 0, busy => 0 },
{ columns => ["id", "project", "jobset", "job", "drvpath", "finished", "busy"] });
for my $d (@dependentBuilds) {
next unless $dependentDrvs->has($d->drvpath);
print STDERR "failing dependent build ", $d->id, " of ", $d->project->name, ":", $d->jobset->name, ":", $d->job->name, "\n";
$d->update(
{ finished => 1
, logfile => ''
, iscachedbuild => 0
, buildstatus => $drvPath eq $d->drvpath ? 1 : 2
, starttime => $time
, stoptime => $time
, errormsg => undef
});
my $step = $d->buildsteps->create(
{ stepnr => nextFreeStepNr($d)
, type => 0 # = build
, drvpath => $drvPath
, busy => 0
, status => $status
, starttime => $startTime
, stoptime => $stopTime
, errormsg => $errorMsg
, machine => $machine
, propagatedfrom => $propagatedFrom->id
});
addBuildStepOutputs($step);
push @$dependents, $d;
}
});
}
sub doBuild {
my ($build) = @_;
my %outputs;
$outputs{$_->name} = $_->path foreach $build->buildoutputs->all;
my $drvPath = $build->drvpath;
my $maxsilent = $build->maxsilent;
my $timeout = $build->timeout;
my $isCachedBuild = 1;
my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
my $startTime = time();
my $stopTime = undef;
my $buildStatus = 0; # = succeeded
my $errormsg = undef;
my $dependents = [];
if (!isValidPath($drvPath)) {
$buildStatus = 3;
$errormsg = "derivation was garbage-collected prior to build";
goto done;
}
unless (all { isValidPath($_) } values(%outputs)) {
$isCachedBuild = 0;
# Do the build.
my $thisBuildFailed = 0;
my $someBuildFailed = 0;
# Run Nix to perform the build, and monitor the stderr output
# to get notifications about specific build steps, the
# associated log files, etc.
my $cmd = "nix-store --realise $drvPath " .
"--timeout $timeout " .
"--max-silent-time $maxsilent " .
"--option build-max-log-size 67108864 " .
"--option print-missing false " .
"--keep-going --fallback " .
"--no-build-output --log-type flat --print-build-trace " .
"--add-root " . gcRootFor($outputs{out} // $outputs{(sort keys %outputs)[0]}) . " 2>&1";
my $buildStepNr = nextFreeStepNr($build);
my %buildSteps;
open OUT, "$cmd |" or die;
while (<OUT>) {
$errormsg .= $_;
unless (/^@\s+/) {
print STDERR "$_";
next;
}
if (/^@\s+build-started\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
, drvpath => $drvPathStep
, system => $3
, busy => 1
, starttime => time
});
addBuildStepOutputs($step);
});
}
elsif (/^@\s+build-remote\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
my $machine = $2;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({machine => $machine});
});
}
elsif (/^@\s+build-remote-start\s+(\S+)\s+/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({starttime => time});
});
}
elsif (/^@\s+build-remote-done\s+(\S+)\s+/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({stoptime => time});
});
}
elsif (/^@\s+build-succeeded\s+(\S+)\s+(\S+)$/) {
my $drvPathStep = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$step->update({busy => 0, status => 0, stoptime => time});
$step->update({stoptime => time}) unless defined $step->update;
});
}
elsif (/^@\s+build-failed\s+(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) {
my $drvPathStep = $1;
$someBuildFailed = 1;
$thisBuildFailed = 1 if $drvPath eq $drvPathStep;
my $errorMsg;
my $status = 1;
if ($3 eq "cached") {
$status = 8;
} elsif ($3 eq "timeout") {
$status = 7;
} else {
$errorMsg = $4;
}
my $now = time;
my $stepStartTime = $now;
my $stepStopTime = $now;
my $machine = "";
txn_do($db, sub {
if ($buildSteps{$drvPathStep}) {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$drvPathStep}}) or die;
$stepStartTime = $step->starttime;
$stepStopTime = $now;
$machine = $step->machine;
$step->update({busy => 0, status => $status, errormsg => $errorMsg, stoptime => $now});
}
# Don't write a record if this derivation already
# failed previously. This can happen if this is a
# restarted build.
elsif (scalar $build->buildsteps->search({drvpath => $drvPathStep, type => 0, busy => 0, status => 1}) == 0) {
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$drvPathStep} = $buildStepNr++)
, type => 0 # = build
, drvpath => $drvPathStep
, busy => 0
, status => $status
, starttime => $now
, stoptime => $now
, errormsg => $errorMsg
});
addBuildStepOutputs($step);
}
});
# Immediately fail all builds that depend on this derivation.
failDependents($drvPathStep, $status, $errorMsg, $dependents, $stepStartTime, $stepStopTime, $machine, $build);
}
elsif (/^@\s+substituter-started\s+(\S+)\s+(\S+)$/) {
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->create(
{ stepnr => ($buildSteps{$path} = $buildStepNr++)
, type => 1 # = substitution
, busy => 1
, starttime => time
});
# "out" is kinda fake (substitutions don't have named outputs).
$step->buildstepoutputs->create({ name => "out", path => $path });
});
}
elsif (/^@\s+substituter-succeeded\s+(\S+)$/) {
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
$step->update({busy => 0, status => 0, stoptime => time});
});
}
elsif (/^@\s+substituter-failed\s+(\S+)\s+(\S+)\s+(\S+)$/) {
my $path = $1;
txn_do($db, sub {
my $step = $build->buildsteps->find({stepnr => $buildSteps{$path}}) or die;
$step->update({busy => 0, status => 1, errormsg => $3, stoptime => time});
});
}
else {
print STDERR "unknown Nix trace message: $_";
}
}
close OUT;
my $res = $?;
$stopTime = time();
if ($res != 0) {
if ($thisBuildFailed) { $buildStatus = 1; }
elsif ($someBuildFailed) { $buildStatus = 2; }
else { $buildStatus = 3; }
}
# Only store the output of running Nix if we have a miscellaneous error.
$errormsg = undef unless $buildStatus == 3;
}
done:
txn_do($db, sub {
if ($buildStatus == 0) {
my $size = 0;
my $closureSize = 0;
my $releaseName;
my @closure = computeFSClosure(0, 0, values %outputs);
foreach my $path (@closure) {
my ($deriver, $hash, $time, $narSize, $refs) = queryPathInfo($path, 0);
$closureSize += $narSize;
$size += $narSize if grep { $path eq $_ } values(%outputs);
}
foreach my $path (values %outputs) {
$buildStatus = 6 if $buildStatus == 0 && -f "$path/nix-support/failed";
$releaseName //= getReleaseName($path);
}
$build->update(
{ releasename => $releaseName
, size => $size
, closuresize => $closureSize
});
addBuildProducts($db, $build);
}
# Mark any remaining active build steps as aborted.
$build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
$build->update(
{ finished => 1
, busy => 0
, locker => ''
, logfile => ''
, iscachedbuild => $isCachedBuild
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime // time()
, errormsg => $errormsg
});
});
notifyBuildFinished(\@plugins, $build, $dependents);
}
my $buildId = $ARGV[0] or die "syntax: $0 BUILD-ID\n";
print STDERR "performing build $buildId\n";
if ($ENV{'HYDRA_MAIL_TEST'}) {
my $build = $db->resultset('Builds')->find($buildId);
notifyBuildFinished(\@plugins, $build, []);
exit 0;
}
# Lock the build. If necessary, steal the lock from the parent
# process (runner.pl). This is so that if the runner dies, the
# children (i.e. the build.pl instances) can continue to run and won't
# have the lock taken away.
my $build;
txn_do($db, sub {
$build = $db->resultset('Builds')->find($buildId);
die "build $buildId doesn't exist\n" unless defined $build;
die "build $buildId already done\n" if $build->finished;
if ($build->busy != 0 && $build->locker != getppid) {
die "build $buildId is already being built";
}
$build->update({busy => 1, locker => $$});
$build->buildsteps->search({busy => 1})->delete;
$build->buildproducts->delete;
});
die unless $build;
# Do the build. If it throws an error, unlock the build so that it
# can be retried.
eval {
doBuild $build;
print "done\n";
};
if ($@) {
warn $@;
txn_do($db, sub {
$build->update({busy => 0, locker => $$});
});
}

2
src/script/hydra-eval-guile-jobs.in
View file

@ -104,7 +104,7 @@ symbol/thunk pairs."
(when gc-roots-dir
;; Register DRV as a GC root so that it's not collected by
;; the time 'hydra-build' attempts to build it.
;; the time 'hydra-queue-runner' attempts to build it.
(register-gc-root drv gc-roots-dir))
;; XXX: Add <arg ...> tags?

279
src/script/hydra-queue-runner
View file

@ -1,279 +0,0 @@
#! /var/run/current-system/sw/bin/perl
use strict;
use Cwd;
use File::Basename;
use POSIX qw(dup2 :sys_wait_h);
use Hydra::Schema;
use Hydra::Helper::Nix;
use Hydra::Model::DB;
use IO::Handle;
use Nix::Store;
use Set::Scalar;
chdir Hydra::Model::DB::getHydraPath or die;
my $db = Hydra::Model::DB->new();
STDOUT->autoflush();
my $lastTime;
#$SIG{CHLD} = 'IGNORE';
sub unlockDeadBuilds {
# Unlock builds whose building process has died.
txn_do($db, sub {
my @builds = $db->resultset('Builds')->search({finished => 0, busy => 1});
foreach my $build (@builds) {
my $pid = $build->locker;
my $unlock = 0;
if ($pid == $$) {
if (!defined $lastTime || $build->starttime < $lastTime - 300) {
$unlock = 1;
}
} elsif (kill(0, $pid) != 1) { # see if we can signal the process
$unlock = 1;
}
if ($unlock) {
print "build ", $build->id, " pid $pid died, unlocking\n";
$build->update({ busy => 0, locker => "" });
$build->buildsteps->search({ busy => 1 })->update({ busy => 0, status => 4, stoptime => time });
}
}
});
}
# Given a build, return an arbitrary queued build on which this build
# depends; or undef if no such build exists.
sub findBuildDependencyInQueue {
my ($buildsByDrv, $build) = @_;
return undef unless isValidPath($build->drvpath);
my @deps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(0, 0, $build->drvpath);
return unless scalar @deps > 0;
foreach my $d (@deps) {
my $bs = $buildsByDrv->{$d};
next unless defined $bs;
return $db->resultset('Builds')->find((@$bs)[0]);
}
return undef;
}
sub blockBuilds {
my ($buildsByDrv, $blockedBuilds, $build) = @_;
my @rdeps = grep { /\.drv$/ && $_ ne $build->drvpath } computeFSClosure(1, 0, $build->drvpath);
foreach my $drv (@rdeps) {
my $bs = $buildsByDrv->{$drv};
next if !defined $bs;
$blockedBuilds->insert($_) foreach @$bs;
}
}
sub checkBuilds {
# print "looking for runnable builds...\n";
my @buildsStarted;
my $machines = getMachines;
my %maxConcurrent;
foreach my $machineName (keys %{$machines}) {
foreach my $system (@{${$machines}{$machineName}{'systemTypes'}}) {
$maxConcurrent{$system} = (${$machines}{$machineName}{'maxJobs'} or 0) + ($maxConcurrent{$system} or 0)
}
}
txn_do($db, sub {
# Cache scheduled builds by derivation path to speed up
# findBuildDependencyInQueue.
my $buildsByDrv = {};
push @{$buildsByDrv->{$_->drvpath}}, $_->id
foreach $db->resultset('Builds')->search({ finished => 0 });
# Builds in the queue of which a dependency is already building.
my $blockedBuilds = Set::Scalar->new();
blockBuilds($buildsByDrv, $blockedBuilds, $_)
foreach $db->resultset('Builds')->search({ finished => 0, busy => 1 });
# Get the system types for the runnable builds.
my @systemTypes = $db->resultset('Builds')->search(
{ finished => 0, busy => 0 },
{ join => ['project'], select => ['system'], as => ['system'], distinct => 1 });
# Get the total number of scheduling shares.
my $totalShares = getTotalShares($db) || 1;
# For each system type, select up to the maximum number of
# concurrent build for that system type.
foreach my $system (@systemTypes) {
# How many builds are already currently executing for this
# system type?
my $nrActive = $db->resultset('Builds')->search(
{finished => 0, busy => 1, system => $system->system})->count;
(my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system});
my $max = defined $systemTypeInfo ? $systemTypeInfo->maxconcurrent : $maxConcurrent{$system->system} // 2;
my $extraAllowed = $max - $nrActive;
next if $extraAllowed <= 0;
print STDERR "starting at most $extraAllowed builds for system ${\$system->system}\n";
my $timeSpentPerJobset;
j: while ($extraAllowed-- > 0) {
my @runnableJobsets = $db->resultset('Builds')->search(
{ finished => 0, busy => 0, system => $system->system },
{ select => ['project', 'jobset'], distinct => 1 });
next if @runnableJobsets == 0;
my $windowSize = 24 * 3600;
my $costPerBuild = 30;
my $totalWindowSize = $windowSize * $max;
my @res;
foreach my $b (@runnableJobsets) {
my $jobset = $db->resultset('Jobsets')->find($b->get_column('project'), $b->get_column('jobset')) or die;
my $timeSpent = $timeSpentPerJobset->{$b->get_column('project')}->{$b->get_column('jobset')};
if (!defined $timeSpent) {
$timeSpent = $jobset->builds->search(
{ },
{ where => \ ("(finished = 0)")
, join => 'buildsteps'
, select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
, as => "sum" })->single->get_column("sum") // 0;
$timeSpent += $jobset->builds->search(
{ },
{ where => \ ("(me.stoptime >= " . (time() - $windowSize) . ")")
, join => 'buildsteps'
, select => \ "sum(coalesce(buildsteps.stoptime, ${\time}) - buildsteps.starttime)"
, as => "sum" })->single->get_column("sum") // 0;
# Add a 30s penalty for each started build. This
# is to account for jobsets that have running
# builds but no build steps yet.
$timeSpent += $jobset->builds->search({ finished => 0, busy => 1 })->count * $costPerBuild;
$timeSpentPerJobset->{$b->get_column('project')}->{$b->get_column('jobset')} = $timeSpent;
}
my $share = $jobset->schedulingshares || 1; # prevent division by zero
my $used = $timeSpent / ($totalWindowSize * ($share / $totalShares));
#printf STDERR "%s:%s: %d s, total used = %.2f%%, share used = %.2f%%\n", $jobset->get_column('project'), $jobset->name, $timeSpent, $timeSpent / $totalWindowSize * 100, $used * 100;
push @res, { jobset => $jobset, used => $used };
}
foreach my $r (sort { $a->{used} <=> $b->{used} } @res) {
my $jobset = $r->{jobset};
#print STDERR "selected ", $jobset->get_column('project'), ':', $jobset->name, "\n";
# Select the highest-priority build for this jobset.
my @builds = $jobset->builds->search(
{ finished => 0, busy => 0, system => $system->system },
{ order_by => ["priority DESC", "id"] });
foreach my $build (@builds) {
next if $blockedBuilds->has($build->id);
# Find a dependency of $build that has no queued
# dependencies itself. This isn't strictly necessary,
# but it ensures that Nix builds are done as part of
# their corresponding Hydra builds, rather than as a
# dependency of some other Hydra build.
while (my $dep = findBuildDependencyInQueue($buildsByDrv, $build)) {
$build = $dep;
}
next if $build->busy;
printf STDERR "starting build %d (%s:%s:%s) on %s; jobset at %.2f%% of its share\n",
$build->id, $build->project->name, $build->jobset->name, $build->job->name, $build->system, $r->{used} * 100;
my $logfile = getcwd . "/logs/" . $build->id;
mkdir(dirname $logfile);
unlink($logfile);
$build->update(
{ busy => 1
, locker => $$
, logfile => $logfile
});
push @buildsStarted, $build;
$timeSpentPerJobset->{$jobset->get_column('project')}->{$jobset->name} += $costPerBuild;
blockBuilds($buildsByDrv, $blockedBuilds, $build);
next j;
}
}
last; # nothing found, give up on this system type
}
}
$lastTime = time();
$_->update({ starttime => time() }) foreach @buildsStarted;
});
# Actually start the builds we just selected. We need to do this
# outside the transaction in case it aborts or something.
foreach my $build (@buildsStarted) {
my $id = $build->id;
eval {
my $logfile = $build->logfile;
my $child = fork();
die unless defined $child;
if ($child == 0) {
eval {
open LOG, ">$logfile" or die "cannot create logfile $logfile";
POSIX::dup2(fileno(LOG), 1) or die;
POSIX::dup2(fileno(LOG), 2) or die;
exec("hydra-build", $id);
};
warn "cannot start build $id: $@";
POSIX::_exit(1);
}
};
if ($@) {
warn $@;
txn_do($db, sub {
$build->update({ busy => 0, locker => $$ });
});
}
}
}
if (scalar(@ARGV) == 1 && $ARGV[0] eq "--unlock") {
unlockDeadBuilds;
exit 0;
}
while (1) {
eval {
# Clean up zombies.
while ((waitpid(-1, &WNOHANG)) > 0) { };
unlockDeadBuilds;
checkBuilds;
};
warn $@ if $@;
# print "sleeping...\n";
sleep(5);
}