snaakey/hydra
0
0
Fork 0
forked from lix-project/hydra
Code Pull requests Activity

Browse source
This commit is contained in:
Eelco Dolstra 2008-11-10 13:33:12 +00:00
parent 15edf10986
commit 8f5e7c319c
17 changed files with 366 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/HydraFrontend/lib/HydraFrontend/Schema.pm
View file

@ -8,8 +8,8 @@ use base 'DBIx::Class::Schema';
__PACKAGE__->load_classes;
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:xP97YDrN7Bm2B/BlbQJ7fQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dBO/r6lVlITiJ/HlltKcpQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/HydraFrontend/lib/HydraFrontend/Schema/Buildlogs.pm
View file

@ -21,8 +21,8 @@ __PACKAGE__->set_primary_key("build", "logphase");
__PACKAGE__->belongs_to("build", "HydraFrontend::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:pt0CJFX1pP9Z2TjqrTjTkw
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:eMNna7u2l0ec+OYuvtGRpg
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/HydraFrontend/lib/HydraFrontend/Schema/Buildproducts.pm
View file

@ -21,8 +21,8 @@ __PACKAGE__->set_primary_key("build", "path");
__PACKAGE__->belongs_to("build", "HydraFrontend::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3NKUaF4u4H6ZmIRCeva8yA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LaXQ4zxxvzdKFBRVcjMdMQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/HydraFrontend/lib/HydraFrontend/Schema/Builds.pm
View file

@ -65,8 +65,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:8s5Z03ugocOVb021EwGVag
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C1XPkCXQImyXduKER0Dllg
__PACKAGE__->has_many(dependents => 'HydraFrontend::Schema::Inputs', 'dependency');

4
src/HydraFrontend/lib/HydraFrontend/Schema/Inputs.pm
View file

@ -35,8 +35,8 @@ __PACKAGE__->set_primary_key("id");
__PACKAGE__->belongs_to("build", "HydraFrontend::Schema::Builds", { id => "build" });
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:AzV6B/6CCrroPlO32n2p3A
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:A3Is4VTFkTl2DzrYjzdrZA
__PACKAGE__->belongs_to("dependency", "HydraFrontend::Schema::Builds", { id => "dependency" });

10
src/HydraFrontend/lib/HydraFrontend/Schema/Jobs.pm
View file

@ -46,9 +46,13 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:GubRofAmJ/sbJbjyV3aKSQ
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZF8UB1MtbPuOk7wTSFJR5Q
__PACKAGE__->has_many(
"inputs",
"HydraFrontend::Schema::Inputs",
{ "foreign.job" => "self.id" },
);
# You can replace this text with custom content, and it will be preserved on regeneration
1;

4
src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputalts.pm
View file

@ -33,8 +33,8 @@ __PACKAGE__->belongs_to(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ZjjWLbAWExxOqsDz41A3KA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ibTncC1AslPWt1eiTtwplA
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/HydraFrontend/lib/HydraFrontend/Schema/Jobsetinputs.pm
View file

@ -43,8 +43,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6hzbFjPWQ872UxFhhpxjFg
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:D1UzSZwPtwDmOI7q6g8uKQ
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/HydraFrontend/lib/HydraFrontend/Schema/Jobsets.pm
View file

@ -56,8 +56,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:oRV4yw0DWG5PI0agcM7QHA
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:6Pyrgervmq03S5Nx8QfA1Q
# You can replace this text with custom content, and it will be preserved on regeneration

4
src/HydraFrontend/lib/HydraFrontend/Schema/Projects.pm
View file

@ -29,8 +29,8 @@ __PACKAGE__->has_many(
);
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 10:30:11
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9SeEXSEOH1ocrdkoa7fx5Q
# Created by DBIx::Class::Schema::Loader v0.04005 @ 2008-11-10 14:25:07
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:p8LbF31qRl/JfMK5wfkeCg
# You can replace this text with custom content, and it will be preserved on regeneration

5
src/HydraFrontend/root/hydra.css
View file

@ -157,6 +157,11 @@ ul.productList {
padding-left: 1em;
}
tr.runningJob {
background-color: #ff3030;
color: white;
}
/* Sortable tables */

2
src/HydraFrontend/root/index.tt
View file

@ -9,7 +9,7 @@
</thead>
<tbody>
[% FOREACH job IN jobs -%]
<tr>
<tr [% IF job.busy %]class="runningJob"[% END %] >
<td>[% job.priority %]</td>
<td><tt>[% job.project.name %]</tt></td>
<td><tt>[% job.jobset.name %]</tt></td>

54
src/HydraFrontend/root/project.tt Normal file
View file

@ -0,0 +1,54 @@
[% WRAPPER layout.tt title="Hydra Overview" %]
<h1>Project <tt>[% project.name %]</tt></h1>
<h2>Definition</h2>
[% FOREACH jobset IN project.jobsets -%]
<h3>Jobset <tt>[% jobset.name %]</tt></h3>
<p>
Description: [% jobset.description %]
<br />
Nix expression: <tt>[% jobset.nixexprpath %]</tt> in input <tt>[% jobset.nixexprinput %]</tt>
</p>
<table class="tablesorter">
<thead>
<tr><th>Input name</th><th>Type</th><th>Values</th></tr>
</thead>
<tbody>
[% FOREACH input IN jobset.jobsetinputs -%]
<tr>
<td><tt>[% input.name %]</tt></td>
<td><tt>[% input.type %]</tt></td>
<td>
[% FOREACH alt IN input.jobsetinputalts -%]
[% IF input.type == "string" %]
<tt>"[% alt.value %]"</tt>
[% ELSE %]
<tt>[% alt.uri %]</tt>
[% END %]
[% END %]
</td>
</tr>
[% END %]
</tbody>
</table>
[% END -%]
<h2>Jobs</h2>
<ul>
[% FOREACH jobName IN jobNames -%]
<li><a href="[% c.uri_for('/job' project.name jobName.attrname) %]"><tt>[% jobName.attrname %]</tt></a></li>
[% END %]
</ul>
[% END %]

166
src/build.pl Normal file
View file

@ -0,0 +1,166 @@
#! @perl@ -w
use strict;
use File::Basename;
use HydraFrontend::Schema;
my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
sub isValidPath {
my $path = shift;
return system("nix-store --check-validity $path 2> /dev/null") == 0;
}
sub buildJob {
my ($job) = @_;
my $drvPath = $job->drvpath;
my $outPath = $job->outpath;
my $isCachedBuild = 1;
my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
my $startTime = 0;
my $stopTime = 0;
if (!isValidPath($outPath)) {
$isCachedBuild = 0;
$startTime = time();
print " BUILDING\n";
my $res = system("nix-store --realise $drvPath");
$stopTime = time();
$outputCreated = $res == 0;
}
my $buildStatus;
if ($outputCreated) {
# "Positive" failures, e.g. the builder returned exit code 0
# but flagged some error condition.
$buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;
} else {
$buildStatus = 1; # = Nix failure
}
$db->txn_do(sub {
my $build = $db->resultset('Builds')->create(
{ timestamp => time()
, project => $job->project->name
, jobset => $job->jobset->name
, attrname => $job->attrname
, description => $job->description
, drvpath => $drvPath
, outpath => $outPath
, iscachedbuild => $isCachedBuild
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime
, system => $job->system
});
print " build ID = ", $build->id, "\n";
foreach my $input ($job->inputs) {
$input->job(undef);
$input->build($build->id);
$input->update;
}
my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
if (-e $logPath) {
print " LOG $logPath\n";
$db->resultset('Buildlogs')->create(
{ build => $build->id
, logphase => "full"
, path => $logPath
, type => "raw"
});
}
if ($outputCreated) {
if (-e "$outPath/log") {
foreach my $logPath (glob "$outPath/log/*") {
print " LOG $logPath\n";
$db->resultset('Buildlogs')->create(
{ build => $build->id
, logphase => basename($logPath)
, path => $logPath
, type => "raw"
});
}
}
if (-e "$outPath/nix-support/hydra-build-products") {
open LIST, "$outPath/nix-support/hydra-build-products" or die;
while (<LIST>) {
/^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;
my $type = $1;
my $subtype = $2;
my $path = $3;
die unless -e $path;
$db->resultset('Buildproducts')->create(
{ build => $build->id
, type => $type
, subtype => $subtype
, path => $path
});
}
close LIST;
} else {
$db->resultset('Buildproducts')->create(
{ build => $build->id
, type => "nix-build"
, subtype => ""
, path => $outPath
});
}
}
$job->delete;
});
}
my $jobId = $ARGV[0] or die;
print "building job $jobId\n";
# Lock the job. If necessary, steal the lock from the parent process
# (runner.pl). This is so that if the runner dies, the children
# (i.e. the job builders) can continue to run and won't have the lock
# taken away.
my $job;
$db->txn_do(sub {
($job) = $db->resultset('Jobs')->search({ id => $jobId });
die "job $jobId doesn't exist" unless defined $job;
if ($job->busy != 0 && $job->locker != getppid) {
die "job $jobId is already being built";
}
$job->busy(1);
$job->locker($$);
$job->update;
});
die unless $job;
# Build the job. If it throws an error, unlock the job so that it can
# be retried.
eval {
print "BUILD\n";
buildJob $job;
print "DONE\n";
};
if ($@) {
warn $@;
$db->txn_do(sub {
$job->busy(0);
$job->locker($$);
$job->update;
});
}

6
src/hydra.sql
View file

@ -129,10 +129,10 @@ create table jobs (
id integer primary key autoincrement not null,
timestamp integer not null, -- time this build was added to the db (in Unix time)
priority integer not null,
priority integer not null default 0,
busy integer not null, -- true means someone is building this job now
locker text not null, -- !!! hostname/pid of the process building this job?
busy integer not null default 0, -- true means someone is building this job now
locker text not null default '', -- !!! hostname/pid of the process building this job?
-- Info about the inputs.
project text not null, -- !!! foreign key

65
src/runner.pl Normal file
View file

@ -0,0 +1,65 @@
#! @perl@ -w
use strict;
use HydraFrontend::Schema;
my $db = HydraFrontend::Schema->connect("dbi:SQLite:dbname=hydra.sqlite", "", "", {});
# Unlock jobs whose building process has died.
$db->txn_do(sub {
my @jobs = $db->resultset('Jobs')->search({ busy => 1 });
foreach my $job (@jobs) {
my $pid = $job->locker;
if (kill(0, $pid) != 1) { # see if we can signal the process
print "job ", $job->id, " pid $pid died, unlocking\n";
$job->busy(0);
$job->locker("");
$job->update;
}
}
});
while (1) {
print "looking for runnable jobs...\n";
my $job;
$db->txn_do(sub {
my @jobs = $db->resultset('Jobs')->search({ busy => 0 }, {order_by => ["priority", "timestamp"]});
print "# of available jobs: ", scalar(@jobs), "\n";
if (scalar @jobs > 0) {
$job = $jobs[0];
$job->busy(1);
$job->locker($$);
$job->update;
}
});
# Start the job. We need to do this outside the transaction in
# case it aborts or something.
if (defined $job) {
print "starting job ", $job->id, "\n";
eval {
system("perl -I HydraFrontend/lib -w ./build.pl " . $job->id);
};
if ($@) {
warn $@;
$db->txn_do(sub {
$job->busy(0);
$job->locker($$);
$job->update;
});
}
}
print "sleeping...\n";
sleep(10);
}

174
src/scheduler.pl
View file

@ -2,7 +2,6 @@
use strict;
use XML::Simple;
use File::Basename;
use HydraFrontend::Schema;
@ -15,131 +14,6 @@ sub isValidPath {
}
sub buildJob {
my ($project, $jobset, $jobName, $description, $drvPath, $outPath, $usedInputs, $system) = @_;
if (scalar($db->resultset('Builds')->search({project => $project->name, jobset => $jobset->name, attrname => $jobName, outPath => $outPath})) > 0) {
print " already done\n";
return;
}
my $isCachedBuild = 1;
my $outputCreated = 1; # i.e., the Nix build succeeded (but it could be a positive failure)
my $startTime = 0;
my $stopTime = 0;
if (!isValidPath($outPath)) {
$isCachedBuild = 0;
$startTime = time();
print " BUILDING\n";
my $res = system("nix-store --realise $drvPath");
$stopTime = time();
$outputCreated = $res == 0;
}
my $buildStatus;
if ($outputCreated) {
# "Positive" failures, e.g. the builder returned exit code 0
# but flagged some error condition.
$buildStatus = -e "$outPath/nix-support/failed" ? 2 : 0;
} else {
$buildStatus = 1; # = Nix failure
}
$db->txn_do(sub {
my $build = $db->resultset('Builds')->create(
{ timestamp => time()
, project => $project->name
, jobset => $jobset->name
, attrname => $jobName
, description => $description
, drvpath => $drvPath
, outpath => $outPath
, iscachedbuild => $isCachedBuild
, buildstatus => $buildStatus
, starttime => $startTime
, stoptime => $stopTime
, system => $system
});
print " build ID = ", $build->id, "\n";
foreach my $inputName (keys %{$usedInputs}) {
my $input = $usedInputs->{$inputName};
$db->resultset('Inputs')->create(
{ build => $build->id
, name => $inputName
, type => $input->{type}
, uri => $input->{uri}
#, revision => $input->{orig}->revision
#, tag => $input->{orig}->tag
, value => $input->{value}
, dependency => $input->{id}
, path => ($input->{storePath} or "") # !!! temporary hack
});
}
my $logPath = "/nix/var/log/nix/drvs/" . basename $drvPath;
if (-e $logPath) {
print " LOG $logPath\n";
$db->resultset('Buildlogs')->create(
{ build => $build->id
, logphase => "full"
, path => $logPath
, type => "raw"
});
}
if ($outputCreated) {
if (-e "$outPath/log") {
foreach my $logPath (glob "$outPath/log/*") {
print " LOG $logPath\n";
$db->resultset('Buildlogs')->create(
{ build => $build->id
, logphase => basename($logPath)
, path => $logPath
, type => "raw"
});
}
}
if (-e "$outPath/nix-support/hydra-build-products") {
open LIST, "$outPath/nix-support/hydra-build-products" or die;
while (<LIST>) {
/^(\w+)\s+([\w-]+)\s+(\S+)$/ or die;
my $type = $1;
my $subtype = $2;
my $path = $3;
die unless -e $path;
$db->resultset('Buildproducts')->create(
{ build => $build->id
, type => $type
, subtype => $subtype
, path => $path
});
}
close LIST;
} else {
$db->resultset('Buildproducts')->create(
{ build => $build->id
, type => "nix-build"
, subtype => ""
, path => $outPath
});
}
}
});
}
sub fetchInput {
my ($input, $alt, $inputInfo) = @_;
my $type = $input->type;
@ -186,7 +60,53 @@ sub checkJob {
die unless $job->{drvPath} eq $drvPath;
my $outPath = $job->{outPath};
buildJob($project, $jobset, $jobName, $description, $drvPath, $outPath, $inputInfo, $job->{system});
$db->txn_do(sub {
if (scalar($db->resultset('Builds')->search(
{ project => $project->name, jobset => $jobset->name
, attrname => $jobName, outPath => $outPath })) > 0)
{
print " already done\n";
return;
}
if (scalar($db->resultset('Jobs')->search(
{ project => $project->name, jobset => $jobset->name
, attrname => $jobName, outPath => $outPath })) > 0)
{
print " already queued\n";
return;
}
print " adding to queue\n";
my $job = $db->resultset('Jobs')->create(
{ timestamp => time()
, priority => 0
, busy => 0
, locker => ""
, project => $project->name
, jobset => $jobset->name
, attrname => $jobName
, description => $description
, drvpath => $drvPath
, outpath => $outPath
, system => $job->{system}
});
foreach my $inputName (keys %{$inputInfo}) {
my $input = $inputInfo->{$inputName};
$db->resultset('Inputs')->create(
{ job => $job->id
, name => $inputName
, type => $input->{type}
, uri => $input->{uri}
#, revision => $input->{orig}->revision
#, tag => $input->{orig}->tag
, value => $input->{value}
, dependency => $input->{id}
, path => ($input->{storePath} or "") # !!! temporary hack
});
}
});
};