lix-project/hydra
3
0
Fork 6
Code Issues 2 Pull requests Packages Projects Releases Wiki Activity

Merge the BuildSchedulingInfo table into the Builds table

Browse source 
This simplifies the code and improves performance since it reduces
the number of joins.
This commit is contained in:
Eelco Dolstra 2012-02-29 02:22:49 +01:00
parent 19fe4b9b4a
commit 25334715f8
21 changed files with 194 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/lib/Hydra/Controller/API.pm
View file

@ -136,13 +136,11 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
my $nr = $c->request->params->{nr} ; my $nr = $c->request->params->{nr} ;
error($c, "Parameter not defined!") if !defined $nr; error($c, "Parameter not defined!") if !defined $nr;
my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, join => ['schedulingInfo'] , order_by => ["busy DESC", "priority DESC", "timestamp"], '+select' => ['schedulingInfo.priority', 'schedulingInfo.busy'], '+as' => ['priority', 'busy'] }); my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["busy DESC", "priority DESC", "timestamp"]});
my @list ; my @list;
foreach my $b (@builds) { push @list, buildToHash($_) foreach @builds;
push @list, buildToHash($b) ;
}
$c->stash->{'plain'} = { $c->stash->{'plain'} = {
data => scalar (JSON::Any->objToJson(\@list)) data => scalar (JSON::Any->objToJson(\@list))
}; };
@ -151,18 +149,18 @@ sub queue : Chained('api') PathPart('queue') Args(0) {
sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) { sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) {
my ($self, $c) = @_; my ($self, $c) = @_;
my $nrQueuedBuilds = $c->model('DB::BuildSchedulingInfo')->count(); my $nrQueuedBuilds = $c->model('DB::Builds')->search({finished => 0})->count();
$c->stash->{'plain'} = { $c->stash->{'plain'} = {
data => " $nrQueuedBuilds" data => "$nrQueuedBuilds"
}; };
$c->forward('Hydra::View::Plain'); $c->forward('Hydra::View::Plain');
} }
sub nrrunning : Chained('api') PathPart('nrrunning') Args(0) { sub nrrunning : Chained('api') PathPart('nrrunning') Args(0) {
my ($self, $c) = @_; my ($self, $c) = @_;
my $nrRunningBuilds = $c->model('DB::BuildSchedulingInfo')->search({ busy => 1 }, {})->count(); my $nrRunningBuilds = $c->model('DB::Builds')->search({finished => 0, busy => 1 })->count();
$c->stash->{'plain'} = { $c->stash->{'plain'} = {
data => " $nrRunningBuilds" data => "$nrRunningBuilds"
}; };
$c->forward('Hydra::View::Plain'); $c->forward('Hydra::View::Plain');
} }

12
src/lib/Hydra/Controller/Admin.pm
View file

@ -57,9 +57,9 @@ sub index : Chained('admin') PathPart('') Args(0) {
, '+as' => ['idle'] , '+as' => ['idle']
})]; })];
$c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
{ 'me.busy' => 1, 'schedulingInfo.busy' => 1 }, { finished => 0, 'me.busy' => 1, 'build.busy' => 1, },
{ join => [ 'schedulingInfo', 'build' ] { join => [ 'build' ]
, order_by => [ 'machine' ] , order_by => [ 'machine', 'stepnr' ]
} ) ]; } ) ];
$c->stash->{template} = 'admin.tt'; $c->stash->{template} = 'admin.tt';
} }
@ -296,13 +296,15 @@ sub machine_disable : Chained('machine') PathPart('disable') Args(0) {
sub clear_queue_non_current : Chained('admin') Path('clear-queue-non-current') Args(0) { sub clear_queue_non_current : Chained('admin') Path('clear-queue-non-current') Args(0) {
my ($self, $c) = @_; my ($self, $c) = @_;
$c->model('DB::Builds')->search({iscurrent => 0, busy => 0}, { join => 'schedulingInfo' })->delete_all; # !!! Mark the builds as cancelled instead.
$c->model('DB::Builds')->search({finished => 0, iscurrent => 0, busy => 0})->delete_all;
$c->res->redirect("/admin"); $c->res->redirect("/admin");
} }
sub clear_queue : Chained('admin') Path('clear-queue') Args(0) { sub clear_queue : Chained('admin') Path('clear-queue') Args(0) {
my ($self, $c) = @_; my ($self, $c) = @_;
$c->model('DB::Builds')->search({busy => 0}, { join => 'schedulingInfo' })->delete_all; # !!! Mark the builds as cancelled instead.
$c->model('DB::Builds')->search({finished => 0, busy => 0})->delete_all;
$c->res->redirect("/admin"); $c->res->redirect("/admin");
} }

29
src/lib/Hydra/Controller/Build.pm
View file

@ -41,11 +41,10 @@ sub view_build : Chained('build') PathPart('') Args(0) {
$c->stash->{drvAvailable} = isValidPath $build->drvpath; $c->stash->{drvAvailable} = isValidPath $build->drvpath;
$c->stash->{flashMsg} = $c->flash->{buildMsg}; $c->stash->{flashMsg} = $c->flash->{buildMsg};
my $pathHash = $c->stash->{available} ? queryPathHash($build->outpath) : "Not available"; $c->stash->{pathHash} = $c->stash->{available} ? queryPathHash($build->outpath) : undef;
$c->stash->{pathHash} = $pathHash;
if (!$build->finished && $build->schedulingInfo->busy) { if (!$build->finished && $build->busy) {
my $logfile = $build->schedulingInfo->logfile; my $logfile = $build->logfile;
$c->stash->{logtext} = `cat $logfile` if defined $logfile && -e $logfile; $c->stash->{logtext} = `cat $logfile` if defined $logfile && -e $logfile;
} }
@ -81,13 +80,13 @@ sub view_build : Chained('build') PathPart('') Args(0) {
]; ];
} }
my $r = joinWithResultInfo( $c, $c->model('DB::Builds'))->search( #my $r = joinWithResultInfo( $c, $c->model('DB::Builds'))->search(
{ eval => { -in => $build->jobsetevalmembers->get_column('eval')->as_query } } # { eval => { -in => $build->jobsetevalmembers->all->get_column('eval')->as_query } }
, { join => 'jobsetevalmembers', order_by => [ 'project', 'jobset', 'job'], distinct => 1 } # , { join => 'jobsetevalmembers', order_by => [ 'project', 'jobset', 'job'], distinct => 1 }
); # );
if ($r->count <= 100) { #if ($r->count <= 100) {
$c->stash->{relatedbuilds} = [$r->all]; # $c->stash->{relatedbuilds} = [$r->all];
} #}
} }
@ -141,7 +140,7 @@ sub showLog {
my $url = $c->request->uri->as_string; my $url = $c->request->uri->as_string;
$url =~ s/tail-reload/tail/g; $url =~ s/tail-reload/tail/g;
$c->stash->{url} = $url; $c->stash->{url} = $url;
$c->stash->{reload} = defined $c->stash->{build}->schedulingInfo && $c->stash->{build}->schedulingInfo->busy; $c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy;
$c->stash->{title} = ""; $c->stash->{title} = "";
$c->stash->{contents} = (scalar `$pipestart | tail -n 50`) || " "; $c->stash->{contents} = (scalar `$pipestart | tail -n 50`) || " ";
$c->stash->{template} = 'plain-reload.tt'; $c->stash->{template} = 'plain-reload.tt';
@ -406,21 +405,19 @@ sub cancel : Chained('build') PathPart Args(0) {
txn_do($c->model('DB')->schema, sub { txn_do($c->model('DB')->schema, sub {
error($c, "This build cannot be cancelled.") error($c, "This build cannot be cancelled.")
if $build->finished || $build->schedulingInfo->busy; if $build->finished || $build->busy;
# !!! Actually, it would be nice to be able to cancel busy # !!! Actually, it would be nice to be able to cancel busy
# builds as well, but we would have to send a signal or # builds as well, but we would have to send a signal or
# something to the build process. # something to the build process.
$build->update({finished => 1, timestamp => time}); $build->update({finished => 1, busy => 0, timestamp => time});
$c->model('DB::BuildResultInfo')->create( $c->model('DB::BuildResultInfo')->create(
{ id => $build->id { id => $build->id
, iscachedbuild => 0 , iscachedbuild => 0
, buildstatus => 4 # = cancelled , buildstatus => 4 # = cancelled
}); });
$build->schedulingInfo->delete;
}); });
$c->flash->{buildMsg} = "Build has been cancelled."; $c->flash->{buildMsg} = "Build has been cancelled.";

12
src/lib/Hydra/Controller/Job.pm
View file

@ -36,9 +36,15 @@ sub overview : Chained('job') PathPart('') Args(0) {
, order_by => 'timestamp DESC', rows => 10 , order_by => 'timestamp DESC', rows => 10
}) ]; }) ];
$c->stash->{runningBuilds} = [$c->stash->{job}->builds->search({busy => 1}, { join => ['schedulingInfo', 'project'] , order_by => ["priority DESC", "timestamp"] $c->stash->{runningBuilds} = [
, '+select' => ['project.enabled', 'schedulingInfo.priority', 'schedulingInfo.disabled', 'schedulingInfo.busy'] $c->stash->{job}->builds->search(
, '+as' => ['enabled', 'priority', 'disabled', 'busy'] })]; { busy => 1 },
{ join => ['project']
, order_by => ["priority DESC", "timestamp"]
, '+select' => ['project.enabled']
, '+as' => ['enabled']
}
) ];
$c->stash->{systems} = [$c->stash->{job}->builds->search({iscurrent => 1}, {select => ["system"], distinct => 1})]; $c->stash->{systems} = [$c->stash->{job}->builds->search({iscurrent => 1}, {select => ["system"], distinct => 1})];
} }

6
src/lib/Hydra/Controller/Jobset.pm
View file

@ -65,10 +65,10 @@ sub jobsetIndex {
my @as = (); my @as = ();
push(@select, "job"); push(@as, "job"); push(@select, "job"); push(@as, "job");
foreach my $system (@systems) { foreach my $system (@systems) {
push(@select, "(SELECT buildstatus FROM BuildResultInfo bri NATURAL JOIN Builds b WHERE b.id = (SELECT MAX(id) FROM Builds t WHERE t.project = me.project AND t.jobset = me.jobset AND t.job = me.job AND t.system = '$system' AND t.iscurrent = 1 ))"); push(@select, "(select buildstatus from BuildResultInfo bri join Builds b using (id) where b.id = (select max(id) from Builds t where t.project = me.project and t.jobset = me.jobset and t.job = me.job and t.system = '$system' and t.iscurrent = 1 ))");
push(@as, $system); push(@as, $system);
push(@select, "(SELECT b.id FROM BuildResultInfo bri NATURAL JOIN Builds b WHERE b.id = (SELECT MAX(id) FROM Builds t WHERE t.project = me.project AND t.jobset = me.jobset AND t.job = me.job AND t.system = '$system' AND t.iscurrent = 1 ))"); push(@select, "(select b.id from BuildResultInfo bri join Builds b using (id) where b.id = (select max(id) from Builds t where t.project = me.project and t.jobset = me.jobset and t.job = me.job and t.system = '$system' and t.iscurrent = 1 ))");
push(@as, $system."-build"); push(@as, "$system-build");
} }
$c->stash->{activeJobsStatus} = $c->stash->{activeJobsStatus} =
[ $c->model('DB')->resultset('ActiveJobsForJobset')->search( [ $c->model('DB')->resultset('ActiveJobsForJobset')->search(

10
src/lib/Hydra/Controller/Root.pm
View file

@ -22,8 +22,8 @@ sub begin :Private {
$c->stash->{tracker} = $ENV{"HYDRA_TRACKER"} ; $c->stash->{tracker} = $ENV{"HYDRA_TRACKER"} ;
if (scalar(@args) == 0 || $args[0] ne "static") { if (scalar(@args) == 0 || $args[0] ne "static") {
$c->stash->{nrRunningBuilds} = $c->model('DB::BuildSchedulingInfo')->search({ busy => 1 }, {})->count(); $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count();
$c->stash->{nrQueuedBuilds} = $c->model('DB::BuildSchedulingInfo')->count(); $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count();
} }
} }
@ -74,7 +74,7 @@ sub queue :Local {
my ($self, $c) = @_; my ($self, $c) = @_;
$c->stash->{template} = 'queue.tt'; $c->stash->{template} = 'queue.tt';
$c->stash->{queue} = [$c->model('DB::Builds')->search( $c->stash->{queue} = [$c->model('DB::Builds')->search(
{finished => 0}, {join => ['schedulingInfo', 'project'] , order_by => ["priority DESC", "timestamp"], '+select' => ['project.enabled', 'schedulingInfo.priority', 'schedulingInfo.disabled', 'schedulingInfo.busy'], '+as' => ['enabled', 'priority', 'disabled', 'busy'] })]; {finished => 0}, {join => ['project'] , order_by => ["priority DESC", "timestamp"], '+select' => ['project.enabled'], '+as' => ['enabled'] })];
$c->stash->{flashMsg} = $c->flash->{buildMsg}; $c->stash->{flashMsg} = $c->flash->{buildMsg};
} }
@ -99,8 +99,8 @@ sub timeline :Local {
sub status :Local { sub status :Local {
my ($self, $c) = @_; my ($self, $c) = @_;
$c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search(
{ 'me.busy' => 1, 'schedulingInfo.busy' => 1 }, { 'me.busy' => 1, 'build.busy' => 1 },
{ join => [ 'schedulingInfo', 'build' ] { join => [ 'build' ]
, order_by => [ 'machine' ] , order_by => [ 'machine' ]
} ) ]; } ) ];
} }

26
src/lib/Hydra/Helper/AddBuilds.pm
View file

@ -851,12 +851,11 @@ sub checkBuild {
, nixexprpath => $jobset->nixexprpath , nixexprpath => $jobset->nixexprpath
}); });
$currentBuilds->{$build->id} = 1; $currentBuilds->{$build->id} = 1;
if (isValidPath($outPath)) { if (isValidPath($outPath)) {
print STDERR "marked as cached build ", $build->id, "\n"; print STDERR "marked as cached build ", $build->id, "\n";
$build->update({ finished => 1 }); $build->update({ finished => 1 });
$build->create_related('buildresultinfo', $build->create_related('buildresultinfo',
{ iscachedbuild => 1 { iscachedbuild => 1
, buildstatus => 0 , buildstatus => 0
@ -869,8 +868,8 @@ sub checkBuild {
addBuildProducts($db, $build); addBuildProducts($db, $build);
} else { } else {
print STDERR "added to queue as build ", $build->id, "\n"; print STDERR "added to queue as build ", $build->id, "\n";
$build->create_related('buildschedulinginfo', $build->update(
{ priority => $priority { priority => $priority
, busy => 0 , busy => 0
, locker => "" , locker => ""
}); });
@ -906,24 +905,23 @@ sub restartBuild {
my ($db, $build) = @_; my ($db, $build) = @_;
txn_do($db, sub { txn_do($db, sub {
my $drvpath = $build->drvpath ; my $drvpath = $build->drvpath;
my $outpath = $build->outpath ; my $outpath = $build->outpath;
my $paths = ""; my $paths = "";
foreach my $bs ($build->buildsteps) { foreach my $bs ($build->buildsteps) {
$paths = $paths . " " . $bs->outpath; $paths = $paths . " " . $bs->outpath;
} }
my $r = `nix-store --clear-failed-paths $paths $outpath`; my $r = `nix-store --clear-failed-paths $paths $outpath`;
$build->update({finished => 0, timestamp => time});
$build->resultInfo->delete; $build->update(
{ finished => 0
$db->resultset('BuildSchedulingInfo')->create( , timestamp => time
{ id => $build->id
, priority => 0 # don't know the original priority anymore...
, busy => 0 , busy => 0
, locker => "" , locker => ""
}); });
$build->resultInfo->delete;
}); });
} }

4
src/lib/Hydra/Helper/CatalystUtils.pm
View file

@ -82,9 +82,7 @@ sub getBuildStats {
$c->stash->{scheduledBuilds} = $builds->search({finished => 0}) || 0; $c->stash->{scheduledBuilds} = $builds->search({finished => 0}) || 0;
$c->stash->{busyBuilds} = $builds->search( $c->stash->{busyBuilds} = $builds->search({finished => 0, busy => 1}) || 0;
{finished => 0, busy => 1},
{join => 'schedulingInfo'}) || 0;
my $res; my $res;
$res = $builds->search({}, $res = $builds->search({},

12
src/lib/Hydra/Helper/Nix.pm
View file

@ -195,13 +195,13 @@ sub jobsetOverview {
return $project->jobsets->search( isProjectOwner($c, $project) ? {} : { hidden => 0 }, return $project->jobsets->search( isProjectOwner($c, $project) ? {} : { hidden => 0 },
{ order_by => "name" { order_by => "name"
, "+select" => , "+select" =>
[ "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildSchedulingInfo WHERE me.project = a.project AND me.name = a.jobset AND a.isCurrent = 1)" [ "(select count(*) from Builds as a where a.finished = 0 and me.project = a.project and me.name = a.jobset and a.isCurrent = 1)"
, "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildResultInfo WHERE me.project = a.project AND me.name = a.jobset AND buildstatus <> 0 AND a.isCurrent = 1)" , "(select count(*) from Builds as a join BuildResultInfo r using (id) where me.project = a.project and me.name = a.jobset and buildstatus <> 0 and a.isCurrent = 1)"
, "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildResultInfo WHERE me.project = a.project AND me.name = a.jobset AND buildstatus = 0 AND a.isCurrent = 1)" , "(select count(*) from Builds as a join BuildResultInfo r using (id) where me.project = a.project and me.name = a.jobset and buildstatus = 0 and a.isCurrent = 1)"
, "(SELECT COUNT(*) FROM Builds AS a WHERE me.project = a.project AND me.name = a.jobset AND a.isCurrent = 1)" , "(select count(*) from Builds as a where me.project = a.project and me.name = a.jobset and a.isCurrent = 1)"
] ]
, "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"] , "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"]
}); });
} }
sub getViewResult { sub getViewResult {

8
src/lib/Hydra/Schema/BuildProducts.pm
View file

@ -46,7 +46,7 @@ __PACKAGE__->table("BuildProducts");
=head2 filesize =head2 filesize
data_type: 'integer' data_type: 'bigint'
is_nullable: 1 is_nullable: 1
=head2 sha1hash =head2 sha1hash
@ -91,7 +91,7 @@ __PACKAGE__->add_columns(
"subtype", "subtype",
{ data_type => "text", is_nullable => 0 }, { data_type => "text", is_nullable => 0 },
"filesize", "filesize",
{ data_type => "integer", is_nullable => 1 }, { data_type => "bigint", is_nullable => 1 },
"sha1hash", "sha1hash",
{ data_type => "text", is_nullable => 1 }, { data_type => "text", is_nullable => 1 },
"sha256hash", "sha256hash",
@ -133,8 +133,8 @@ Related object: L<Hydra::Schema::Builds>
__PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {}); __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {});
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 # Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:18
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:suSgQkBLXzu0yD4YicRS1A # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dzTKwZ7bby7kplnSgta3Gw
# You can replace this text with custom content, and it will be preserved on regeneration # You can replace this text with custom content, and it will be preserved on regeneration
1; 1;

120
src/lib/Hydra/Schema/BuildSchedulingInfo.pm
View file

@ -1,120 +0,0 @@
use utf8;
package Hydra::Schema::BuildSchedulingInfo;
# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE
=head1 NAME
Hydra::Schema::BuildSchedulingInfo
=cut
use strict;
use warnings;
use base 'DBIx::Class::Core';
=head1 TABLE: C<BuildSchedulingInfo>
=cut
__PACKAGE__->table("BuildSchedulingInfo");
=head1 ACCESSORS
=head2 id
data_type: 'integer'
is_auto_increment: 1
is_foreign_key: 1
is_nullable: 0
=head2 priority
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 busy
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 locker
data_type: 'text'
default_value: (empty string)
is_nullable: 0
=head2 logfile
data_type: 'text'
is_nullable: 1
=head2 disabled
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 starttime
data_type: 'integer'
is_nullable: 1
=cut
__PACKAGE__->add_columns(
"id",
{
data_type => "integer",
is_auto_increment => 1,
is_foreign_key => 1,
is_nullable => 0,
},
"priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"busy",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"locker",
{ data_type => "text", default_value => "", is_nullable => 0 },
"logfile",
{ data_type => "text", is_nullable => 1 },
"disabled",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"starttime",
{ data_type => "integer", is_nullable => 1 },
);
=head1 PRIMARY KEY
=over 4
=item * L</id>
=back
=cut
__PACKAGE__->set_primary_key("id");
=head1 RELATIONS
=head2 id
Type: belongs_to
Related object: L<Hydra::Schema::Builds>
=cut
__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }, {});
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uz7y9Ly+ADRrtrPfEk9lGA
# You can replace this text with custom content, and it will be preserved on regeneration
1;

6
src/lib/Hydra/Schema/BuildSteps.pm
View file

@ -151,10 +151,4 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {})
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 # Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5H+OkGT0zQEWkAjU+OlBdg # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5H+OkGT0zQEWkAjU+OlBdg
__PACKAGE__->belongs_to(
"schedulingInfo",
"Hydra::Schema::BuildSchedulingInfo",
{ id => "build" },
);
1; 1;

83
src/lib/Hydra/Schema/Builds.pm
View file

@ -130,6 +130,39 @@ __PACKAGE__->table("Builds");
data_type: 'text' data_type: 'text'
is_nullable: 1 is_nullable: 1
=head2 priority
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 busy
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 locker
data_type: 'text'
is_nullable: 1
=head2 logfile
data_type: 'text'
is_nullable: 1
=head2 disabled
data_type: 'integer'
default_value: 0
is_nullable: 0
=head2 starttime
data_type: 'integer'
is_nullable: 1
=cut =cut
__PACKAGE__->add_columns( __PACKAGE__->add_columns(
@ -173,6 +206,18 @@ __PACKAGE__->add_columns(
{ data_type => "text", is_nullable => 1 }, { data_type => "text", is_nullable => 1 },
"nixexprpath", "nixexprpath",
{ data_type => "text", is_nullable => 1 }, { data_type => "text", is_nullable => 1 },
"priority",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"busy",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"locker",
{ data_type => "text", is_nullable => 1 },
"logfile",
{ data_type => "text", is_nullable => 1 },
"disabled",
{ data_type => "integer", default_value => 0, is_nullable => 0 },
"starttime",
{ data_type => "integer", is_nullable => 1 },
); );
=head1 PRIMARY KEY =head1 PRIMARY KEY
@ -249,21 +294,6 @@ __PACKAGE__->might_have(
{}, {},
); );
=head2 buildschedulinginfo
Type: might_have
Related object: L<Hydra::Schema::BuildSchedulingInfo>
=cut
__PACKAGE__->might_have(
"buildschedulinginfo",
"Hydra::Schema::BuildSchedulingInfo",
{ "foreign.id" => "self.id" },
{},
);
=head2 buildsteps =head2 buildsteps
Type: has_many Type: has_many
@ -350,8 +380,8 @@ __PACKAGE__->has_many(
); );
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 # Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:54
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RRtBPTdD946kA5133+c4kw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:VnnyFTwnLncGb2Dj2/giiA
use Hydra::Helper::Nix; use Hydra::Helper::Nix;
@ -369,12 +399,6 @@ __PACKAGE__->has_many(
{ "foreign.build" => "self.id" }, { "foreign.build" => "self.id" },
); );
__PACKAGE__->belongs_to(
"schedulingInfo",
"Hydra::Schema::BuildSchedulingInfo",
{ id => "id" },
);
__PACKAGE__->belongs_to( __PACKAGE__->belongs_to(
"resultInfo", "resultInfo",
"Hydra::Schema::BuildResultInfo", "Hydra::Schema::BuildResultInfo",
@ -410,11 +434,11 @@ sub makeQueries {
my $joinWithStatusChange = my $joinWithStatusChange =
<<QUERY; <<QUERY;
natural join BuildResultInfo r join BuildResultInfo r using (id)
left join Builds b on left join Builds b on
b.id = b.id =
(select max(id) (select max(c.id)
from builds c natural join buildresultinfo r2 from builds c join buildresultinfo r2 on c.id = r2.id
where where
x.project = c.project and x.jobset = c.jobset and x.job = c.job and x.system = c.system and x.project = c.project and x.jobset = c.jobset and x.job = c.job and x.system = c.system and
x.id > c.id and x.id > c.id and
@ -432,11 +456,12 @@ QUERY
x.id, x.finished, x.timestamp, x.project, x.jobset, x.job, x.nixname, x.id, x.finished, x.timestamp, x.project, x.jobset, x.job, x.nixname,
x.description, x.drvpath, x.outpath, x.system, x.longdescription, x.description, x.drvpath, x.outpath, x.system, x.longdescription,
x.license, x.homepage, x.maintainers, x.isCurrent, x.nixExprInput, x.license, x.homepage, x.maintainers, x.isCurrent, x.nixExprInput,
x.nixExprPath, x.maxsilent, x.timeout, x.nixExprPath, x.maxsilent, x.timeout, x.priority, x.busy, x.locker,
x.logfile, x.disabled, x.startTime,
b.id as statusChangeId, b.timestamp as statusChangeTime b.id as statusChangeId, b.timestamp as statusChangeTime
from from
(select (select
(select max(id) from builds b (select max(b.id) from builds b
where where
project = activeJobs.project and jobset = activeJobs.jobset project = activeJobs.project and jobset = activeJobs.jobset
and job = activeJobs.job and system = activeJobs.system and job = activeJobs.job and system = activeJobs.system
@ -457,7 +482,7 @@ QUERY
select * select *
from from
(select (select
(select max(id) from builds b (select max(b.id) from builds b
where where
project = activeJobs.project and jobset = activeJobs.jobset project = activeJobs.project and jobset = activeJobs.jobset
and job = activeJobs.job and system = activeJobs.system and job = activeJobs.job and system = activeJobs.system

22
src/lib/Hydra/Schema/SchemaVersion.pm
View file

@ -26,31 +26,15 @@ __PACKAGE__->table("SchemaVersion");
=head2 version =head2 version
data_type: 'integer' data_type: 'integer'
is_auto_increment: 1
is_nullable: 0 is_nullable: 0
=cut =cut
__PACKAGE__->add_columns( __PACKAGE__->add_columns("version", { data_type => "integer", is_nullable => 0 });
"version",
{ data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
);
=head1 PRIMARY KEY
=over 4
=item * L</version>
=back
=cut
__PACKAGE__->set_primary_key("version");
# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 # Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:18
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:F/jsSRq8pxR4mWq/N4qYGw # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LFD28W0GvvrOOylCM98SEQ
# You can replace this text with custom code or comments, and it will be preserved on regeneration # You can replace this text with custom code or comments, and it will be preserved on regeneration

8
src/root/build.tt
View file

@ -70,7 +70,7 @@
<h1> <h1>
Job <tt>[% project.name %]:[% jobset.name %]:[% job.name %]</tt> build [% id %] Job <tt>[% project.name %]:[% jobset.name %]:[% job.name %]</tt> build [% id %]
[% IF !build.finished %] [% IF !build.finished %]
[% IF build.schedulingInfo.busy %] [% IF build.busy %]
(currently building) (currently building)
[% ELSE %] [% ELSE %]
(scheduled) (scheduled)
@ -127,7 +127,7 @@
<th>System:</th> <th>System:</th>
<td><tt>[% build.system %]</tt></td> <td><tt>[% build.system %]</tt></td>
</tr> </tr>
[% IF !build.schedulingInfo %] [% IF !build.finished %]
<tr> <tr>
<th>Duration:</th> <th>Duration:</th>
<td> <td>
@ -316,12 +316,14 @@
[% END %] [% END %]
</td> </td>
</tr> </tr>
[% IF pathHash %]
<tr> <tr>
<th>Output store path hash:</th> <th>Output store path hash:</th>
<td> <td>
<tt>[% pathHash %]</tt> <tt>[% pathHash %]</tt>
</td> </td>
</tr> </tr>
[% END %]
<tr> <tr>
<th>Time added:</th> <th>Time added:</th>
<td>[% INCLUDE renderDateTime timestamp = build.timestamp %]</td> <td>[% INCLUDE renderDateTime timestamp = build.timestamp %]</td>
@ -366,7 +368,7 @@
[% IF !build.finished %] [% IF !build.finished %]
<tr> <tr>
<th>Priority:</th> <th>Priority:</th>
<td>[% build.schedulingInfo.priority %]</td> <td>[% build.priority %]</td>
</tr> </tr>
[% END %] [% END %]
[% IF build.finished && build.buildproducts %] [% IF build.finished && build.buildproducts %]

12
src/root/common.tt
View file

@ -93,7 +93,7 @@
[%- FOREACH build IN builds -%] [%- FOREACH build IN builds -%]
<tr class="clickable <tr class="clickable
[%- IF showSchedulingInfo -%] [%- IF showSchedulingInfo -%]
[%- IF build.get_column('busy') %]runningBuild[% ELSIF build.get_column('disabled') == 1 || build.get_column('enabled') == 0 %]disabledBuild[% END -%] [%- IF build.busy %]runningBuild[% ELSIF build.disabled == 1 || build.get_column('enabled') == 0 %]disabledBuild[% END -%]
[%- ELSE -%] [%- ELSE -%]
[%- IF odd %] odd [% END; odd = !odd -%] [%- IF odd %] odd [% END; odd = !odd -%]
[%- END %]" [%- END %]"
@ -105,8 +105,8 @@
[%- END -%] [%- END -%]
<td><a href="[% c.uri_for('/build' build.id) %]">[% build.id %]</a></td> <td><a href="[% c.uri_for('/build' build.id) %]">[% build.id %]</a></td>
[%- IF showSchedulingInfo -%] [%- IF showSchedulingInfo -%]
<td>[% IF build.get_column('busy') %]<img src="/static/images/running.gif" alt="Running" />[% ELSIF build.get_column('disabled') == 1 || build.get_column('enabled') == 0 %]Disabled[% END %]</td> <td>[% IF build.busy %]<img src="/static/images/running.gif" alt="Running" />[% ELSIF build.disabled == 1 || build.get_column('enabled') == 0 %]Disabled[% END %]</td>
<td>[% build.get_column('priority') %]</td> <td>[% build.priority %]</td>
[%- END -%] [%- END -%]
<td>[%- INCLUDE renderFullJobNameOfBuild -%]</td> <td>[%- INCLUDE renderFullJobNameOfBuild -%]</td>
<td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td> <td>[% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %]</td>
@ -203,7 +203,7 @@
[%- BLOCK renderBuildStatusIcon -%] [%- BLOCK renderBuildStatusIcon -%]
[%- finished = build != undef ? build.finished : 1 -%] [%- finished = build != undef ? build.finished : 1 -%]
[%- busy = busy != undef ? busy : build.schedulingInfo.busy -%] [%- busy = busy != undef ? busy : build.busy -%]
[%- buildstatus = buildstatus != undef ? buildstatus : build.resultInfo.buildstatus -%] [%- buildstatus = buildstatus != undef ? buildstatus : build.resultInfo.buildstatus -%]
[%- IF finished -%] [%- IF finished -%]
[%- IF buildstatus == 0 -%] [%- IF buildstatus == 0 -%]
@ -255,9 +255,9 @@
<button id="restart" type="submit">Restart</button> <button id="restart" type="submit">Restart</button>
</form> </form>
[% END %] [% END %]
[% ELSIF build.schedulingInfo.busy %] [% ELSIF build.busy %]
<strong>Build in progress</strong> <strong>Build in progress</strong>
since [% INCLUDE renderDateTime timestamp = build.schedulingInfo.starttime %] since [% INCLUDE renderDateTime timestamp = build.starttime %]
[% ELSE %] [% ELSE %]
<strong>Scheduled to be built</strong> <strong>Scheduled to be built</strong>
[% IF c.user_exists %] [% IF c.user_exists %]

10
src/script/hydra-build
View file

@ -398,7 +398,7 @@ sub doBuild {
} }
txn_do($db, sub { txn_do($db, sub {
$build->update({finished => 1, timestamp => time}); $build->update({finished => 1, busy => 0, locker => '', logfile => '', timestamp => time});
my $releaseName = getReleaseName($outPath); my $releaseName = getReleaseName($outPath);
@ -423,8 +423,6 @@ sub doBuild {
if ($buildStatus == 0 || $buildStatus == 6) { if ($buildStatus == 0 || $buildStatus == 6) {
addBuildProducts($db, $build); addBuildProducts($db, $build);
} }
$build->schedulingInfo->delete;
}); });
sendEmailNotification $build; sendEmailNotification $build;
@ -453,10 +451,10 @@ txn_do($db, sub {
$build = $db->resultset('Builds')->find($buildId); $build = $db->resultset('Builds')->find($buildId);
die "build $buildId doesn't exist" unless defined $build; die "build $buildId doesn't exist" unless defined $build;
die "build $buildId already done" if defined $build->resultInfo; die "build $buildId already done" if defined $build->resultInfo;
if ($build->schedulingInfo->busy != 0 && $build->schedulingInfo->locker != getppid) { if ($build->busy != 0 && $build->locker != getppid) {
die "build $buildId is already being built"; die "build $buildId is already being built";
} }
$build->schedulingInfo->update({busy => 1, locker => $$}); $build->update({busy => 1, locker => $$});
$build->buildsteps->search({busy => 1})->delete_all; $build->buildsteps->search({busy => 1})->delete_all;
$build->buildproducts->delete_all; $build->buildproducts->delete_all;
}); });
@ -472,6 +470,6 @@ eval {
if ($@) { if ($@) {
warn $@; warn $@;
txn_do($db, sub { txn_do($db, sub {
$build->schedulingInfo->update({busy => 0, locker => $$}); $build->update({busy => 0, locker => $$});
}); });
} }

40
src/script/hydra-queue-runner
View file

@ -20,10 +20,9 @@ STDOUT->autoflush();
sub unlockDeadBuilds { sub unlockDeadBuilds {
# Unlock builds whose building process has died. # Unlock builds whose building process has died.
txn_do($db, sub { txn_do($db, sub {
my @builds = $db->resultset('Builds')->search( my @builds = $db->resultset('Builds')->search({finished => 0, busy => 1});
{finished => 0, busy => 1}, {join => 'schedulingInfo'});
foreach my $build (@builds) { foreach my $build (@builds) {
my $pid = $build->schedulingInfo->locker; my $pid = $build->locker;
my $unlock = 0; my $unlock = 0;
if ($pid == $$) { if ($pid == $$) {
# Work around sqlite locking timeouts: if the child # Work around sqlite locking timeouts: if the child
@ -32,7 +31,7 @@ sub unlockDeadBuilds {
# So if after a minute it hasn't been updated, # So if after a minute it hasn't been updated,
# unlock the build. !!! need a better fix for those # unlock the build. !!! need a better fix for those
# locking timeouts. # locking timeouts.
if ($build->schedulingInfo->starttime + 60 < time) { if ($build->starttime + 60 < time) {
$unlock = 1; $unlock = 1;
} }
} elsif (kill(0, $pid) != 1) { # see if we can signal the process } elsif (kill(0, $pid) != 1) { # see if we can signal the process
@ -40,9 +39,9 @@ sub unlockDeadBuilds {
} }
if ($unlock) { if ($unlock) {
print "build ", $build->id, " pid $pid died, unlocking\n"; print "build ", $build->id, " pid $pid died, unlocking\n";
$build->schedulingInfo->busy(0); $build->busy(0);
$build->schedulingInfo->locker(""); $build->locker("");
$build->schedulingInfo->update; $build->update;
} }
} }
}); });
@ -64,7 +63,7 @@ sub findBuildDependencyInQueue {
($depBuild) = $db->resultset('Builds')->search( ($depBuild) = $db->resultset('Builds')->search(
{ drvpath => [ @drvs ], finished => 0, busy => 0, enabled => 1, disabled => 0 }, { drvpath => [ @drvs ], finished => 0, busy => 0, enabled => 1, disabled => 0 },
{ join => ['schedulingInfo', 'project'], rows => 1 } ) ; { join => ['project'], rows => 1 } ) ;
return $depBuild; return $depBuild;
} }
@ -79,7 +78,7 @@ sub checkBuilds {
# Get the system types for the runnable builds. # Get the system types for the runnable builds.
my @systemTypes = $db->resultset('Builds')->search( my @systemTypes = $db->resultset('Builds')->search(
{ finished => 0, busy => 0, enabled => 1, disabled => 0 }, { finished => 0, busy => 0, enabled => 1, disabled => 0 },
{ join => ['schedulingInfo', 'project'], select => ['system'], as => ['system'], distinct => 1 }); { join => ['project'], select => ['system'], as => ['system'], distinct => 1 });
# For each system type, select up to the maximum number of # For each system type, select up to the maximum number of
# concurrent build for that system type. Choose the highest # concurrent build for that system type. Choose the highest
@ -88,8 +87,7 @@ sub checkBuilds {
# How many builds are already currently executing for this # How many builds are already currently executing for this
# system type? # system type?
my $nrActive = $db->resultset('Builds')->search( my $nrActive = $db->resultset('Builds')->search(
{finished => 0, busy => 1, system => $system->system}, {finished => 0, busy => 1, system => $system->system})->count;
{join => 'schedulingInfo'})->count;
# How many extra builds can we start? # How many extra builds can we start?
(my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system}); (my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system});
@ -100,7 +98,7 @@ sub checkBuilds {
# Select the highest-priority builds to start. # Select the highest-priority builds to start.
my @builds = $extraAllowed == 0 ? () : $db->resultset('Builds')->search( my @builds = $extraAllowed == 0 ? () : $db->resultset('Builds')->search(
{ finished => 0, busy => 0, system => $system->system, enabled => 1, disabled => 0 }, { finished => 0, busy => 0, system => $system->system, enabled => 1, disabled => 0 },
{ join => ['schedulingInfo', 'project'], order_by => ["priority DESC", "timestamp"], { join => ['project'], order_by => ["priority DESC", "timestamp"],
rows => $extraAllowed }); rows => $extraAllowed });
print "system type `", $system->system, print "system type `", $system->system,
@ -114,11 +112,11 @@ sub checkBuilds {
my $logfile = getcwd . "/logs/" . $build->id; my $logfile = getcwd . "/logs/" . $build->id;
mkdir(dirname $logfile); mkdir(dirname $logfile);
unlink($logfile); unlink($logfile);
$build->schedulingInfo->busy(1); $build->busy(1);
$build->schedulingInfo->locker($$); $build->locker($$);
$build->schedulingInfo->logfile($logfile); $build->logfile($logfile);
$build->schedulingInfo->starttime(time); $build->starttime(time);
$build->schedulingInfo->update; $build->update;
push @buildsStarted, $build; push @buildsStarted, $build;
} }
} }
@ -130,7 +128,7 @@ sub checkBuilds {
my $id = $build->id; my $id = $build->id;
print "starting build $id (", $build->project->name, ":", $build->jobset->name, ':', $build->job->name, ") on ", $build->system, "\n"; print "starting build $id (", $build->project->name, ":", $build->jobset->name, ':', $build->job->name, ") on ", $build->system, "\n";
eval { eval {
my $logfile = $build->schedulingInfo->logfile; my $logfile = $build->logfile;
my $child = fork(); my $child = fork();
die unless defined $child; die unless defined $child;
if ($child == 0) { if ($child == 0) {
@ -147,9 +145,9 @@ sub checkBuilds {
if ($@) { if ($@) {
warn $@; warn $@;
txn_do($db, sub { txn_do($db, sub {
$build->schedulingInfo->busy(0); $build->busy(0);
$build->schedulingInfo->locker($$); $build->locker($$);
$build->schedulingInfo->update; $build->update;
}); });
} }
} }

2
src/script/hydra-update-gc-roots
View file

@ -112,7 +112,7 @@ keepBuild $_ foreach @buildsToKeep;
# For scheduled builds, we register the derivation as a GC root. # For scheduled builds, we register the derivation as a GC root.
print STDERR "*** looking for scheduled builds\n"; print STDERR "*** looking for scheduled builds\n";
foreach my $build ($db->resultset('Builds')->search({finished => 0}, {join => 'schedulingInfo'})) { foreach my $build ($db->resultset('Builds')->search({finished => 0})) {
if (isValidPath($build->drvpath)) { if (isValidPath($build->drvpath)) {
print STDERR "keeping scheduled build ", $build->id, " (", print STDERR "keeping scheduled build ", $build->id, " (",
strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n"; strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n";

27
src/sql/hydra.sql
View file

@ -114,11 +114,6 @@ create table Jobs (
); );
-- This table contains all wbuilds, either scheduled or finished. For
-- scheduled builds, additional info (such as the priority) can be
-- found in the BuildSchedulingInfo table. For finished builds,
-- additional info (such as the logs, build products, etc.) can be
-- found in several tables, such as BuildResultInfo and BuildProducts.
create table Builds ( create table Builds (
#ifdef POSTGRESQL #ifdef POSTGRESQL
id serial primary key not null, id serial primary key not null,
@ -156,29 +151,22 @@ create table Builds (
-- build. -- build.
nixExprInput text, nixExprInput text,
nixExprPath text, nixExprPath text,
foreign key (project) references Projects(name) on update cascade,
foreign key (project, jobset) references Jobsets(project, name) on update cascade,
foreign key (project, jobset, job) references Jobs(project, jobset, name) on update cascade
);
-- Information about scheduled builds.
-- Info for a scheduled build.
create table BuildSchedulingInfo (
id integer primary key not null,
priority integer not null default 0, priority integer not null default 0,
busy integer not null default 0, -- true means someone is building this job now busy integer not null default 0, -- true means someone is building this job now
locker text not null default '', -- !!! hostname/pid of the process building this job? locker text, -- !!! hostname/pid of the process building this job?
logfile text, -- if busy, the path of the logfile logfile text, -- if busy, the path of the logfile
disabled integer not null default 0, disabled integer not null default 0, -- !!! boolean
startTime integer, -- if busy, time we started startTime integer, -- if busy, time we started
foreign key (id) references Builds(id) on delete cascade foreign key (project) references Projects(name) on update cascade,
foreign key (project, jobset) references Jobsets(project, name) on update cascade,
foreign key (project, jobset, job) references Jobs(project, jobset, name) on update cascade
); );
@ -523,7 +511,6 @@ create index IndexBuildInputsOnBuild on BuildInputs(build);
create index IndexBuildInputsOnDependency on BuildInputs(dependency); create index IndexBuildInputsOnDependency on BuildInputs(dependency);
create index IndexBuildProducstOnBuildAndType on BuildProducts(build, type); create index IndexBuildProducstOnBuildAndType on BuildProducts(build, type);
create index IndexBuildProductsOnBuild on BuildProducts(build); create index IndexBuildProductsOnBuild on BuildProducts(build);
create index IndexBuildSchedulingInfoOnBuild on BuildSchedulingInfo(id); -- idem
create index IndexBuildStepsOnBuild on BuildSteps(build); create index IndexBuildStepsOnBuild on BuildSteps(build);
create index IndexBuildStepsOnDrvpathTypeBusyStatus on BuildSteps(drvpath, type, busy, status); create index IndexBuildStepsOnDrvpathTypeBusyStatus on BuildSteps(drvpath, type, busy, status);
create index IndexBuildStepsOnOutpath on BuildSteps(outpath); create index IndexBuildStepsOnOutpath on BuildSteps(outpath);

26
src/sql/upgrade-2.sql Normal file
View file

@ -0,0 +1,26 @@
alter table Builds
add column priority integer not null default 0,
add column busy integer not null default 0,
add column locker text,
add column logfile text,
add column disabled integer not null default 0,
add column startTime integer;
--alter table Builds
-- add column isCachedBuild integer,
-- add column buildStatus integer,
-- add column errorMsg text;
update Builds b set
priority = (select priority from BuildSchedulingInfo s where s.id = b.id),
busy = (select busy from BuildSchedulingInfo s where s.id = b.id),
disabled = (select disabled from BuildSchedulingInfo s where s.id = b.id),
locker = (select locker from BuildSchedulingInfo s where s.id = b.id),
logfile = (select logfile from BuildSchedulingInfo s where s.id = b.id)
where exists (select 1 from BuildSchedulingInfo s where s.id = b.id);
update Builds b set
startTime = ((select startTime from BuildSchedulingInfo s where s.id = b.id) union (select startTime from BuildResultInfo r where r.id = b.id));
-- isCachedBuild = (select isCachedBuild from BuildResultInfo r where r.id = b.id),
-- buildStatus = (select buildStatus from BuildResultInfo r where r.id = b.id),
-- errorMsg = (select errorMsg from BuildResultInfo r where r.id = b.id);