From 25334715f89b7a69a1607401499e3f16d824fd83 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Feb 2012 02:22:49 +0100 Subject: [PATCH] Merge the BuildSchedulingInfo table into the Builds table This simplifies the code and improves performance since it reduces the number of joins. --- src/lib/Hydra/Controller/API.pm | 20 ++-- src/lib/Hydra/Controller/Admin.pm | 12 +- src/lib/Hydra/Controller/Build.pm | 29 +++-- src/lib/Hydra/Controller/Job.pm | 12 +- src/lib/Hydra/Controller/Jobset.pm | 6 +- src/lib/Hydra/Controller/Root.pm | 10 +- src/lib/Hydra/Helper/AddBuilds.pm | 26 ++--- src/lib/Hydra/Helper/CatalystUtils.pm | 4 +- src/lib/Hydra/Helper/Nix.pm | 12 +- src/lib/Hydra/Schema/BuildProducts.pm | 8 +- src/lib/Hydra/Schema/BuildSchedulingInfo.pm | 120 -------------------- src/lib/Hydra/Schema/BuildSteps.pm | 6 - src/lib/Hydra/Schema/Builds.pm | 83 +++++++++----- src/lib/Hydra/Schema/SchemaVersion.pm | 22 +--- src/root/build.tt | 8 +- src/root/common.tt | 12 +- src/script/hydra-build | 10 +- src/script/hydra-queue-runner | 40 ++++--- src/script/hydra-update-gc-roots | 2 +- src/sql/hydra.sql | 27 ++--- src/sql/upgrade-2.sql | 26 +++++ 21 files changed, 194 insertions(+), 301 deletions(-) delete mode 100644 src/lib/Hydra/Schema/BuildSchedulingInfo.pm create mode 100644 src/sql/upgrade-2.sql diff --git a/src/lib/Hydra/Controller/API.pm b/src/lib/Hydra/Controller/API.pm index 9ddc8799..aef8d173 100644 --- a/src/lib/Hydra/Controller/API.pm +++ b/src/lib/Hydra/Controller/API.pm @@ -136,13 +136,11 @@ sub queue : Chained('api') PathPart('queue') Args(0) { my $nr = $c->request->params->{nr} ; error($c, "Parameter not defined!") if !defined $nr; - my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, join => ['schedulingInfo'] , order_by => ["busy DESC", "priority DESC", "timestamp"], '+select' => ['schedulingInfo.priority', 'schedulingInfo.busy'], '+as' => ['priority', 'busy'] }); + my @builds = $c->model('DB::Builds')->search({finished => 0}, {rows => $nr, order_by => ["busy DESC", "priority DESC", "timestamp"]}); - my @list ; - foreach my $b (@builds) { - push @list, buildToHash($b) ; - } - + my @list; + push @list, buildToHash($_) foreach @builds; + $c->stash->{'plain'} = { data => scalar (JSON::Any->objToJson(\@list)) }; @@ -151,18 +149,18 @@ sub queue : Chained('api') PathPart('queue') Args(0) { sub nrqueue : Chained('api') PathPart('nrqueue') Args(0) { my ($self, $c) = @_; - my $nrQueuedBuilds = $c->model('DB::BuildSchedulingInfo')->count(); + my $nrQueuedBuilds = $c->model('DB::Builds')->search({finished => 0})->count(); $c->stash->{'plain'} = { - data => " $nrQueuedBuilds" + data => "$nrQueuedBuilds" }; $c->forward('Hydra::View::Plain'); } sub nrrunning : Chained('api') PathPart('nrrunning') Args(0) { my ($self, $c) = @_; - my $nrRunningBuilds = $c->model('DB::BuildSchedulingInfo')->search({ busy => 1 }, {})->count(); - $c->stash->{'plain'} = { - data => " $nrRunningBuilds" + my $nrRunningBuilds = $c->model('DB::Builds')->search({finished => 0, busy => 1 })->count(); + $c->stash->{'plain'} = { + data => "$nrRunningBuilds" }; $c->forward('Hydra::View::Plain'); } diff --git a/src/lib/Hydra/Controller/Admin.pm b/src/lib/Hydra/Controller/Admin.pm index cfec647e..da32c2ae 100644 --- a/src/lib/Hydra/Controller/Admin.pm +++ b/src/lib/Hydra/Controller/Admin.pm @@ -57,9 +57,9 @@ sub index : Chained('admin') PathPart('') Args(0) { , '+as' => ['idle'] })]; $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( - { 'me.busy' => 1, 'schedulingInfo.busy' => 1 }, - { join => [ 'schedulingInfo', 'build' ] - , order_by => [ 'machine' ] + { finished => 0, 'me.busy' => 1, 'build.busy' => 1, }, + { join => [ 'build' ] + , order_by => [ 'machine', 'stepnr' ] } ) ]; $c->stash->{template} = 'admin.tt'; } @@ -296,13 +296,15 @@ sub machine_disable : Chained('machine') PathPart('disable') Args(0) { sub clear_queue_non_current : Chained('admin') Path('clear-queue-non-current') Args(0) { my ($self, $c) = @_; - $c->model('DB::Builds')->search({iscurrent => 0, busy => 0}, { join => 'schedulingInfo' })->delete_all; + # !!! Mark the builds as cancelled instead. + $c->model('DB::Builds')->search({finished => 0, iscurrent => 0, busy => 0})->delete_all; $c->res->redirect("/admin"); } sub clear_queue : Chained('admin') Path('clear-queue') Args(0) { my ($self, $c) = @_; - $c->model('DB::Builds')->search({busy => 0}, { join => 'schedulingInfo' })->delete_all; + # !!! Mark the builds as cancelled instead. + $c->model('DB::Builds')->search({finished => 0, busy => 0})->delete_all; $c->res->redirect("/admin"); } diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index d09e6129..f1e2eb86 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -41,11 +41,10 @@ sub view_build : Chained('build') PathPart('') Args(0) { $c->stash->{drvAvailable} = isValidPath $build->drvpath; $c->stash->{flashMsg} = $c->flash->{buildMsg}; - my $pathHash = $c->stash->{available} ? queryPathHash($build->outpath) : "Not available"; - $c->stash->{pathHash} = $pathHash; + $c->stash->{pathHash} = $c->stash->{available} ? queryPathHash($build->outpath) : undef; - if (!$build->finished && $build->schedulingInfo->busy) { - my $logfile = $build->schedulingInfo->logfile; + if (!$build->finished && $build->busy) { + my $logfile = $build->logfile; $c->stash->{logtext} = `cat $logfile` if defined $logfile && -e $logfile; } @@ -81,13 +80,13 @@ sub view_build : Chained('build') PathPart('') Args(0) { ]; } - my $r = joinWithResultInfo( $c, $c->model('DB::Builds'))->search( - { eval => { -in => $build->jobsetevalmembers->get_column('eval')->as_query } } - , { join => 'jobsetevalmembers', order_by => [ 'project', 'jobset', 'job'], distinct => 1 } - ); - if ($r->count <= 100) { - $c->stash->{relatedbuilds} = [$r->all]; - } + #my $r = joinWithResultInfo( $c, $c->model('DB::Builds'))->search( + # { eval => { -in => $build->jobsetevalmembers->all->get_column('eval')->as_query } } + # , { join => 'jobsetevalmembers', order_by => [ 'project', 'jobset', 'job'], distinct => 1 } + # ); + #if ($r->count <= 100) { + # $c->stash->{relatedbuilds} = [$r->all]; + #} } @@ -141,7 +140,7 @@ sub showLog { my $url = $c->request->uri->as_string; $url =~ s/tail-reload/tail/g; $c->stash->{url} = $url; - $c->stash->{reload} = defined $c->stash->{build}->schedulingInfo && $c->stash->{build}->schedulingInfo->busy; + $c->stash->{reload} = !$c->stash->{build}->finished && $c->stash->{build}->busy; $c->stash->{title} = ""; $c->stash->{contents} = (scalar `$pipestart | tail -n 50`) || " "; $c->stash->{template} = 'plain-reload.tt'; @@ -406,21 +405,19 @@ sub cancel : Chained('build') PathPart Args(0) { txn_do($c->model('DB')->schema, sub { error($c, "This build cannot be cancelled.") - if $build->finished || $build->schedulingInfo->busy; + if $build->finished || $build->busy; # !!! Actually, it would be nice to be able to cancel busy # builds as well, but we would have to send a signal or # something to the build process. - $build->update({finished => 1, timestamp => time}); + $build->update({finished => 1, busy => 0, timestamp => time}); $c->model('DB::BuildResultInfo')->create( { id => $build->id , iscachedbuild => 0 , buildstatus => 4 # = cancelled }); - - $build->schedulingInfo->delete; }); $c->flash->{buildMsg} = "Build has been cancelled."; diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 59ae2664..4c4d3e3d 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -36,9 +36,15 @@ sub overview : Chained('job') PathPart('') Args(0) { , order_by => 'timestamp DESC', rows => 10 }) ]; - $c->stash->{runningBuilds} = [$c->stash->{job}->builds->search({busy => 1}, { join => ['schedulingInfo', 'project'] , order_by => ["priority DESC", "timestamp"] - , '+select' => ['project.enabled', 'schedulingInfo.priority', 'schedulingInfo.disabled', 'schedulingInfo.busy'] - , '+as' => ['enabled', 'priority', 'disabled', 'busy'] })]; + $c->stash->{runningBuilds} = [ + $c->stash->{job}->builds->search( + { busy => 1 }, + { join => ['project'] + , order_by => ["priority DESC", "timestamp"] + , '+select' => ['project.enabled'] + , '+as' => ['enabled'] + } + ) ]; $c->stash->{systems} = [$c->stash->{job}->builds->search({iscurrent => 1}, {select => ["system"], distinct => 1})]; } diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index e90570dd..ff6e11d9 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -65,10 +65,10 @@ sub jobsetIndex { my @as = (); push(@select, "job"); push(@as, "job"); foreach my $system (@systems) { - push(@select, "(SELECT buildstatus FROM BuildResultInfo bri NATURAL JOIN Builds b WHERE b.id = (SELECT MAX(id) FROM Builds t WHERE t.project = me.project AND t.jobset = me.jobset AND t.job = me.job AND t.system = '$system' AND t.iscurrent = 1 ))"); + push(@select, "(select buildstatus from BuildResultInfo bri join Builds b using (id) where b.id = (select max(id) from Builds t where t.project = me.project and t.jobset = me.jobset and t.job = me.job and t.system = '$system' and t.iscurrent = 1 ))"); push(@as, $system); - push(@select, "(SELECT b.id FROM BuildResultInfo bri NATURAL JOIN Builds b WHERE b.id = (SELECT MAX(id) FROM Builds t WHERE t.project = me.project AND t.jobset = me.jobset AND t.job = me.job AND t.system = '$system' AND t.iscurrent = 1 ))"); - push(@as, $system."-build"); + push(@select, "(select b.id from BuildResultInfo bri join Builds b using (id) where b.id = (select max(id) from Builds t where t.project = me.project and t.jobset = me.jobset and t.job = me.job and t.system = '$system' and t.iscurrent = 1 ))"); + push(@as, "$system-build"); } $c->stash->{activeJobsStatus} = [ $c->model('DB')->resultset('ActiveJobsForJobset')->search( diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 14c5a0b6..de0d29f5 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -22,8 +22,8 @@ sub begin :Private { $c->stash->{tracker} = $ENV{"HYDRA_TRACKER"} ; if (scalar(@args) == 0 || $args[0] ne "static") { - $c->stash->{nrRunningBuilds} = $c->model('DB::BuildSchedulingInfo')->search({ busy => 1 }, {})->count(); - $c->stash->{nrQueuedBuilds} = $c->model('DB::BuildSchedulingInfo')->count(); + $c->stash->{nrRunningBuilds} = $c->model('DB::Builds')->search({ finished => 0, busy => 1 }, {})->count(); + $c->stash->{nrQueuedBuilds} = $c->model('DB::Builds')->search({ finished => 0 })->count(); } } @@ -74,7 +74,7 @@ sub queue :Local { my ($self, $c) = @_; $c->stash->{template} = 'queue.tt'; $c->stash->{queue} = [$c->model('DB::Builds')->search( - {finished => 0}, {join => ['schedulingInfo', 'project'] , order_by => ["priority DESC", "timestamp"], '+select' => ['project.enabled', 'schedulingInfo.priority', 'schedulingInfo.disabled', 'schedulingInfo.busy'], '+as' => ['enabled', 'priority', 'disabled', 'busy'] })]; + {finished => 0}, {join => ['project'] , order_by => ["priority DESC", "timestamp"], '+select' => ['project.enabled'], '+as' => ['enabled'] })]; $c->stash->{flashMsg} = $c->flash->{buildMsg}; } @@ -99,8 +99,8 @@ sub timeline :Local { sub status :Local { my ($self, $c) = @_; $c->stash->{steps} = [ $c->model('DB::BuildSteps')->search( - { 'me.busy' => 1, 'schedulingInfo.busy' => 1 }, - { join => [ 'schedulingInfo', 'build' ] + { 'me.busy' => 1, 'build.busy' => 1 }, + { join => [ 'build' ] , order_by => [ 'machine' ] } ) ]; } diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index f05c9632..52fae73b 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -851,12 +851,11 @@ sub checkBuild { , nixexprpath => $jobset->nixexprpath }); - $currentBuilds->{$build->id} = 1; if (isValidPath($outPath)) { print STDERR "marked as cached build ", $build->id, "\n"; - $build->update({ finished => 1 }); + $build->update({ finished => 1 }); $build->create_related('buildresultinfo', { iscachedbuild => 1 , buildstatus => 0 @@ -869,8 +868,8 @@ sub checkBuild { addBuildProducts($db, $build); } else { print STDERR "added to queue as build ", $build->id, "\n"; - $build->create_related('buildschedulinginfo', - { priority => $priority + $build->update( + { priority => $priority , busy => 0 , locker => "" }); @@ -906,24 +905,23 @@ sub restartBuild { my ($db, $build) = @_; txn_do($db, sub { - my $drvpath = $build->drvpath ; - my $outpath = $build->outpath ; + my $drvpath = $build->drvpath; + my $outpath = $build->outpath; my $paths = ""; foreach my $bs ($build->buildsteps) { - $paths = $paths . " " . $bs->outpath; + $paths = $paths . " " . $bs->outpath; } my $r = `nix-store --clear-failed-paths $paths $outpath`; - $build->update({finished => 0, timestamp => time}); - $build->resultInfo->delete; - - $db->resultset('BuildSchedulingInfo')->create( - { id => $build->id - , priority => 0 # don't know the original priority anymore... + $build->update( + { finished => 0 + , timestamp => time , busy => 0 , locker => "" - }); + }); + + $build->resultInfo->delete; }); } diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index e5dcdc82..91c307f3 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -82,9 +82,7 @@ sub getBuildStats { $c->stash->{scheduledBuilds} = $builds->search({finished => 0}) || 0; - $c->stash->{busyBuilds} = $builds->search( - {finished => 0, busy => 1}, - {join => 'schedulingInfo'}) || 0; + $c->stash->{busyBuilds} = $builds->search({finished => 0, busy => 1}) || 0; my $res; $res = $builds->search({}, diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 2a699455..1f6a993c 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -195,13 +195,13 @@ sub jobsetOverview { return $project->jobsets->search( isProjectOwner($c, $project) ? {} : { hidden => 0 }, { order_by => "name" , "+select" => - [ "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildSchedulingInfo WHERE me.project = a.project AND me.name = a.jobset AND a.isCurrent = 1)" - , "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildResultInfo WHERE me.project = a.project AND me.name = a.jobset AND buildstatus <> 0 AND a.isCurrent = 1)" - , "(SELECT COUNT(*) FROM Builds AS a NATURAL JOIN BuildResultInfo WHERE me.project = a.project AND me.name = a.jobset AND buildstatus = 0 AND a.isCurrent = 1)" - , "(SELECT COUNT(*) FROM Builds AS a WHERE me.project = a.project AND me.name = a.jobset AND a.isCurrent = 1)" + [ "(select count(*) from Builds as a where a.finished = 0 and me.project = a.project and me.name = a.jobset and a.isCurrent = 1)" + , "(select count(*) from Builds as a join BuildResultInfo r using (id) where me.project = a.project and me.name = a.jobset and buildstatus <> 0 and a.isCurrent = 1)" + , "(select count(*) from Builds as a join BuildResultInfo r using (id) where me.project = a.project and me.name = a.jobset and buildstatus = 0 and a.isCurrent = 1)" + , "(select count(*) from Builds as a where me.project = a.project and me.name = a.jobset and a.isCurrent = 1)" ] - , "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"] - }); + , "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"] + }); } sub getViewResult { diff --git a/src/lib/Hydra/Schema/BuildProducts.pm b/src/lib/Hydra/Schema/BuildProducts.pm index 6b531937..e7a16c64 100644 --- a/src/lib/Hydra/Schema/BuildProducts.pm +++ b/src/lib/Hydra/Schema/BuildProducts.pm @@ -46,7 +46,7 @@ __PACKAGE__->table("BuildProducts"); =head2 filesize - data_type: 'integer' + data_type: 'bigint' is_nullable: 1 =head2 sha1hash @@ -91,7 +91,7 @@ __PACKAGE__->add_columns( "subtype", { data_type => "text", is_nullable => 0 }, "filesize", - { data_type => "integer", is_nullable => 1 }, + { data_type => "bigint", is_nullable => 1 }, "sha1hash", { data_type => "text", is_nullable => 1 }, "sha256hash", @@ -133,8 +133,8 @@ Related object: L __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {}); -# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:suSgQkBLXzu0yD4YicRS1A +# Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:18 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dzTKwZ7bby7kplnSgta3Gw # You can replace this text with custom content, and it will be preserved on regeneration 1; diff --git a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm b/src/lib/Hydra/Schema/BuildSchedulingInfo.pm deleted file mode 100644 index db622325..00000000 --- a/src/lib/Hydra/Schema/BuildSchedulingInfo.pm +++ /dev/null @@ -1,120 +0,0 @@ -use utf8; -package Hydra::Schema::BuildSchedulingInfo; - -# Created by DBIx::Class::Schema::Loader -# DO NOT MODIFY THE FIRST PART OF THIS FILE - -=head1 NAME - -Hydra::Schema::BuildSchedulingInfo - -=cut - -use strict; -use warnings; - -use base 'DBIx::Class::Core'; - -=head1 TABLE: C - -=cut - -__PACKAGE__->table("BuildSchedulingInfo"); - -=head1 ACCESSORS - -=head2 id - - data_type: 'integer' - is_auto_increment: 1 - is_foreign_key: 1 - is_nullable: 0 - -=head2 priority - - data_type: 'integer' - default_value: 0 - is_nullable: 0 - -=head2 busy - - data_type: 'integer' - default_value: 0 - is_nullable: 0 - -=head2 locker - - data_type: 'text' - default_value: (empty string) - is_nullable: 0 - -=head2 logfile - - data_type: 'text' - is_nullable: 1 - -=head2 disabled - - data_type: 'integer' - default_value: 0 - is_nullable: 0 - -=head2 starttime - - data_type: 'integer' - is_nullable: 1 - -=cut - -__PACKAGE__->add_columns( - "id", - { - data_type => "integer", - is_auto_increment => 1, - is_foreign_key => 1, - is_nullable => 0, - }, - "priority", - { data_type => "integer", default_value => 0, is_nullable => 0 }, - "busy", - { data_type => "integer", default_value => 0, is_nullable => 0 }, - "locker", - { data_type => "text", default_value => "", is_nullable => 0 }, - "logfile", - { data_type => "text", is_nullable => 1 }, - "disabled", - { data_type => "integer", default_value => 0, is_nullable => 0 }, - "starttime", - { data_type => "integer", is_nullable => 1 }, -); - -=head1 PRIMARY KEY - -=over 4 - -=item * L - -=back - -=cut - -__PACKAGE__->set_primary_key("id"); - -=head1 RELATIONS - -=head2 id - -Type: belongs_to - -Related object: L - -=cut - -__PACKAGE__->belongs_to("id", "Hydra::Schema::Builds", { id => "id" }, {}); - - -# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Uz7y9Ly+ADRrtrPfEk9lGA - -# You can replace this text with custom content, and it will be preserved on regeneration -1; diff --git a/src/lib/Hydra/Schema/BuildSteps.pm b/src/lib/Hydra/Schema/BuildSteps.pm index 393f1845..39a54222 100644 --- a/src/lib/Hydra/Schema/BuildSteps.pm +++ b/src/lib/Hydra/Schema/BuildSteps.pm @@ -151,10 +151,4 @@ __PACKAGE__->belongs_to("build", "Hydra::Schema::Builds", { id => "build" }, {}) # Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:5H+OkGT0zQEWkAjU+OlBdg -__PACKAGE__->belongs_to( - "schedulingInfo", - "Hydra::Schema::BuildSchedulingInfo", - { id => "build" }, -); - 1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 6fbd4f86..96cd8bd6 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -130,6 +130,39 @@ __PACKAGE__->table("Builds"); data_type: 'text' is_nullable: 1 +=head2 priority + + data_type: 'integer' + default_value: 0 + is_nullable: 0 + +=head2 busy + + data_type: 'integer' + default_value: 0 + is_nullable: 0 + +=head2 locker + + data_type: 'text' + is_nullable: 1 + +=head2 logfile + + data_type: 'text' + is_nullable: 1 + +=head2 disabled + + data_type: 'integer' + default_value: 0 + is_nullable: 0 + +=head2 starttime + + data_type: 'integer' + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -173,6 +206,18 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 1 }, "nixexprpath", { data_type => "text", is_nullable => 1 }, + "priority", + { data_type => "integer", default_value => 0, is_nullable => 0 }, + "busy", + { data_type => "integer", default_value => 0, is_nullable => 0 }, + "locker", + { data_type => "text", is_nullable => 1 }, + "logfile", + { data_type => "text", is_nullable => 1 }, + "disabled", + { data_type => "integer", default_value => 0, is_nullable => 0 }, + "starttime", + { data_type => "integer", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -249,21 +294,6 @@ __PACKAGE__->might_have( {}, ); -=head2 buildschedulinginfo - -Type: might_have - -Related object: L - -=cut - -__PACKAGE__->might_have( - "buildschedulinginfo", - "Hydra::Schema::BuildSchedulingInfo", - { "foreign.id" => "self.id" }, - {}, -); - =head2 buildsteps Type: has_many @@ -350,8 +380,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RRtBPTdD946kA5133+c4kw +# Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:54 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:VnnyFTwnLncGb2Dj2/giiA use Hydra::Helper::Nix; @@ -369,12 +399,6 @@ __PACKAGE__->has_many( { "foreign.build" => "self.id" }, ); -__PACKAGE__->belongs_to( - "schedulingInfo", - "Hydra::Schema::BuildSchedulingInfo", - { id => "id" }, -); - __PACKAGE__->belongs_to( "resultInfo", "Hydra::Schema::BuildResultInfo", @@ -410,11 +434,11 @@ sub makeQueries { my $joinWithStatusChange = < c.id and @@ -432,11 +456,12 @@ QUERY x.id, x.finished, x.timestamp, x.project, x.jobset, x.job, x.nixname, x.description, x.drvpath, x.outpath, x.system, x.longdescription, x.license, x.homepage, x.maintainers, x.isCurrent, x.nixExprInput, - x.nixExprPath, x.maxsilent, x.timeout, + x.nixExprPath, x.maxsilent, x.timeout, x.priority, x.busy, x.locker, + x.logfile, x.disabled, x.startTime, b.id as statusChangeId, b.timestamp as statusChangeTime from (select - (select max(id) from builds b + (select max(b.id) from builds b where project = activeJobs.project and jobset = activeJobs.jobset and job = activeJobs.job and system = activeJobs.system @@ -457,7 +482,7 @@ QUERY select * from (select - (select max(id) from builds b + (select max(b.id) from builds b where project = activeJobs.project and jobset = activeJobs.jobset and job = activeJobs.job and system = activeJobs.system diff --git a/src/lib/Hydra/Schema/SchemaVersion.pm b/src/lib/Hydra/Schema/SchemaVersion.pm index 4bc90886..778cc04c 100644 --- a/src/lib/Hydra/Schema/SchemaVersion.pm +++ b/src/lib/Hydra/Schema/SchemaVersion.pm @@ -26,31 +26,15 @@ __PACKAGE__->table("SchemaVersion"); =head2 version data_type: 'integer' - is_auto_increment: 1 is_nullable: 0 =cut -__PACKAGE__->add_columns( - "version", - { data_type => "integer", is_auto_increment => 1, is_nullable => 0 }, -); - -=head1 PRIMARY KEY - -=over 4 - -=item * L - -=back - -=cut - -__PACKAGE__->set_primary_key("version"); +__PACKAGE__->add_columns("version", { data_type => "integer", is_nullable => 0 }); -# Created by DBIx::Class::Schema::Loader v0.07014 @ 2011-12-05 14:15:43 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:F/jsSRq8pxR4mWq/N4qYGw +# Created by DBIx::Class::Schema::Loader v0.07014 @ 2012-02-29 00:47:18 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LFD28W0GvvrOOylCM98SEQ # You can replace this text with custom code or comments, and it will be preserved on regeneration diff --git a/src/root/build.tt b/src/root/build.tt index ee704fb8..2936b114 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -70,7 +70,7 @@

Job [% project.name %]:[% jobset.name %]:[% job.name %] build [% id %] [% IF !build.finished %] - [% IF build.schedulingInfo.busy %] + [% IF build.busy %] (currently building) [% ELSE %] (scheduled) @@ -127,7 +127,7 @@ System: [% build.system %] - [% IF !build.schedulingInfo %] + [% IF !build.finished %] Duration: @@ -316,12 +316,14 @@ [% END %] + [% IF pathHash %] Output store path hash: [% pathHash %] + [% END %] Time added: [% INCLUDE renderDateTime timestamp = build.timestamp %] @@ -366,7 +368,7 @@ [% IF !build.finished %] Priority: - [% build.schedulingInfo.priority %] + [% build.priority %] [% END %] [% IF build.finished && build.buildproducts %] diff --git a/src/root/common.tt b/src/root/common.tt index 245527da..aba91d91 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -93,7 +93,7 @@ [%- FOREACH build IN builds -%] [% build.id %] [%- IF showSchedulingInfo -%] - [% IF build.get_column('busy') %]Running[% ELSIF build.get_column('disabled') == 1 || build.get_column('enabled') == 0 %]Disabled[% END %] - [% build.get_column('priority') %] + [% IF build.busy %]Running[% ELSIF build.disabled == 1 || build.get_column('enabled') == 0 %]Disabled[% END %] + [% build.priority %] [%- END -%] [%- INCLUDE renderFullJobNameOfBuild -%] [% !showSchedulingInfo and build.get_column('releasename') ? build.get_column('releasename') : build.nixname %] @@ -203,7 +203,7 @@ [%- BLOCK renderBuildStatusIcon -%] [%- finished = build != undef ? build.finished : 1 -%] - [%- busy = busy != undef ? busy : build.schedulingInfo.busy -%] + [%- busy = busy != undef ? busy : build.busy -%] [%- buildstatus = buildstatus != undef ? buildstatus : build.resultInfo.buildstatus -%] [%- IF finished -%] [%- IF buildstatus == 0 -%] @@ -255,9 +255,9 @@ [% END %] - [% ELSIF build.schedulingInfo.busy %] + [% ELSIF build.busy %] Build in progress - since [% INCLUDE renderDateTime timestamp = build.schedulingInfo.starttime %] + since [% INCLUDE renderDateTime timestamp = build.starttime %] [% ELSE %] Scheduled to be built [% IF c.user_exists %] diff --git a/src/script/hydra-build b/src/script/hydra-build index d85b2779..9a816033 100755 --- a/src/script/hydra-build +++ b/src/script/hydra-build @@ -398,7 +398,7 @@ sub doBuild { } txn_do($db, sub { - $build->update({finished => 1, timestamp => time}); + $build->update({finished => 1, busy => 0, locker => '', logfile => '', timestamp => time}); my $releaseName = getReleaseName($outPath); @@ -423,8 +423,6 @@ sub doBuild { if ($buildStatus == 0 || $buildStatus == 6) { addBuildProducts($db, $build); } - - $build->schedulingInfo->delete; }); sendEmailNotification $build; @@ -453,10 +451,10 @@ txn_do($db, sub { $build = $db->resultset('Builds')->find($buildId); die "build $buildId doesn't exist" unless defined $build; die "build $buildId already done" if defined $build->resultInfo; - if ($build->schedulingInfo->busy != 0 && $build->schedulingInfo->locker != getppid) { + if ($build->busy != 0 && $build->locker != getppid) { die "build $buildId is already being built"; } - $build->schedulingInfo->update({busy => 1, locker => $$}); + $build->update({busy => 1, locker => $$}); $build->buildsteps->search({busy => 1})->delete_all; $build->buildproducts->delete_all; }); @@ -472,6 +470,6 @@ eval { if ($@) { warn $@; txn_do($db, sub { - $build->schedulingInfo->update({busy => 0, locker => $$}); + $build->update({busy => 0, locker => $$}); }); } diff --git a/src/script/hydra-queue-runner b/src/script/hydra-queue-runner index 4410f6ea..745a990c 100755 --- a/src/script/hydra-queue-runner +++ b/src/script/hydra-queue-runner @@ -20,10 +20,9 @@ STDOUT->autoflush(); sub unlockDeadBuilds { # Unlock builds whose building process has died. txn_do($db, sub { - my @builds = $db->resultset('Builds')->search( - {finished => 0, busy => 1}, {join => 'schedulingInfo'}); + my @builds = $db->resultset('Builds')->search({finished => 0, busy => 1}); foreach my $build (@builds) { - my $pid = $build->schedulingInfo->locker; + my $pid = $build->locker; my $unlock = 0; if ($pid == $$) { # Work around sqlite locking timeouts: if the child @@ -32,7 +31,7 @@ sub unlockDeadBuilds { # So if after a minute it hasn't been updated, # unlock the build. !!! need a better fix for those # locking timeouts. - if ($build->schedulingInfo->starttime + 60 < time) { + if ($build->starttime + 60 < time) { $unlock = 1; } } elsif (kill(0, $pid) != 1) { # see if we can signal the process @@ -40,9 +39,9 @@ sub unlockDeadBuilds { } if ($unlock) { print "build ", $build->id, " pid $pid died, unlocking\n"; - $build->schedulingInfo->busy(0); - $build->schedulingInfo->locker(""); - $build->schedulingInfo->update; + $build->busy(0); + $build->locker(""); + $build->update; } } }); @@ -64,7 +63,7 @@ sub findBuildDependencyInQueue { ($depBuild) = $db->resultset('Builds')->search( { drvpath => [ @drvs ], finished => 0, busy => 0, enabled => 1, disabled => 0 }, - { join => ['schedulingInfo', 'project'], rows => 1 } ) ; + { join => ['project'], rows => 1 } ) ; return $depBuild; } @@ -79,7 +78,7 @@ sub checkBuilds { # Get the system types for the runnable builds. my @systemTypes = $db->resultset('Builds')->search( { finished => 0, busy => 0, enabled => 1, disabled => 0 }, - { join => ['schedulingInfo', 'project'], select => ['system'], as => ['system'], distinct => 1 }); + { join => ['project'], select => ['system'], as => ['system'], distinct => 1 }); # For each system type, select up to the maximum number of # concurrent build for that system type. Choose the highest @@ -88,8 +87,7 @@ sub checkBuilds { # How many builds are already currently executing for this # system type? my $nrActive = $db->resultset('Builds')->search( - {finished => 0, busy => 1, system => $system->system}, - {join => 'schedulingInfo'})->count; + {finished => 0, busy => 1, system => $system->system})->count; # How many extra builds can we start? (my $systemTypeInfo) = $db->resultset('SystemTypes')->search({system => $system->system}); @@ -100,7 +98,7 @@ sub checkBuilds { # Select the highest-priority builds to start. my @builds = $extraAllowed == 0 ? () : $db->resultset('Builds')->search( { finished => 0, busy => 0, system => $system->system, enabled => 1, disabled => 0 }, - { join => ['schedulingInfo', 'project'], order_by => ["priority DESC", "timestamp"], + { join => ['project'], order_by => ["priority DESC", "timestamp"], rows => $extraAllowed }); print "system type `", $system->system, @@ -114,11 +112,11 @@ sub checkBuilds { my $logfile = getcwd . "/logs/" . $build->id; mkdir(dirname $logfile); unlink($logfile); - $build->schedulingInfo->busy(1); - $build->schedulingInfo->locker($$); - $build->schedulingInfo->logfile($logfile); - $build->schedulingInfo->starttime(time); - $build->schedulingInfo->update; + $build->busy(1); + $build->locker($$); + $build->logfile($logfile); + $build->starttime(time); + $build->update; push @buildsStarted, $build; } } @@ -130,7 +128,7 @@ sub checkBuilds { my $id = $build->id; print "starting build $id (", $build->project->name, ":", $build->jobset->name, ':', $build->job->name, ") on ", $build->system, "\n"; eval { - my $logfile = $build->schedulingInfo->logfile; + my $logfile = $build->logfile; my $child = fork(); die unless defined $child; if ($child == 0) { @@ -147,9 +145,9 @@ sub checkBuilds { if ($@) { warn $@; txn_do($db, sub { - $build->schedulingInfo->busy(0); - $build->schedulingInfo->locker($$); - $build->schedulingInfo->update; + $build->busy(0); + $build->locker($$); + $build->update; }); } } diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots index 59061a56..454fa1f2 100755 --- a/src/script/hydra-update-gc-roots +++ b/src/script/hydra-update-gc-roots @@ -112,7 +112,7 @@ keepBuild $_ foreach @buildsToKeep; # For scheduled builds, we register the derivation as a GC root. print STDERR "*** looking for scheduled builds\n"; -foreach my $build ($db->resultset('Builds')->search({finished => 0}, {join => 'schedulingInfo'})) { +foreach my $build ($db->resultset('Builds')->search({finished => 0})) { if (isValidPath($build->drvpath)) { print STDERR "keeping scheduled build ", $build->id, " (", strftime("%Y-%m-%d %H:%M:%S", localtime($build->timestamp)), ")\n"; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 6f4a8254..7cac6678 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -114,11 +114,6 @@ create table Jobs ( ); --- This table contains all wbuilds, either scheduled or finished. For --- scheduled builds, additional info (such as the priority) can be --- found in the BuildSchedulingInfo table. For finished builds, --- additional info (such as the logs, build products, etc.) can be --- found in several tables, such as BuildResultInfo and BuildProducts. create table Builds ( #ifdef POSTGRESQL id serial primary key not null, @@ -156,29 +151,22 @@ create table Builds ( -- build. nixExprInput text, nixExprPath text, - - foreign key (project) references Projects(name) on update cascade, - foreign key (project, jobset) references Jobsets(project, name) on update cascade, - foreign key (project, jobset, job) references Jobs(project, jobset, name) on update cascade -); - --- Info for a scheduled build. -create table BuildSchedulingInfo ( - id integer primary key not null, - + -- Information about scheduled builds. priority integer not null default 0, busy integer not null default 0, -- true means someone is building this job now - locker text not null default '', -- !!! hostname/pid of the process building this job? + locker text, -- !!! hostname/pid of the process building this job? logfile text, -- if busy, the path of the logfile - disabled integer not null default 0, + disabled integer not null default 0, -- !!! boolean startTime integer, -- if busy, time we started - - foreign key (id) references Builds(id) on delete cascade + + foreign key (project) references Projects(name) on update cascade, + foreign key (project, jobset) references Jobsets(project, name) on update cascade, + foreign key (project, jobset, job) references Jobs(project, jobset, name) on update cascade ); @@ -523,7 +511,6 @@ create index IndexBuildInputsOnBuild on BuildInputs(build); create index IndexBuildInputsOnDependency on BuildInputs(dependency); create index IndexBuildProducstOnBuildAndType on BuildProducts(build, type); create index IndexBuildProductsOnBuild on BuildProducts(build); -create index IndexBuildSchedulingInfoOnBuild on BuildSchedulingInfo(id); -- idem create index IndexBuildStepsOnBuild on BuildSteps(build); create index IndexBuildStepsOnDrvpathTypeBusyStatus on BuildSteps(drvpath, type, busy, status); create index IndexBuildStepsOnOutpath on BuildSteps(outpath); diff --git a/src/sql/upgrade-2.sql b/src/sql/upgrade-2.sql new file mode 100644 index 00000000..75df66f4 --- /dev/null +++ b/src/sql/upgrade-2.sql @@ -0,0 +1,26 @@ +alter table Builds + add column priority integer not null default 0, + add column busy integer not null default 0, + add column locker text, + add column logfile text, + add column disabled integer not null default 0, + add column startTime integer; + +--alter table Builds +-- add column isCachedBuild integer, +-- add column buildStatus integer, +-- add column errorMsg text; + +update Builds b set + priority = (select priority from BuildSchedulingInfo s where s.id = b.id), + busy = (select busy from BuildSchedulingInfo s where s.id = b.id), + disabled = (select disabled from BuildSchedulingInfo s where s.id = b.id), + locker = (select locker from BuildSchedulingInfo s where s.id = b.id), + logfile = (select logfile from BuildSchedulingInfo s where s.id = b.id) + where exists (select 1 from BuildSchedulingInfo s where s.id = b.id); + +update Builds b set + startTime = ((select startTime from BuildSchedulingInfo s where s.id = b.id) union (select startTime from BuildResultInfo r where r.id = b.id)); +-- isCachedBuild = (select isCachedBuild from BuildResultInfo r where r.id = b.id), +-- buildStatus = (select buildStatus from BuildResultInfo r where r.id = b.id), +-- errorMsg = (select errorMsg from BuildResultInfo r where r.id = b.id);