From 452c8e36d17f33bafbdc540593cb8e381d40e843 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Aug 2013 20:11:34 +0200 Subject: [PATCH] Materialize the number of finished builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NrBuilds table tracks the value of ‘select count(*) from Builds where finished = 0’, keeping it up to date via a trigger. This is necessary to make the /all page fast, since otherwise it needs to do a sequential scan on the Builds table. --- src/lib/Hydra/Base/Controller/ListBuilds.pm | 5 +- src/lib/Hydra/Controller/Root.pm | 1 + src/lib/Hydra/Schema/NrBuilds.pm | 75 +++++++++++++++++++++ src/script/hydra-init | 4 +- src/sql/hydra.sql | 32 ++++++++- src/sql/upgrade-17.sql | 23 +++++++ tests/query-all-tables.pl | 4 +- 7 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 src/lib/Hydra/Schema/NrBuilds.pm create mode 100644 src/sql/upgrade-17.sql diff --git a/src/lib/Hydra/Base/Controller/ListBuilds.pm b/src/lib/Hydra/Base/Controller/ListBuilds.pm index 6dce9c0d..59163e77 100644 --- a/src/lib/Hydra/Base/Controller/ListBuilds.pm +++ b/src/lib/Hydra/Base/Controller/ListBuilds.pm @@ -56,13 +56,12 @@ sub all : Chained('get_builds') PathPart { my $resultsPerPage = 20; - my $nrBuilds = $c->stash->{allBuilds}->search({finished => 1})->count; - $c->stash->{baseUri} = $c->uri_for($self->action_for("all"), $c->req->captures); $c->stash->{page} = $page; $c->stash->{resultsPerPage} = $resultsPerPage; - $c->stash->{total} = $nrBuilds; + $c->stash->{total} = $c->stash->{allBuilds}->search({finished => 1})->count + unless defined $c->stash->{total}; $c->stash->{builds} = [ $c->stash->{allBuilds}->search( { finished => 1 }, diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index cde5fb10..1ddd6a0c 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -155,6 +155,7 @@ sub get_builds : Chained('/') PathPart('') CaptureArgs(0) { $c->stash->{allJobs} = $c->model('DB::Jobs'); $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceeded'); $c->stash->{channelBaseName} = "everything"; + $c->stash->{total} = $c->model('DB::NrBuilds')->find('finished')->count; } diff --git a/src/lib/Hydra/Schema/NrBuilds.pm b/src/lib/Hydra/Schema/NrBuilds.pm new file mode 100644 index 00000000..27ae2e83 --- /dev/null +++ b/src/lib/Hydra/Schema/NrBuilds.pm @@ -0,0 +1,75 @@ +use utf8; +package Hydra::Schema::NrBuilds; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::NrBuilds + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("NrBuilds"); + +=head1 ACCESSORS + +=head2 what + + data_type: 'text' + is_nullable: 0 + +=head2 count + + data_type: 'integer' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "what", + { data_type => "text", is_nullable => 0 }, + "count", + { data_type => "integer", is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("what"); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-12 17:59:18 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CK8eJGC803nGj0wnete9xg + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/src/script/hydra-init b/src/script/hydra-init index 25abde15..33c60408 100755 --- a/src/script/hydra-init +++ b/src/script/hydra-init @@ -51,12 +51,12 @@ for (my $n = $schemaVersion; $n < $maxSchemaVersion; $n++) { my @statements = $sql_splitter->split($schema); eval { $dbh->begin_work; - sub run { + sub run_ { my ($stm) = @_; print STDERR "executing SQL statement: $stm\n"; $dbh->do($_); } - run($_) foreach @statements; + run_($_) foreach @statements; $db->resultset('SchemaVersion')->update({version => $m}); $dbh->commit; }; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index c8108d73..4562dc41 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -514,6 +514,36 @@ create table NewsItems ( ); +-- Cache of the number of finished builds. +create table NrBuilds ( + what text primary key not null, + count integer not null +); + +insert into NrBuilds(what, count) values('finished', 0); + +#ifdef POSTGRESQL + +create function modifyNrBuildsFinished() returns trigger as $$ + begin + if ((tg_op = 'INSERT' and new.finished = 1) or + (tg_op = 'UPDATE' and old.finished = 0 and new.finished = 1)) then + update NrBuilds set count = count + 1 where what = 'finished'; + elsif ((tg_op = 'DELETE' and old.finished = 1) or + (tg_op = 'UPDATE' and old.finished = 1 and new.finished = 0)) then + update NrBuilds set count = count - 1 where what = 'finished'; + end if; + return null; + end; +$$ language plpgsql; + +create trigger NrBuildsFinished after insert or update or delete on Builds + for each row + execute procedure modifyNrBuildsFinished(); + +#endif + + -- Some indices. create index IndexBuildInputsOnBuild on BuildInputs(build); @@ -534,7 +564,7 @@ create index IndexBuildsOnJobAndSystem on Builds(project, jobset, job, system); create index IndexBuildsOnJobset on Builds(project, jobset); create index IndexBuildsOnProject on Builds(project); create index IndexBuildsOnTimestamp on Builds(timestamp); -create index IndexBuildsOnJobsetFinishedTimestamp on Builds(project, jobset, finished, timestamp DESC); +create index IndexBuildsOnJobsetFinishedTimestamp on Builds(project, jobset, finished, timestamp DESC); -- obsolete? create index IndexBuildsOnJobFinishedId on builds(project, jobset, job, system, finished, id DESC); create index IndexBuildsOnJobSystemCurrent on Builds(project, jobset, job, system, isCurrent); create index IndexBuildsOnDrvPath on Builds(drvPath); diff --git a/src/sql/upgrade-17.sql b/src/sql/upgrade-17.sql new file mode 100644 index 00000000..bf827f75 --- /dev/null +++ b/src/sql/upgrade-17.sql @@ -0,0 +1,23 @@ +create table NrBuilds ( + what text primary key not null, + count integer not null +); + +create function modifyNrBuildsFinished() returns trigger as $$ + begin + if ((tg_op = 'INSERT' and new.finished = 1) or + (tg_op = 'UPDATE' and old.finished = 0 and new.finished = 1)) then + update NrBuilds set count = count + 1 where what = 'finished'; + elsif ((tg_op = 'DELETE' and old.finished = 1) or + (tg_op = 'UPDATE' and old.finished = 1 and new.finished = 0)) then + update NrBuilds set count = count - 1 where what = 'finished'; + end if; + return null; + end; +$$ language plpgsql; + +create trigger NrBuildsFinished after insert or update or delete on Builds + for each row + execute procedure modifyNrBuildsFinished(); + +insert into NrBuilds(what, count) select 'finished', count(*) from Builds where finished = 1; diff --git a/tests/query-all-tables.pl b/tests/query-all-tables.pl index 55cc779f..fc3234ed 100755 --- a/tests/query-all-tables.pl +++ b/tests/query-all-tables.pl @@ -7,11 +7,11 @@ my $db = Hydra::Model::DB->new; my @sources = $db->schema->sources; my $nrtables = scalar(@sources); -use Test::Simple tests => 43; +use Test::Simple tests => 44; foreach my $source (@sources) { my $title = "Basic select query for $source"; - if ($source eq "SchemaVersion") { + if ($source eq "SchemaVersion" || $source eq "NrBuilds") { ok(scalar($db->resultset($source)->all) == 1, $title); } elsif( $source !~ m/^(LatestSucceeded|JobStatus|ActiveJobs)/) { ok(scalar($db->resultset($source)->all) == 0, $title);