From 16a84f4bf5ce0a3dfc925bf903bfbce34e514460 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Apr 2009 15:37:21 +0000 Subject: [PATCH] * Big speed-up of the job status page and the channel generation (such as the manifest). The builds are now determined in one SQL query rather than a zillion ones. --- src/lib/Hydra/Base/Controller/ListBuilds.pm | 14 +++----- src/lib/Hydra/Base/Controller/NixChannel.pm | 2 +- src/lib/Hydra/Controller/Job.pm | 7 ++-- src/lib/Hydra/Controller/Jobset.pm | 5 ++- src/lib/Hydra/Controller/Project.pm | 5 ++- src/lib/Hydra/Controller/Root.pm | 3 +- src/lib/Hydra/Helper/CatalystUtils.pm | 39 ++++++++------------- src/lib/Hydra/Schema/Builds.pm | 22 ++++++++++++ src/root/channel-contents.tt | 2 +- src/sql/hydra.sql | 2 ++ 10 files changed, 60 insertions(+), 41 deletions(-) diff --git a/src/lib/Hydra/Base/Controller/ListBuilds.pm b/src/lib/Hydra/Base/Controller/ListBuilds.pm index 0a80760c..b187e97c 100644 --- a/src/lib/Hydra/Base/Controller/ListBuilds.pm +++ b/src/lib/Hydra/Base/Controller/ListBuilds.pm @@ -10,9 +10,8 @@ use Hydra::Helper::CatalystUtils; sub jobstatus : Chained('get_builds') PathPart Args(0) { my ($self, $c) = @_; $c->stash->{template} = 'jobstatus.tt'; - my $jobs = $c->stash->{allJobs}; $c->stash->{latestBuilds} = - getLatestBuilds($c, ref $jobs eq "ARRAY" ? $jobs : scalar $jobs->search({active => 1}), {}); + [joinWithResultInfo($c, $c->stash->{jobStatus})->all]; } @@ -33,12 +32,9 @@ sub all : Chained('get_builds') PathPart { $c->stash->{resultsPerPage} = $resultsPerPage; $c->stash->{totalBuilds} = $nrBuilds; - $c->stash->{builds} = [$c->stash->{allBuilds}->search( + $c->stash->{builds} = [joinWithResultInfo($c, $c->stash->{allBuilds})->search( { finished => 1 }, - { join => 'resultInfo' - , '+select' => ["resultInfo.buildstatus", "resultInfo.releasename"] - , '+as' => ["buildstatus", "releasename"] - , order_by => "timestamp DESC" + { order_by => "timestamp DESC" , rows => $resultsPerPage , page => $page })]; } @@ -49,11 +45,11 @@ sub nix : Chained('get_builds') PathPart('channel') CaptureArgs(1) { eval { if ($channelName eq "latest") { $c->stash->{channelName} = $c->stash->{channelBaseName} . "-latest"; - getChannelData($c, getLatestBuilds($c, scalar $c->stash->{allJobs}->search({active => 1}), {buildStatus => 0})); + getChannelData($c, scalar($c->stash->{latestSucceeded})); } elsif ($channelName eq "all") { $c->stash->{channelName} = $c->stash->{channelBaseName} . "-all"; - getChannelData($c, [$c->stash->{allBuilds}->all]); + getChannelData($c, scalar($c->stash->{allBuilds})); } else { error($c, "Unknown channel `$channelName'."); diff --git a/src/lib/Hydra/Base/Controller/NixChannel.pm b/src/lib/Hydra/Base/Controller/NixChannel.pm index 7c3d06d9..d2755128 100644 --- a/src/lib/Hydra/Base/Controller/NixChannel.pm +++ b/src/lib/Hydra/Base/Controller/NixChannel.pm @@ -65,7 +65,7 @@ sub nixexprs : Chained('nix') PathPart('nixexprs.tar.bz2') Args(0) { sub name { my ($build) = @_; - return $build->resultInfo->releasename || $build->nixname; + return $build->get_column('releasename') || $build->nixname; } diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 748c9ace..8b46de2f 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -30,9 +30,10 @@ sub overview : Chained('job') PathPart('') Args(0) { sub get_builds : Chained('job') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; $c->stash->{allBuilds} = $c->stash->{job}->builds; - #$c->stash->{allJobs} = # !!! horribly hacky - # $c->stash->{job}->jobset->jobs->search({name => $c->stash->{job}->name}); - $c->stash->{allJobs} = [$c->stash->{job}]; + $c->stash->{jobStatus} = $c->model('DB')->resultset('JobStatusForJob') + ->search({}, {bind => [$c->stash->{project}->name, $c->stash->{jobset}->name, $c->stash->{job}->name]}); + $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceededForJob') + ->search({}, {bind => [$c->stash->{project}->name, $c->stash->{jobset}->name, $c->stash->{job}->name]}); $c->stash->{channelBaseName} = $c->stash->{project}->name . "-" . $c->stash->{jobset}->name . "-" . $c->stash->{job}->name; } diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 4cfd8de3..99601894 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -36,7 +36,10 @@ sub index : Chained('jobset') PathPart('') Args(0) { sub get_builds : Chained('jobset') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; $c->stash->{allBuilds} = $c->stash->{jobset}->builds; - $c->stash->{allJobs} = $c->stash->{jobset}->jobs; + $c->stash->{jobStatus} = $c->model('DB')->resultset('JobStatusForJobset') + ->search({}, {bind => [$c->stash->{project}->name, $c->stash->{jobset}->name]}); + $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceededForJobset') + ->search({}, {bind => [$c->stash->{project}->name, $c->stash->{jobset}->name]}); $c->stash->{channelBaseName} = $c->stash->{project}->name . "-" . $c->stash->{jobset}->name; } diff --git a/src/lib/Hydra/Controller/Project.pm b/src/lib/Hydra/Controller/Project.pm index ebe73a0d..1c2c0514 100644 --- a/src/lib/Hydra/Controller/Project.pm +++ b/src/lib/Hydra/Controller/Project.pm @@ -170,7 +170,10 @@ sub updateProject { sub get_builds : Chained('project') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; $c->stash->{allBuilds} = $c->stash->{project}->builds; - $c->stash->{allJobs} = $c->stash->{project}->jobs; + $c->stash->{jobStatus} = $c->model('DB')->resultset('JobStatusForProject') + ->search({}, {bind => [$c->stash->{project}->name]}); + $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceededForProject') + ->search({}, {bind => [$c->stash->{project}->name]}); $c->stash->{channelBaseName} = $c->stash->{project}->name; } diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 3fefd8c2..f730a243 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -225,7 +225,8 @@ sub release :Local { sub get_builds : Chained('/') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; $c->stash->{allBuilds} = $c->model('DB::Builds'); - $c->stash->{allJobs} = $c->model('DB::Jobs'); + $c->stash->{jobStatus} = $c->model('DB')->resultset('JobStatus'); + $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceeded'); $c->stash->{channelBaseName} = "everything"; } diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index 2e9b403c..fffc66f2 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -7,7 +7,7 @@ use Hydra::Helper::Nix; our @ISA = qw(Exporter); our @EXPORT = qw( - getBuild getBuildStats getLatestBuilds getChannelData + getBuild getBuildStats joinWithResultInfo getChannelData error notFound requireLogin requireProjectOwner requireAdmin requirePost trim @@ -43,37 +43,28 @@ sub getBuildStats { } -# Return the latest build for each job. -sub getLatestBuilds { - my ($c, $jobs, $extraAttrs) = @_; +# Add the releaseName and buildStatus attributes from the +# BuildResultInfo table for each build. +sub joinWithResultInfo { + my ($c, $source) = @_; - my @res = (); - - # !!! this could be done more efficiently. - - foreach my $job (ref $jobs eq "ARRAY" ? @{$jobs} : $jobs->all) { - foreach my $system ($job->builds->search({}, {select => ['system'], distinct => 1})) { - my ($build) = $job->builds->search( - { finished => 1, system => $system->system, %$extraAttrs }, - { join => 'resultInfo', order_by => 'timestamp DESC', rows => 1 - , '+select' => ["resultInfo.releasename", "resultInfo.buildstatus"] - , '+as' => ["releasename", "buildstatus"] - }); - push @res, $build if defined $build; - } - } - - return [@res]; + return $source->search( + { }, + { join => 'resultInfo' + , '+select' => ["resultInfo.releasename", "resultInfo.buildstatus"] + , '+as' => ["releasename", "buildstatus"] + }); } sub getChannelData { my ($c, $builds) = @_; + + my @builds2 = joinWithResultInfo($c, $builds) + ->search_literal("exists (select 1 from buildproducts where build = me.id and type = 'nix-build')"); my @storePaths = (); - foreach my $build (@{$builds}) { - # !!! better do this in getLatestBuilds with a join. - next unless $build->buildproducts->find({type => "nix-build"}); + foreach my $build (@builds2) { next unless isValidPath($build->outpath); push @storePaths, $build->outpath; my $pkgName = $build->nixname . "-" . $build->system . "-" . $build->id; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index d3bc72a9..eea2e6d5 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -102,4 +102,26 @@ __PACKAGE__->belongs_to( { id => "id" }, ); + +sub makeSource { + my ($name, $query) = @_; + my $source = __PACKAGE__->result_source_instance(); + my $new_source = $source->new($source); + $new_source->source_name($name); + $new_source->name(\ "($query)"); + Hydra::Schema->register_extra_source($name => $new_source); +} + +sub makeQueries { + my ($name, $constraint) = @_; + makeSource('JobStatus' . $name, "select * from (select project, jobset, job, system, max(timestamp) timestamp from Builds where finished = 1 $constraint group by project, jobset, job, system) natural join Builds"); + makeSource('LatestSucceeded' . $name, "select * from (select project, jobset, job, system, max(timestamp) timestamp from Builds natural join BuildResultInfo where finished = 1 and buildStatus = 0 $constraint group by project, jobset, job, system) natural join Builds"); +} + +makeQueries('', ""); +makeQueries('ForProject', "and project = ?"); +makeQueries('ForJobset', "and project = ? and jobset = ?"); +makeQueries('ForJob', "and project = ? and jobset = ? and job = ?"); + + 1; diff --git a/src/root/channel-contents.tt b/src/root/channel-contents.tt index 90936a16..eee6ddd3 100644 --- a/src/root/channel-contents.tt +++ b/src/root/channel-contents.tt @@ -56,7 +56,7 @@ install the package simply by clicking on the packages below.

[% build.id %] - [% build.resultInfo.releasename || build.nixname %] + [% build.get_column('releasename') || build.nixname %] [% build.system %] [% IF build.homepage -%] diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 2a9a474b..1562e17c 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -405,3 +405,5 @@ create table ReleaseSetJobs ( -- Some indices. create index IndexBuildInputsByBuild on BuildInputs(build); create index IndexBuildInputsByDependency on BuildInputs(dependency); +create index IndexBuildsByTimestamp on Builds(timestamp); +--create index IndexBuildsByJobAndSystem on Builds(project, jobset, job, system);