From 508d99d6113bc0071d0f4fa100fea357beb6c832 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 1 Jun 2021 11:16:47 -0400 Subject: [PATCH 1/4] Join to builds via jobset_id when easy --- src/lib/Hydra/Controller/Build.pm | 3 +-- src/lib/Hydra/Controller/Channel.pm | 3 +-- src/lib/Hydra/Helper/CatalystUtils.pm | 6 ++---- src/script/hydra-eval-jobset | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 79cc4c80..80ca15d2 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -83,8 +83,7 @@ sub build_GET { if ($build->finished) { $c->stash->{prevBuilds} = [$c->model('DB::Builds')->search( - { project => $c->stash->{project}->name - , jobset => $c->stash->{jobset}->name + { jobset_id => $c->stash->{jobset}->id , job => $c->stash->{job} , 'me.system' => $build->system , finished => 1 diff --git a/src/lib/Hydra/Controller/Channel.pm b/src/lib/Hydra/Controller/Channel.pm index 902d6019..67de61ae 100644 --- a/src/lib/Hydra/Controller/Channel.pm +++ b/src/lib/Hydra/Controller/Channel.pm @@ -22,8 +22,7 @@ sub channel : Chained('/') PathPart('channel/custom') CaptureArgs(3) { my $lastSuccessful = $c->model('DB::Builds')->find( { 'eval.hasnewbuilds' => 1 - , project => $projectName - , jobset => $jobsetName + , jobset_id => $c->stash->{jobset}->id, , job => $channelName , buildstatus => 0 }, diff --git a/src/lib/Hydra/Helper/CatalystUtils.pm b/src/lib/Hydra/Helper/CatalystUtils.pm index ec19b61d..53ce53c4 100644 --- a/src/lib/Hydra/Helper/CatalystUtils.pm +++ b/src/lib/Hydra/Helper/CatalystUtils.pm @@ -64,8 +64,7 @@ sub getNextBuild { (my $nextBuild) = $c->model('DB::Builds')->search( { finished => 1 , system => $build->system - , project => $build->get_column('project') - , jobset => $build->get_column('jobset') + , jobset_id => $build->get_column('jobset_id') , job => $build->get_column('job') , 'me.id' => { '>' => $build->id } }, {rows => 1, order_by => "me.id ASC"}); @@ -81,8 +80,7 @@ sub getPreviousSuccessfulBuild { (my $prevBuild) = $c->model('DB::Builds')->search( { finished => 1 , system => $build->system - , project => $build->get_column('project') - , jobset => $build->get_column('jobset') + , jobset_id => $build->get_column('jobset_id') , job => $build->get_column('job') , buildstatus => 0 , 'me.id' => { '<' => $build->id } diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index e6189b3b..77938c85 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -426,7 +426,7 @@ sub checkBuild { # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. - { project => $jobset->get_column('project'), jobset => $jobset->name, job => $jobName, + { jobset_id => $jobset->get_column('id'), job => $jobName, name => $firstOutputName, path => $firstOutputPath }, { rows => 1, columns => ['id'], join => ['buildoutputs'] }); if (defined $prevBuild) { From 719d0a61342d8bfd7ea2d272e32df0b2f106fbaa Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 1 Jun 2021 11:19:33 -0400 Subject: [PATCH 2/4] jobset overview: join jobsets to builds by jobset id --- src/lib/Hydra/Helper/Nix.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 3a7b532e..e198a56c 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -115,10 +115,10 @@ sub jobsetOverview_ { return $jobsets->search({}, { order_by => ["hidden ASC", "enabled DESC", "name"] , "+select" => - [ "(select count(*) from Builds as a where a.finished = 0 and me.project = a.project and me.name = a.jobset and a.isCurrent = 1)" - , "(select count(*) from Builds as a where a.finished = 1 and me.project = a.project and me.name = a.jobset and buildstatus <> 0 and a.isCurrent = 1)" - , "(select count(*) from Builds as a where a.finished = 1 and me.project = a.project and me.name = a.jobset and buildstatus = 0 and a.isCurrent = 1)" - , "(select count(*) from Builds as a where me.project = a.project and me.name = a.jobset and a.isCurrent = 1)" + [ "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 0 and a.isCurrent = 1)" + , "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 1 and buildstatus <> 0 and a.isCurrent = 1)" + , "(select count(*) from Builds as a where me.id = a.jobset_id and a.finished = 1 and buildstatus = 0 and a.isCurrent = 1)" + , "(select count(*) from Builds as a where me.id = a.jobset_id and a.isCurrent = 1)" ] , "+as" => ["nrscheduled", "nrfailed", "nrsucceeded", "nrtotal"] }); From a9e4ede006e35c83db39f071cb8b672b518e26a3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 1 Jun 2021 11:23:22 -0400 Subject: [PATCH 3/4] SQL: create better indexes for builds based on the jobset id These are primarily used by the jobsetOverview renders. --- src/sql/hydra.sql | 3 +++ src/sql/upgrade-75.sql | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 src/sql/upgrade-75.sql diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index a73da0fb..c664d854 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -612,6 +612,9 @@ create index IndexBuildsOnFinished on Builds(finished) where finished = 0; create index IndexBuildsOnIsCurrent on Builds(isCurrent) where isCurrent = 1; create index IndexBuildsOnJobsetIsCurrent on Builds(project, jobset, isCurrent) where isCurrent = 1; create index IndexBuildsOnJobIsCurrent on Builds(project, jobset, job, isCurrent) where isCurrent = 1; +create index IndexBuildsJobsetIdCurrentUnfinished on Builds(jobset_id) where isCurrent = 1 and finished = 0; +create index IndexBuildsJobsetIdCurrentFinishedStatus on Builds(jobset_id, buildstatus) where isCurrent = 1 and finished = 1; +create index IndexBuildsJobsetIdCurrent on Builds(jobset_id) where isCurrent = 1; create index IndexBuildsOnJobset on Builds(project, jobset); create index IndexBuildsOnProject on Builds(project); create index IndexBuildsOnTimestamp on Builds(timestamp); diff --git a/src/sql/upgrade-75.sql b/src/sql/upgrade-75.sql new file mode 100644 index 00000000..ba0e7dbd --- /dev/null +++ b/src/sql/upgrade-75.sql @@ -0,0 +1,5 @@ +-- These take about 9 minutes in total on a replica of hydra.nixos.org + +create index IndexBuildsJobsetIdCurrentUnfinished on Builds(jobset_id) where isCurrent = 1 and finished = 0; +create index IndexBuildsJobsetIdCurrentFinishedStatus on Builds(jobset_id, buildstatus) where isCurrent = 1 and finished = 1; +create index IndexBuildsJobsetIdCurrent on Builds(jobset_id) where isCurrent = 1; From 382ba590ad74b6cb1e7dc9c6e8146e7934a8db90 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 1 Jun 2021 11:47:05 -0400 Subject: [PATCH 4/4] Drop unused query from GET Build. --- src/lib/Hydra/Controller/Build.pm | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index 80ca15d2..c769edef 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -81,25 +81,6 @@ sub build_GET { } } - if ($build->finished) { - $c->stash->{prevBuilds} = [$c->model('DB::Builds')->search( - { jobset_id => $c->stash->{jobset}->id - , job => $c->stash->{job} - , 'me.system' => $build->system - , finished => 1 - , buildstatus => 0 - , 'me.id' => { '<=' => $build->id } - } - , { join => "actualBuildStep" - , "+select" => ["actualBuildStep.stoptime - actualBuildStep.starttime"] - , "+as" => ["actualBuildTime"] - , order_by => "me.id DESC" - , rows => 50 - } - ) - ]; - } - # Get the first eval of which this build was a part. ($c->stash->{nrEvals}) = $build->jobsetevals->search({ hasnewbuilds => 1 })->count; $c->stash->{eval} = getFirstEval($build);