From 2cdcc7f1885d708e3278ebe3ce42447afa5576e2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 5 Feb 2020 19:42:07 -0500 Subject: [PATCH 1/7] Jobs.jobset_id: make not-null --- src/lib/Hydra/Schema/Jobs.pm | 15 +++++---------- src/sql/hydra.sql | 2 +- src/sql/upgrade-63.sql | 7 +++++++ 3 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 src/sql/upgrade-63.sql diff --git a/src/lib/Hydra/Schema/Jobs.pm b/src/lib/Hydra/Schema/Jobs.pm index d4126e32..b00f3e0f 100644 --- a/src/lib/Hydra/Schema/Jobs.pm +++ b/src/lib/Hydra/Schema/Jobs.pm @@ -51,7 +51,7 @@ __PACKAGE__->table("jobs"); data_type: 'integer' is_foreign_key: 1 - is_nullable: 1 + is_nullable: 0 =head2 name @@ -66,7 +66,7 @@ __PACKAGE__->add_columns( "jobset", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "jobset_id", - { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, "name", { data_type => "text", is_nullable => 0 }, ); @@ -139,12 +139,7 @@ __PACKAGE__->belongs_to( "jobset", "Hydra::Schema::Jobsets", { id => "jobset_id" }, - { - is_deferrable => 0, - join_type => "LEFT", - on_delete => "CASCADE", - on_update => "NO ACTION", - }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, ); =head2 jobset_project_jobset @@ -197,7 +192,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:30:58 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dFusVjxb423gIEoadAw9sw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:33:28 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C5Tyh8Ke4yC6q7KIFVOHcQ 1; diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 8349e3be..f6c05bf7 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -146,7 +146,7 @@ create table JobsetInputAlts ( create table Jobs ( project text not null, jobset text not null, - jobset_id integer null, + jobset_id integer not null, name text not null, primary key (project, jobset, name), diff --git a/src/sql/upgrade-63.sql b/src/sql/upgrade-63.sql new file mode 100644 index 00000000..a7476e68 --- /dev/null +++ b/src/sql/upgrade-63.sql @@ -0,0 +1,7 @@ +-- Make the Jobs.jobset_id column NOT NULL. If this upgrade fails, +-- either the admin didn't run the backfiller or there is a bug. If +-- the admin ran the backfiller and there are null columns, it is +-- very important to figure out where the nullable columns came from. + +ALTER TABLE Jobs + ALTER COLUMN jobset_id SET NOT NULL; From 8ef08f1385dbfa530de49857610c902006210013 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 5 Feb 2020 15:25:52 -0500 Subject: [PATCH 2/7] Builds.jobset_id: make not-null --- src/lib/Hydra/Schema/Builds.pm | 15 +++++---------- src/sql/hydra.sql | 2 +- src/sql/upgrade-64.sql | 7 +++++++ 3 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 src/sql/upgrade-64.sql diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 53454867..937bd5fc 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -68,7 +68,7 @@ __PACKAGE__->table("builds"); data_type: 'integer' is_foreign_key: 1 - is_nullable: 1 + is_nullable: 0 =head2 job @@ -222,7 +222,7 @@ __PACKAGE__->add_columns( "jobset", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "jobset_id", - { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, "job", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "nixname", @@ -466,12 +466,7 @@ __PACKAGE__->belongs_to( "jobset", "Hydra::Schema::Jobsets", { id => "jobset_id" }, - { - is_deferrable => 0, - join_type => "LEFT", - on_delete => "CASCADE", - on_update => "NO ACTION", - }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, ); =head2 jobset_project_jobset @@ -578,8 +573,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:32:28 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RvrINOAowDcde8Nd9VD6rQ +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:34:25 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EEXlcKN/ydXJ129vT0jTUw __PACKAGE__->has_many( "dependents", diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index f6c05bf7..f1d143dc 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -170,7 +170,7 @@ create table Builds ( -- Info about the inputs. project text not null, jobset text not null, - jobset_id integer null, + jobset_id integer not null, job text not null, -- Info about the build result. diff --git a/src/sql/upgrade-64.sql b/src/sql/upgrade-64.sql new file mode 100644 index 00000000..76afa9e7 --- /dev/null +++ b/src/sql/upgrade-64.sql @@ -0,0 +1,7 @@ +-- Make the Builds.jobset_id column NOT NULL. If this upgrade fails, +-- either the admin didn't run the backfiller or there is a bug. If +-- the admin ran the backfiller and there are null columns, it is +-- very important to figure out where the nullable columns came from. + +ALTER TABLE Builds + ALTER COLUMN jobset_id SET NOT NULL; From 3c392b8cd8917e4cb403735904d1c4d71d26b4e1 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 5 Feb 2020 17:30:23 -0500 Subject: [PATCH 3/7] Jobsets.jobs: Fetch via Jobsets.id --- src/lib/Hydra/Schema/Jobsets.pm | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index fbbb253c..249d6aa4 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -441,10 +441,7 @@ Related object: L __PACKAGE__->has_many( "jobs", "Hydra::Schema::Jobs", - { - "foreign.jobset" => "self.name", - "foreign.project" => "self.project", - }, + { "foreign.jobset_id" => "self.id" }, undef, ); From 7c71f9df28ee8fe552f2aeaed734112542fd0638 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 5 Feb 2020 16:35:39 -0500 Subject: [PATCH 4/7] Jobsets.builds: Fetch via Jobsets.id --- src/lib/Hydra/Schema/Jobsets.pm | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 249d6aa4..572b5642 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -423,10 +423,7 @@ Related object: L __PACKAGE__->has_many( "builds", "Hydra::Schema::Builds", - { - "foreign.jobset" => "self.name", - "foreign.project" => "self.project", - }, + { "foreign.jobset_id" => "self.id" }, undef, ); From 66fbbd969251b96ef0a9958c3fa08ea32baa3808 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 5 Feb 2020 18:27:03 -0500 Subject: [PATCH 5/7] Jobs.builds: Fetch via Jobs.jobset_id --- src/lib/Hydra/Schema/Jobs.pm | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/lib/Hydra/Schema/Jobs.pm b/src/lib/Hydra/Schema/Jobs.pm index b00f3e0f..714497f1 100644 --- a/src/lib/Hydra/Schema/Jobs.pm +++ b/src/lib/Hydra/Schema/Jobs.pm @@ -195,4 +195,22 @@ __PACKAGE__->has_many( # Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:33:28 # DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:C5Tyh8Ke4yC6q7KIFVOHcQ +=head2 builds + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "builds", + "Hydra::Schema::Builds", + { + "foreign.job" => "self.name", + "foreign.jobset_id" => "self.jobset_id", + }, + undef, +); + 1; From f0f41eaaff4474c8433f1eaf2924357cf95e015a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 5 Feb 2020 17:02:56 -0500 Subject: [PATCH 6/7] LatestSucceededForJob{,set}: Filter with jobset_id --- src/lib/Hydra/Controller/Job.pm | 6 +++--- src/lib/Hydra/Controller/Jobset.pm | 2 +- src/lib/Hydra/Schema/Builds.pm | 4 ++-- src/script/hydra-eval-jobset | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 1b36e41f..bed362e6 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -82,7 +82,7 @@ sub overview : Chained('job') PathPart('') Args(0) { # If this is an aggregate job, then get its constituents. my @constituents = $c->model('DB::Builds')->search( { aggregate => { -in => $job->builds->search({}, { columns => ["id"], order_by => "id desc", rows => 15 })->as_query } }, - { join => 'aggregateconstituents_constituents', + { join => 'aggregateconstituents_constituents', columns => ['id', 'job', 'finished', 'buildstatus'], +select => ['aggregateconstituents_constituents.aggregate'], +as => ['aggregate'] @@ -99,7 +99,7 @@ sub overview : Chained('job') PathPart('') Args(0) { foreach my $agg (keys %$aggregates) { # FIXME: could be done in one query. - $aggregates->{$agg}->{build} = + $aggregates->{$agg}->{build} = $c->model('DB::Builds')->find({id => $agg}, {columns => [@buildListColumns]}) or die; } @@ -172,7 +172,7 @@ sub get_builds : Chained('job') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; $c->stash->{allBuilds} = $c->stash->{job}->builds; $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceededForJob') - ->search({}, {bind => [$c->stash->{project}->name, $c->stash->{jobset}->name, $c->stash->{job}->name]}); + ->search({}, {bind => [$c->stash->{jobset}->name, $c->stash->{job}->name]}); $c->stash->{channelBaseName} = $c->stash->{project}->name . "-" . $c->stash->{jobset}->name . "-" . $c->stash->{job}->name; } diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 75284c10..fdf5e1c3 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -162,7 +162,7 @@ sub get_builds : Chained('jobsetChain') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; $c->stash->{allBuilds} = $c->stash->{jobset}->builds; $c->stash->{latestSucceeded} = $c->model('DB')->resultset('LatestSucceededForJobset') - ->search({}, {bind => [$c->stash->{project}->name, $c->stash->{jobset}->name]}); + ->search({}, {bind => [$c->stash->{jobset}->name]}); $c->stash->{channelBaseName} = $c->stash->{project}->name . "-" . $c->stash->{jobset}->name; } diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 937bd5fc..fdbc5ed6 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -637,8 +637,8 @@ QUERY makeQueries('', ""); makeQueries('ForProject', "and project = ?"); -makeQueries('ForJobset', "and project = ? and jobset = ?"); -makeQueries('ForJob', "and project = ? and jobset = ? and job = ?"); +makeQueries('ForJobset', "and jobset_id = (select id from jobsets j where j.name = ?)"); +makeQueries('ForJob', "and jobset_id = (select id from jobsets j where j.name = ?) and job = ?"); my %hint = ( diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index d0cc3e8a..97da1084 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -143,7 +143,7 @@ sub fetchInputSystemBuild { $jobsetName ||= $jobset->name; my @latestBuilds = $db->resultset('LatestSucceededForJob') - ->search({}, {bind => [$projectName, $jobsetName, $jobName]}); + ->search({}, {bind => [$jobsetName, $jobName]}); my @validBuilds = (); foreach my $build (@latestBuilds) { From 2637a7ad76e67b956c835cb8c7c8a75437c9c31e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Sun, 9 Feb 2020 15:48:11 -0500 Subject: [PATCH 7/7] Builds: index literally what latest-finished queries --- src/sql/hydra.sql | 1 + src/sql/upgrade-65.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 src/sql/upgrade-65.sql diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index f1d143dc..1f4288f7 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -682,6 +682,7 @@ create index IndexBuildsOnProject on Builds(project); create index IndexBuildsOnTimestamp on Builds(timestamp); create index IndexBuildsOnFinishedStopTime on Builds(finished, stoptime DESC); create index IndexBuildsOnJobFinishedId on builds(project, jobset, job, system, finished, id DESC); +create index IndexFinishedSuccessfulBuilds on Builds(id DESC, buildstatus, finished, job, jobset_id) where buildstatus = 0 and finished = 1; create index IndexBuildsOnDrvPath on Builds(drvPath); create index IndexCachedHgInputsOnHash on CachedHgInputs(uri, branch, sha256hash); create index IndexCachedGitInputsOnHash on CachedGitInputs(uri, branch, sha256hash); diff --git a/src/sql/upgrade-65.sql b/src/sql/upgrade-65.sql new file mode 100644 index 00000000..05986887 --- /dev/null +++ b/src/sql/upgrade-65.sql @@ -0,0 +1,4 @@ +-- Index more exactly what the latest-finished query looks for. +create index IndexFinishedSuccessfulBuilds + on Builds(id DESC, buildstatus, finished, job, jobset_id) + where buildstatus = 0 and finished = 1;