diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 013fd09d..53454867 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -64,6 +64,12 @@ __PACKAGE__->table("builds"); is_foreign_key: 1 is_nullable: 0 +=head2 jobset_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + =head2 job data_type: 'text' @@ -215,6 +221,8 @@ __PACKAGE__->add_columns( { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "jobset", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "jobset_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "job", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "nixname", @@ -457,6 +465,26 @@ Related object: L __PACKAGE__->belongs_to( "jobset", "Hydra::Schema::Jobsets", + { id => "jobset_id" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "NO ACTION", + }, +); + +=head2 jobset_project_jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset_project_jobset", + "Hydra::Schema::Jobsets", { name => "jobset", project => "project" }, { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, ); @@ -550,8 +578,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:22:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:H3hs+zEywsUmwTWKfSE8wQ +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:32:28 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:RvrINOAowDcde8Nd9VD6rQ __PACKAGE__->has_many( "dependents", diff --git a/src/lib/Hydra/Schema/Jobs.pm b/src/lib/Hydra/Schema/Jobs.pm index bdecef3a..d4126e32 100644 --- a/src/lib/Hydra/Schema/Jobs.pm +++ b/src/lib/Hydra/Schema/Jobs.pm @@ -47,6 +47,12 @@ __PACKAGE__->table("jobs"); is_foreign_key: 1 is_nullable: 0 +=head2 jobset_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + =head2 name data_type: 'text' @@ -59,6 +65,8 @@ __PACKAGE__->add_columns( { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "jobset", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "jobset_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "name", { data_type => "text", is_nullable => 0 }, ); @@ -130,6 +138,26 @@ Related object: L __PACKAGE__->belongs_to( "jobset", "Hydra::Schema::Jobsets", + { id => "jobset_id" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "NO ACTION", + }, +); + +=head2 jobset_project_jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset_project_jobset", + "Hydra::Schema::Jobsets", { name => "jobset", project => "project" }, { is_deferrable => 0, on_delete => "CASCADE", on_update => "CASCADE" }, ); @@ -169,7 +197,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:22:36 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:sYa6dZNK+stMAnTH0Tmn8A +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-06 12:30:58 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:dFusVjxb423gIEoadAw9sw 1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 9e9d4773..fbbb253c 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -40,6 +40,13 @@ __PACKAGE__->table("jobsets"); data_type: 'text' is_nullable: 0 +=head2 id + + data_type: 'integer' + is_auto_increment: 1 + is_nullable: 0 + sequence: 'jobsets_id_seq' + =head2 project data_type: 'text' @@ -153,6 +160,13 @@ __PACKAGE__->table("jobsets"); __PACKAGE__->add_columns( "name", { data_type => "text", is_nullable => 0 }, + "id", + { + data_type => "integer", + is_auto_increment => 1, + is_nullable => 0, + sequence => "jobsets_id_seq", + }, "project", { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, "description", @@ -209,6 +223,20 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("project", "name"); +=head1 UNIQUE CONSTRAINTS + +=head2 C + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->add_unique_constraint("jobsets_id_unique", ["id"]); + =head1 RELATIONS =head2 buildmetrics @@ -229,7 +257,7 @@ __PACKAGE__->has_many( undef, ); -=head2 builds +=head2 builds_jobset_ids Type: has_many @@ -238,7 +266,22 @@ Related object: L =cut __PACKAGE__->has_many( - "builds", + "builds_jobset_ids", + "Hydra::Schema::Builds", + { "foreign.jobset_id" => "self.id" }, + undef, +); + +=head2 builds_project_jobsets + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "builds_project_jobsets", "Hydra::Schema::Builds", { "foreign.jobset" => "self.name", @@ -247,7 +290,7 @@ __PACKAGE__->has_many( undef, ); -=head2 jobs +=head2 jobs_jobset_ids Type: has_many @@ -256,7 +299,22 @@ Related object: L =cut __PACKAGE__->has_many( - "jobs", + "jobs_jobset_ids", + "Hydra::Schema::Jobs", + { "foreign.jobset_id" => "self.id" }, + undef, +); + +=head2 jobs_project_jobsets + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobs_project_jobsets", "Hydra::Schema::Jobs", { "foreign.jobset" => "self.name", @@ -350,8 +408,49 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-09 15:21:11 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:FVP1/AWjdKTlY6djrG592A +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2020-02-09 15:32:17 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:P8+t7rgpOqkGwRdM2b+3Bw + + +=head2 builds + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "builds", + "Hydra::Schema::Builds", + { + "foreign.jobset" => "self.name", + "foreign.project" => "self.project", + }, + undef, +); + +=head2 jobs + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "jobs", + "Hydra::Schema::Jobs", + { + "foreign.jobset" => "self.name", + "foreign.project" => "self.project", + }, + undef, +); + +__PACKAGE__->add_column( + "+id" => { retrieve_on_insert => 1 } +); my %hint = ( columns => [ diff --git a/src/script/Makefile.am b/src/script/Makefile.am index 5852bc85..9deb6f29 100644 --- a/src/script/Makefile.am +++ b/src/script/Makefile.am @@ -3,6 +3,7 @@ EXTRA_DIST = \ hydra-eval-guile-jobs.in distributable_scripts = \ + hydra-backfill-ids \ hydra-init \ hydra-eval-jobset \ hydra-server \ diff --git a/src/script/hydra-backfill-ids b/src/script/hydra-backfill-ids new file mode 100755 index 00000000..d9fc362a --- /dev/null +++ b/src/script/hydra-backfill-ids @@ -0,0 +1,164 @@ +#! /usr/bin/env perl + +use strict; +use utf8; +use Hydra::Model::DB; + +STDOUT->autoflush(); +STDERR->autoflush(1); +binmode STDERR, ":encoding(utf8)"; + +my $db = Hydra::Model::DB->new(); +my $vacuum = $db->storage->dbh->prepare("VACUUM;"); + +my $dryRun = defined $ENV{'HYDRA_DRY_RUN'}; + +my $batchSize = 10000; +my $iterationsPerVacuum = 500; + +sub backfillJobsJobsetId { + my ($skipLocked) = @_; + my $logPrefix; + + if ($skipLocked) { + $logPrefix = "(pass 1/2)"; + } else { + $logPrefix = "(pass 2/2)"; + } + + print STDERR "$logPrefix Backfilling Jobs records where jobset_id is NULL...\n"; + + my $totalToGoSth = $db->storage->dbh->prepare(<execute(); + my ($totalToGo) = $totalToGoSth->fetchrow_array; + + my $skipLockedStmt = $skipLocked ? "FOR UPDATE SKIP LOCKED" : ""; + my $update10kJobs = $db->storage->dbh->prepare(<execute($batchSize); + print STDERR "$logPrefix (batch #$iteration; $totalToGo remaining) Jobs.jobset_id: affected $affected rows...\n"; + $totalToGo -= $affected; + + if ($iteration % $iterationsPerVacuum == 0) { + print STDERR "$logPrefix (batch #$iteration) Vacuuming...\n"; + $vacuum->execute(); + } + } while ($affected > 0); + + + if ($skipLocked) { + backfillJobsJobsetId(0); + } +} + + +sub backfillBuildsJobsetId { + my ($skipLocked) = @_; + my $logPrefix; + + if ($skipLocked) { + $logPrefix = "(pass 1/2)"; + print STDERR "$logPrefix Backfilling unlocked Builds records where jobset_id is NULL...\n"; + } else { + $logPrefix = "(pass 2/2)"; + print STDERR "$logPrefix Backfilling all Builds records where jobset_id is NULL...\n"; + } + + my $skipLockedStmt = $skipLocked ? "FOR UPDATE SKIP LOCKED" : ""; + my $update10kBuilds = $db->storage->dbh->prepare(<<"QUERY"); +WITH updateprogress AS ( + UPDATE builds + SET jobset_id = ( + SELECT jobsets.id + FROM jobsets + WHERE jobsets.name = builds.jobset + AND jobsets.project = builds.project + ) + WHERE builds.id in ( + SELECT buildprime.id + FROM builds buildprime + WHERE buildprime.jobset_id IS NULL + AND buildprime.id >= ? + ORDER BY buildprime.id + $skipLockedStmt + LIMIT ? + ) + RETURNING id +) +SELECT + count(*) AS affected, + max(updateprogress.id) AS highest_id +FROM updateprogress; + +QUERY + + my $lowestNullIdSth = $db->storage->dbh->prepare(<execute(); + my ($highestId) = $lowestNullIdSth->fetchrow_array; + + my $totalToGoSth = $db->storage->dbh->prepare(<= ? +QUERY + $totalToGoSth->execute($highestId); + my ($totalToGo) = $totalToGoSth->fetchrow_array; + + print STDERR "$logPrefix Total Builds records without a jobset_id: $totalToGo, starting at $highestId\n"; + + my $iteration = 0; + my $affected; + do { + my $previousHighId = $highestId; + $iteration++; + $update10kBuilds->execute($highestId, $batchSize); + ($affected, $highestId) = $update10kBuilds->fetchrow_array; + + print STDERR "$logPrefix (batch #$iteration; $totalToGo remaining) Builds.jobset_id: affected $affected rows; max ID: $previousHighId -> $highestId\n"; + $totalToGo -= $affected; + + if ($iteration % $iterationsPerVacuum == 0) { + print STDERR "$logPrefix (batch #$iteration) Vacuuming...\n"; + $vacuum->execute(); + } + } while ($affected > 0); + + if ($skipLocked) { + backfillBuildsJobsetId(0); + } +} + +die "syntax: $0\n" unless @ARGV == 0; + +print STDERR "Beginning with a VACUUM\n"; +$vacuum->execute(); + +backfillJobsJobsetId(1); +backfillBuildsJobsetId(1); + +print STDERR "Ending with a VACUUM\n"; +$vacuum->execute(); diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index fcc60a6c..d0cc3e8a 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -417,7 +417,12 @@ sub checkBuild { my $build; txn_do($db, sub { - my $job = $jobset->jobs->update_or_create({ name => $jobName }); + my $job = $jobset->jobs->update_or_create({ + name => $jobName, + jobset_id => $jobset->id, + project => $jobset->project, + jobset => $jobset->name, + }); # Don't add a build that has already been scheduled for this # job, or has been built but is still a "current" build for @@ -464,6 +469,9 @@ sub checkBuild { # Add the build to the database. $build = $job->builds->create( { timestamp => $time + , project => $jobset->project + , jobset => $jobset->name + , jobset_id => $jobset->id , description => null($buildInfo->{description}) , license => null($buildInfo->{license}) , homepage => null($buildInfo->{homepage}) diff --git a/src/script/hydra-init b/src/script/hydra-init index d813d9fe..1a5f2439 100755 --- a/src/script/hydra-init +++ b/src/script/hydra-init @@ -44,6 +44,17 @@ my @versions = $db->resultset('SchemaVersion')->all; die "couldn't get Hydra schema version!" if scalar @versions != 1; my $schemaVersion = $versions[0]->version; +if ($schemaVersion <= 60) { + print STDERR <