From d58142b3f004eb74095165dbd72e0d59e0cf18ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Aug 2013 01:59:29 +0200 Subject: [PATCH] Store aggregate members in the database For presentation purposes, we need to know what builds are part of an aggregate build. So at evaluation time, look at the "members" attribute, find the corresponding builds in the eval, and create a mapping in the AggregateMembers table. --- src/c/hydra-eval-jobs.cc | 17 ++++ src/lib/Hydra/Controller/Build.pm | 2 +- src/lib/Hydra/Helper/AddBuilds.pm | 24 ++--- src/lib/Hydra/Schema/AggregateMembers.pm | 111 +++++++++++++++++++++++ src/lib/Hydra/Schema/Builds.pm | 54 ++++++++++- src/script/hydra-evaluator | 31 +++++-- src/sql/hydra.sql | 7 ++ src/sql/upgrade-19.sql | 5 + 8 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 src/lib/Hydra/Schema/AggregateMembers.pm create mode 100644 src/sql/upgrade-19.sql diff --git a/src/c/hydra-eval-jobs.cc b/src/c/hydra-eval-jobs.cc index 4ff01057..ca2fb7fb 100644 --- a/src/c/hydra-eval-jobs.cc +++ b/src/c/hydra-eval-jobs.cc @@ -160,6 +160,23 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, } xmlAttrs["maintainers"] = maintainers; + /* If this is an aggregate, then get its members. */ + Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); + if (a != v.attrs->end() && state.forceBool(*a->value)) { + Bindings::iterator a = v.attrs->find(state.symbols.create("members")); + if (a == v.attrs->end()) + throw EvalError("derivation must have a ‘members’ attribute"); + PathSet context; + state.coerceToString(*a->value, context, true, false); + PathSet drvs; + foreach (PathSet::iterator, i, context) + if (i->at(0) == '!') { + size_t index = i->find("!", 1); + drvs.insert(string(*i, index + 1)); + } + xmlAttrs["members"] = concatStringsSep(" ", drvs); + } + /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index cded877f..6fef3ed8 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -578,7 +578,7 @@ sub clone_submit : Chained('buildChain') PathPart('clone/submit') Args(0) { my %currentBuilds; my $newBuild = checkBuild( - $c->model('DB'), $build->project, $build->jobset, + $c->model('DB'), $build->jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds, undef, {}, $c->hydra_plugins); error($c, "This build has already been performed.") unless $newBuild; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 6f51b98d..f1e21b49 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -288,13 +288,9 @@ sub evalJobs { my $validJob = 1; foreach my $arg (@{$job->{arg}}) { my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}]; - if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) { - $validJob = 0; - } - } - if ($validJob) { - push(@filteredJobs, $job); + $validJob = 0 if $input->{type} eq "sysbuild" && $input->{system} ne $job->{system}; } + push(@filteredJobs, $job) if $validJob; } $jobs->{job} = \@filteredJobs; @@ -390,7 +386,7 @@ sub getPrevJobsetEval { # Check whether to add the build described by $buildInfo. sub checkBuild { - my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap, $plugins) = @_; + my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; my @outputNames = sort keys %{$buildInfo->{output}}; die unless scalar @outputNames; @@ -411,9 +407,7 @@ sub checkBuild { my $build; txn_do($db, sub { - my $job = $jobset->jobs->update_or_create( - { name => $jobName - }); + my $job = $jobset->jobs->update_or_create({ name => $jobName }); # Don't add a build that has already been scheduled for this # job, or has been built but is still a "current" build for @@ -434,19 +428,19 @@ sub checkBuild { # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. - { project => $project->name, jobset => $jobset->name, job => $job->name, + { project => $jobset->project->name, jobset => $jobset->name, job => $jobName, name => $firstOutputName, path => $firstOutputPath }, { rows => 1, columns => ['id'], join => ['buildoutputs'] }); if (defined $prevBuild) { print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; - $buildIds->{$prevBuild->id} = 0; + $buildMap->{$prevBuild->id} = { new => 0, drvPath => $drvPath }; return; } } # Prevent multiple builds with the same (job, outPath) from # being added. - my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath}; + my $prev = $$jobOutPathMap{$jobName . "\t" . $firstOutputPath}; if (defined $prev) { print STDERR " already scheduled as build ", $prev, "\n"; return; @@ -512,8 +506,8 @@ sub checkBuild { $build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} }) foreach @outputNames; - $buildIds->{$build->id} = 1; - $$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id; + $buildMap->{$build->id} = { new => 1, drvPath => $drvPath }; + $$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id; if ($build->iscachedbuild) { print STDERR " marked as cached build ", $build->id, "\n"; diff --git a/src/lib/Hydra/Schema/AggregateMembers.pm b/src/lib/Hydra/Schema/AggregateMembers.pm new file mode 100644 index 00000000..4a037663 --- /dev/null +++ b/src/lib/Hydra/Schema/AggregateMembers.pm @@ -0,0 +1,111 @@ +use utf8; +package Hydra::Schema::AggregateMembers; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::AggregateMembers + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("AggregateMembers"); + +=head1 ACCESSORS + +=head2 aggregate + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 member + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "aggregate", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "member", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("aggregate", "member"); + +=head1 RELATIONS + +=head2 aggregate + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "aggregate", + "Hydra::Schema::Builds", + { id => "aggregate" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + +=head2 member + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "member", + "Hydra::Schema::Builds", + { id => "member" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jHJtO2baXiprv0OcWCLZ+w + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index ea27d579..6280e08f 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -288,6 +288,36 @@ __PACKAGE__->set_primary_key("id"); =head1 RELATIONS +=head2 aggregatemembers_aggregates + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "aggregatemembers_aggregates", + "Hydra::Schema::AggregateMembers", + { "foreign.aggregate" => "self.id" }, + undef, +); + +=head2 aggregatemembers_members + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "aggregatemembers_members", + "Hydra::Schema::AggregateMembers", + { "foreign.member" => "self.id" }, + undef, +); + =head2 buildinputs_builds Type: has_many @@ -468,9 +498,29 @@ __PACKAGE__->has_many( undef, ); +=head2 aggregates -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:isCEXACY/PwkvgKHcXvAIg +Type: many_to_many + +Composing rels: L -> aggregate + +=cut + +__PACKAGE__->many_to_many("aggregates", "aggregatemembers_members", "aggregate"); + +=head2 members + +Type: many_to_many + +Composing rels: L -> member + +=cut + +__PACKAGE__->many_to_many("members", "aggregatemembers_members", "member"); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9jqsol/evbHYjusT09hLtw __PACKAGE__->has_many( "dependents", diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index 5bfed9e3..a6dfde46 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -144,11 +144,11 @@ sub checkJobsetWrapped { $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); # Schedule each successfully evaluated job. - my %buildIds; + my %buildMap; foreach my $job (permute @{$jobs->{job}}) { next if $job->{jobName} eq ""; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; - checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap, $plugins); + checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); } # Update the last checked times and error messages for each @@ -162,8 +162,8 @@ sub checkJobsetWrapped { foreach $jobset->jobs->all; my $hasNewBuilds = 0; - while (my ($id, $new) = each %buildIds) { - $hasNewBuilds = 1 if $new; + while (my ($id, $x) = each %buildMap) { + $hasNewBuilds = 1 if $x->{new}; } my $ev = $jobset->jobsetevals->create( @@ -172,12 +172,29 @@ sub checkJobsetWrapped { , checkouttime => abs($checkoutStop - $checkoutStart) , evaltime => abs($evalStop - $evalStart) , hasnewbuilds => $hasNewBuilds - , nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef + , nrbuilds => $hasNewBuilds ? scalar(keys %buildMap) : undef }); if ($hasNewBuilds) { - while (my ($id, $new) = each %buildIds) { - $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); + # Create JobsetEvalMembers mappings. + my %drvPathToId; + while (my ($id, $x) = each %buildMap) { + $ev->jobsetevalmembers->create({ build => $id, isnew => $x->{new} }); + $drvPathToId{$x->{drvPath}} = $id; + } + + # Create AggregateMembers mappings. + foreach my $job (@{$jobs->{job}}) { + next unless $job->{members}; + my $id = $drvPathToId{$job->{drvPath}} or die; + foreach my $drvPath (split / /, $job->{members}) { + my $member = $drvPathToId{$drvPath}; + if (defined $member) { + $db->resultset('AggregateMembers')->update_or_create({aggregate => $id, member => $member}); + } else { + warn "aggregate job ‘$job->{jobName}’ has a member ‘$drvPath’ that doesn't correspond to a Hydra build\n"; + } + } } foreach my $name (keys %{$inputInfo}) { diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 3001ab10..ee601c62 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -514,6 +514,13 @@ create table NewsItems ( ); +create table AggregateMembers ( + aggregate integer not null references Builds(id) on delete cascade, + member integer not null references Builds(id) on delete cascade, + primary key (aggregate, member) +); + + -- Cache of the number of finished builds. create table NrBuilds ( what text primary key not null, diff --git a/src/sql/upgrade-19.sql b/src/sql/upgrade-19.sql new file mode 100644 index 00000000..3d1e849b --- /dev/null +++ b/src/sql/upgrade-19.sql @@ -0,0 +1,5 @@ +create table AggregateMembers ( + aggregate integer not null references Builds(id) on delete cascade, + member integer not null references Builds(id) on delete cascade, + primary key (aggregate, member) +);